Encapsulate Opus frames in Ogg during audio offload

PiperOrigin-RevId: 508053559
This commit is contained in:
michaelkatz 2023-02-08 13:32:52 +00:00 committed by microkatz
parent 16db2bd0a1
commit 4854e771d7
8 changed files with 644 additions and 1 deletions

View File

@ -41,6 +41,8 @@
* Video:
* Map HEVC HDR10 format to `HEVCProfileMain10HDR10` instead of
`HEVCProfileMain10`.
* Encapsulate Opus frames in Ogg packets in direct playbacks
(passthrough).
* Text:
* Fix `TextRenderer` passing an invalid (negative) index to
`Subtitle.getEventTime` if a subtitle file contains no cues.

View File

@ -1699,7 +1699,7 @@ public final class DefaultAudioSink implements AudioSink {
: (Ac3Util.parseTrueHdSyncframeAudioSampleCount(buffer, syncframeOffset)
* Ac3Util.TRUEHD_RECHUNK_SAMPLE_COUNT);
case C.ENCODING_OPUS:
return OpusUtil.parsePacketAudioSampleCount(buffer);
return OpusUtil.parseOggPacketAudioSampleCount(buffer);
case C.ENCODING_PCM_16BIT:
case C.ENCODING_PCM_16BIT_BIG_ENDIAN:
case C.ENCODING_PCM_24BIT:

View File

@ -0,0 +1,166 @@
/*
* Copyright (C) 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.exoplayer.audio;
import static androidx.media3.common.audio.AudioProcessor.EMPTY_BUFFER;
import static androidx.media3.common.util.Assertions.checkNotNull;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import androidx.media3.decoder.DecoderInputBuffer;
import androidx.media3.extractor.OpusUtil;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
/** A packetizer that encapsulates OPUS audio encodings in OGG packets. */
@UnstableApi
public final class OggOpusAudioPacketizer {
/** ID Header and Comment Header pages are 0 and 1 respectively */
private static final int FIRST_AUDIO_SAMPLE_PAGE_SEQUENCE = 2;
private ByteBuffer outputBuffer;
private int pageSequenceNumber;
private int granulePosition;
/** Creates an instance. */
public OggOpusAudioPacketizer() {
outputBuffer = EMPTY_BUFFER;
granulePosition = 0;
pageSequenceNumber = FIRST_AUDIO_SAMPLE_PAGE_SEQUENCE;
}
/**
* Packetizes the audio data between the position and limit of the {@code inputBuffer}.
*
* @param inputBuffer The input buffer to packetize. It must be a direct {@link ByteBuffer} with
* LITTLE_ENDIAN order. The contents will be overwritten with the Ogg packet. The caller
* retains ownership of the provided buffer.
*/
public void packetize(DecoderInputBuffer inputBuffer) {
checkNotNull(inputBuffer.data);
if (inputBuffer.data.limit() - inputBuffer.data.position() == 0) {
return;
}
outputBuffer = packetizeInternal(inputBuffer.data);
inputBuffer.clear();
inputBuffer.ensureSpaceForWrite(outputBuffer.remaining());
inputBuffer.data.put(outputBuffer);
inputBuffer.flip();
}
/** Resets the packetizer. */
public void reset() {
outputBuffer = EMPTY_BUFFER;
granulePosition = 0;
pageSequenceNumber = FIRST_AUDIO_SAMPLE_PAGE_SEQUENCE;
}
/**
* Fill outputBuffer with an Ogg packet encapsulating the inputBuffer.
*
* @param inputBuffer contains Opus to wrap in Ogg packet
* @return {@link ByteBuffer} containing Ogg packet
*/
private ByteBuffer packetizeInternal(ByteBuffer inputBuffer) {
int position = inputBuffer.position();
int limit = inputBuffer.limit();
int inputBufferSize = limit - position;
// inputBufferSize divisible by 255 requires extra '0' terminating lacing value
int numSegments = (inputBufferSize + 255) / 255;
int headerSize = 27 + numSegments;
int outputPacketSize = headerSize + inputBufferSize;
// Resample the little endian input and update the output buffers.
ByteBuffer buffer = replaceOutputBuffer(outputPacketSize);
// Capture Pattern for Page [OggS]
buffer.put((byte) 'O');
buffer.put((byte) 'g');
buffer.put((byte) 'g');
buffer.put((byte) 'S');
// StreamStructure Version
buffer.put((byte) 0);
// header_type_flag
buffer.put((byte) 0x00);
// granule_position
int numSamples = OpusUtil.parsePacketAudioSampleCount(inputBuffer);
granulePosition += numSamples;
buffer.putLong(granulePosition);
// bitstream_serial_number
buffer.putInt(0);
// page_sequence_number
buffer.putInt(pageSequenceNumber);
pageSequenceNumber++;
// CRC_checksum
buffer.putInt(0);
// number_page_segments
buffer.put((byte) numSegments);
// Segment_table
int bytesLeft = inputBufferSize;
for (int i = 0; i < numSegments; i++) {
if (bytesLeft >= 255) {
buffer.put((byte) 255);
bytesLeft -= 255;
} else {
buffer.put((byte) bytesLeft);
bytesLeft = 0;
}
}
for (int i = position; i < limit; i++) {
buffer.put(inputBuffer.get(i));
}
inputBuffer.position(inputBuffer.limit());
buffer.flip();
int checksum =
Util.crc32(
buffer.array(),
buffer.arrayOffset(),
buffer.limit() - buffer.position(),
/* initialValue= */ 0);
buffer.putInt(22, checksum);
buffer.position(0);
return buffer;
}
/**
* Replaces the current output buffer with a buffer of at least {@code size} bytes and returns it.
* Callers should write to the returned buffer then {@link ByteBuffer#flip()} it so it can be read
* via buffer.
*/
private ByteBuffer replaceOutputBuffer(int size) {
if (outputBuffer.capacity() < size) {
outputBuffer = ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN);
} else {
outputBuffer.clear();
}
return outputBuffer;
}
}

View File

@ -67,6 +67,7 @@ import androidx.media3.exoplayer.DecoderReuseEvaluation.DecoderDiscardReasons;
import androidx.media3.exoplayer.ExoPlaybackException;
import androidx.media3.exoplayer.FormatHolder;
import androidx.media3.exoplayer.analytics.PlayerId;
import androidx.media3.exoplayer.audio.OggOpusAudioPacketizer;
import androidx.media3.exoplayer.drm.DrmSession;
import androidx.media3.exoplayer.drm.DrmSession.DrmSessionException;
import androidx.media3.exoplayer.drm.FrameworkCryptoConfig;
@ -309,6 +310,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
private final long[] pendingOutputStreamStartPositionsUs;
private final long[] pendingOutputStreamOffsetsUs;
private final long[] pendingOutputStreamSwitchTimesUs;
private final OggOpusAudioPacketizer oggOpusAudioPacketizer;
@Nullable private Format inputFormat;
@Nullable private Format outputFormat;
@ -410,6 +412,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
// endianness.
bypassBatchBuffer.ensureSpaceForWrite(/* length= */ 0);
bypassBatchBuffer.data.order(ByteOrder.nativeOrder());
oggOpusAudioPacketizer = new OggOpusAudioPacketizer();
codecOperatingRate = CODEC_OPERATING_RATE_UNSET;
codecAdaptationWorkaroundMode = ADAPTATION_WORKAROUND_MODE_NEVER;
@ -728,6 +731,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
bypassSampleBuffer.clear();
bypassSampleBufferPending = false;
bypassEnabled = false;
oggOpusAudioPacketizer.reset();
}
protected void releaseCodec() {
@ -2313,6 +2317,13 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
}
// Try to append the buffer to the batch buffer.
bypassSampleBuffer.flip();
if (inputFormat != null
&& inputFormat.sampleMimeType != null
&& inputFormat.sampleMimeType.equals(MimeTypes.AUDIO_OPUS)) {
oggOpusAudioPacketizer.packetize(bypassSampleBuffer);
}
if (!bypassBatchBuffer.append(bypassSampleBuffer)) {
bypassSampleBufferPending = true;
return;

View File

@ -0,0 +1,168 @@
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.exoplayer.e2etest;
import android.content.Context;
import androidx.annotation.Nullable;
import androidx.media3.common.Format;
import androidx.media3.common.MediaItem;
import androidx.media3.common.Player;
import androidx.media3.exoplayer.DefaultRenderersFactory;
import androidx.media3.exoplayer.ExoPlayer;
import androidx.media3.exoplayer.audio.AudioCapabilities;
import androidx.media3.exoplayer.audio.AudioSink;
import androidx.media3.exoplayer.audio.DefaultAudioSink;
import androidx.media3.exoplayer.audio.ForwardingAudioSink;
import androidx.media3.test.utils.DumpFileAsserts;
import androidx.media3.test.utils.Dumper;
import androidx.media3.test.utils.FakeClock;
import androidx.media3.test.utils.robolectric.ShadowMediaCodecConfig;
import androidx.media3.test.utils.robolectric.TestPlayerRunHelper;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
@RunWith(AndroidJUnit4.class)
public class OggOpusPlaybackTest {
public static final String INPUT_FILE = "bear.opus";
@Rule
public ShadowMediaCodecConfig mediaCodecConfig =
ShadowMediaCodecConfig.forAllSupportedMimeTypes();
@Test
public void checkOggOpusEncodings() throws Exception {
Context applicationContext = ApplicationProvider.getApplicationContext();
OffloadRenderersFactory offloadRenderersFactory =
new OffloadRenderersFactory(applicationContext);
ExoPlayer player =
new ExoPlayer.Builder(applicationContext, offloadRenderersFactory)
.setClock(new FakeClock(/* isAutoAdvancing= */ true))
.build();
player.setMediaItem(MediaItem.fromUri("asset:///media/ogg/" + INPUT_FILE));
player.prepare();
player.play();
TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED);
player.release();
DumpFileAsserts.assertOutput(
applicationContext,
offloadRenderersFactory,
"playbackdumps/ogg/" + INPUT_FILE + ".oggOpus.dump");
}
private static class OffloadRenderersFactory extends DefaultRenderersFactory
implements Dumper.Dumpable {
private DumpingAudioSink dumpingAudioSink;
/**
* @param context A {@link Context}.
*/
public OffloadRenderersFactory(Context context) {
super(context);
setEnableAudioOffload(true);
}
@Override
protected AudioSink buildAudioSink(
Context context,
boolean enableFloatOutput,
boolean enableAudioTrackPlaybackParams,
boolean enableOffload) {
dumpingAudioSink =
new DumpingAudioSink(
new DefaultAudioSink.Builder()
.setAudioCapabilities(AudioCapabilities.getCapabilities(context))
.setEnableFloatOutput(enableFloatOutput)
.setEnableAudioTrackPlaybackParams(enableAudioTrackPlaybackParams)
.setOffloadMode(DefaultAudioSink.OFFLOAD_MODE_ENABLED_GAPLESS_REQUIRED)
.build());
return dumpingAudioSink;
}
@Override
public void dump(Dumper dumper) {
dumpingAudioSink.dump(dumper);
}
}
private static class DumpingAudioSink extends ForwardingAudioSink implements Dumper.Dumpable {
/** All handleBuffer interactions recorded with this audio sink. */
private final List<CapturedInputBuffer> capturedInteractions;
public DumpingAudioSink(AudioSink sink) {
super(sink);
capturedInteractions = new ArrayList<>();
}
@Override
public void configure(
Format inputFormat, int specifiedBufferSize, @Nullable int[] outputChannels)
throws ConfigurationException {
// Bypass configure of base DefaultAudioSink
}
@Override
public boolean supportsFormat(Format format) {
return true;
}
@Override
public boolean handleBuffer(
ByteBuffer buffer, long presentationTimeUs, int encodedAccessUnitCount)
throws InitializationException, WriteException {
capturedInteractions.add(
new CapturedInputBuffer(peekBytes(buffer, 0, buffer.limit() - buffer.position())));
return true;
}
@Override
public void dump(Dumper dumper) {
dumper.startBlock("SinkDump (OggOpus)");
dumper.add("buffers.length", capturedInteractions.size());
for (int i = 0; i < capturedInteractions.size(); i++) {
dumper.add("buffers[" + i + "]", capturedInteractions.get(i).contents);
}
dumper.endBlock();
}
private byte[] peekBytes(ByteBuffer buffer, int offset, int size) {
int originalPosition = buffer.position();
buffer.position(offset);
byte[] bytes = new byte[size];
buffer.get(bytes);
buffer.position(originalPosition);
return bytes;
}
}
/** Data record */
private static class CapturedInputBuffer {
private final byte[] contents;
private CapturedInputBuffer(byte[] contents) {
this.contents = contents;
}
}
}

View File

@ -66,6 +66,25 @@ public class OpusUtil {
return initializationData;
}
/**
* Returns the number of audio samples in the given Ogg encapuslated Opus packet.
*
* <p>The buffer's position is not modified.
*
* @param buffer The audio packet.
* @return Returns the number of audio samples in the packet.
*/
public static int parseOggPacketAudioSampleCount(ByteBuffer buffer) {
// RFC 3433 section 6 - The Ogg page format.
int numPageSegments = buffer.get(/* index= */ 26);
int indexFirstOpusPacket = 27 + numPageSegments; // Skip Ogg header and segment table.
long packetDurationUs =
getPacketDurationUs(
buffer.get(indexFirstOpusPacket),
buffer.limit() > 1 ? buffer.get(indexFirstOpusPacket + 1) : 0);
return (int) (packetDurationUs * SAMPLE_RATE / C.MICROS_PER_SECOND);
}
/**
* Returns the number of audio samples in the given audio packet.
*

View File

@ -0,0 +1,11 @@
SinkDump (OggOpus):
buffers.length = 9
buffers[0] = length 4046, hash 68FA8318
buffers[1] = length 3848, hash B3105060
buffers[2] = length 3747, hash 63B6648B
buffers[3] = length 3752, hash B5C28B9D
buffers[4] = length 3776, hash AC7CEC0B
buffers[5] = length 3829, hash B64088F2
buffers[6] = length 3745, hash 1C46E49A
buffers[7] = length 3726, hash 2BC03F39
buffers[8] = length 2772, hash A6C7BB9

View File

@ -0,0 +1,266 @@
/*
* Copyright (C) 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.test.utils;
import static java.lang.Math.min;
import android.os.Environment;
import androidx.annotation.Nullable;
import androidx.media3.common.util.Assertions;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import androidx.media3.exoplayer.audio.AudioSink;
import androidx.media3.exoplayer.audio.ForwardingAudioSink;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
/**
* A sink for audio buffers that writes output audio as .ogg files with a given path prefix. When
* new audio data is handled after flushing the audio packetizer, a counter is incremented and its
* value is appended to the output file name.
*
* <p>Note: if writing to external storage it's necessary to grant the {@code
* WRITE_EXTERNAL_STORAGE} permission.
*/
@UnstableApi
public final class OggFileAudioBufferSink extends ForwardingAudioSink {
/** Opus streams are always 48000 Hz. */
public static final int SAMPLE_RATE = 48_000;
private static final String TAG = "OggFileAudioBufferSink";
private static final int OGG_ID_HEADER_LENGTH = 47;
private static final int OGG_COMMENT_HEADER_LENGTH = 52;
private final byte[] scratchBuffer;
private final ByteBuffer scratchByteBuffer;
private final String outputFileNamePrefix;
@Nullable private RandomAccessFile randomAccessFile;
private int counter;
/**
* Creates an instance.
*
* @param audioSink The base audioSink calls are forwarded to.
* @param outputFileNamePrefix The prefix for output files.
*/
public OggFileAudioBufferSink(AudioSink audioSink, String outputFileNamePrefix) {
super(audioSink);
this.outputFileNamePrefix = outputFileNamePrefix;
counter = 0;
scratchBuffer = new byte[1024];
scratchByteBuffer = ByteBuffer.wrap(scratchBuffer).order(ByteOrder.LITTLE_ENDIAN);
}
@Override
public void flush() {
super.flush();
try {
resetInternal();
} catch (IOException e) {
Log.e(TAG, "Error resetting", e);
}
}
@Override
public void reset() {
super.reset();
try {
resetInternal();
} catch (IOException e) {
Log.e(TAG, "Error resetting", e);
}
}
@Override
public boolean handleBuffer(
ByteBuffer buffer, long presentationTimeUs, int encodedAccessUnitCount)
throws InitializationException, WriteException {
handleBuffer(buffer);
return super.handleBuffer(buffer, presentationTimeUs, encodedAccessUnitCount);
}
private void handleBuffer(ByteBuffer buffer) {
try {
maybePrepareFile();
writeBuffer(buffer);
} catch (IOException e) {
Log.e(TAG, "Error writing data", e);
}
}
private void maybePrepareFile() throws IOException {
if (randomAccessFile != null) {
return;
}
RandomAccessFile randomAccessFile = new RandomAccessFile(getNextOutputFileName(), "rw");
scratchByteBuffer.clear();
writeIdHeaderPacket();
writeCommentHeaderPacket();
randomAccessFile.write(scratchBuffer, 0, scratchByteBuffer.position());
this.randomAccessFile = randomAccessFile;
}
private void writeOggPacketHeader(int pageSequenceNumber, boolean isIdHeaderPacket) {
// Capture Pattern for Page [OggS]
scratchByteBuffer.put((byte) 'O');
scratchByteBuffer.put((byte) 'g');
scratchByteBuffer.put((byte) 'g');
scratchByteBuffer.put((byte) 'S');
// StreamStructure Version
scratchByteBuffer.put((byte) 0);
// header-type
scratchByteBuffer.put(isIdHeaderPacket ? (byte) 0x02 : (byte) 0x00);
// granule_position
scratchByteBuffer.putLong((long) 0);
// bitstream_serial_number
scratchByteBuffer.putInt(0);
// page_sequence_number
scratchByteBuffer.putInt(pageSequenceNumber);
// CRC_checksum
scratchByteBuffer.putInt(0);
// number_page_segments
scratchByteBuffer.put((byte) 1);
}
private void writeIdHeaderPacket() {
// Id Header
writeOggPacketHeader(/* pageSequenceNumber= */ 0, /* isIdHeaderPacket= */ true);
// Payload Size = 19
scratchByteBuffer.put((byte) 19);
// OggOpus Id Header Capture Pattern 8
scratchByteBuffer.put((byte) 'O');
scratchByteBuffer.put((byte) 'p');
scratchByteBuffer.put((byte) 'u');
scratchByteBuffer.put((byte) 's');
scratchByteBuffer.put((byte) 'H');
scratchByteBuffer.put((byte) 'e');
scratchByteBuffer.put((byte) 'a');
scratchByteBuffer.put((byte) 'd');
// version
scratchByteBuffer.put((byte) 1);
// output channel count
scratchByteBuffer.put((byte) 2);
// pre-skip
scratchByteBuffer.putShort((short) 312);
// input sample rate
scratchByteBuffer.putInt(SAMPLE_RATE);
// Output Gain
scratchByteBuffer.putShort((short) 0);
// channel mapping family
scratchByteBuffer.put((byte) 0);
int checksum =
Util.crc32(scratchBuffer, /* start= */ 0, OGG_ID_HEADER_LENGTH, /* initialValue= */ 0);
scratchByteBuffer.putInt(/* index= */ 22, checksum);
scratchByteBuffer.position(OGG_ID_HEADER_LENGTH);
}
private void writeCommentHeaderPacket() {
// Id Header
writeOggPacketHeader(/* pageSequenceNumber= */ 1, /* isIdHeaderPacket= */ false);
// Payload Size = 24
scratchByteBuffer.put((byte) 24);
// Comment Header Opus Capture Pattern 8
scratchByteBuffer.put((byte) 'O');
scratchByteBuffer.put((byte) 'p');
scratchByteBuffer.put((byte) 'u');
scratchByteBuffer.put((byte) 's');
scratchByteBuffer.put((byte) 'T');
scratchByteBuffer.put((byte) 'a');
scratchByteBuffer.put((byte) 'g');
scratchByteBuffer.put((byte) 's');
// Vendor Comment String Length
scratchByteBuffer.putInt(8);
// Vendor Comment String
scratchByteBuffer.put((byte) 'G');
scratchByteBuffer.put((byte) 'o');
scratchByteBuffer.put((byte) 'o');
scratchByteBuffer.put((byte) 'g');
scratchByteBuffer.put((byte) 'l');
scratchByteBuffer.put((byte) 'e');
scratchByteBuffer.put((byte) 'r');
scratchByteBuffer.put((byte) 's');
// UserCommentList Length
scratchByteBuffer.putInt(0);
int checksum =
Util.crc32(
scratchBuffer,
OGG_ID_HEADER_LENGTH,
OGG_ID_HEADER_LENGTH + OGG_COMMENT_HEADER_LENGTH,
/* initialValue= */ 0);
scratchByteBuffer.putInt(/* index= */ 69, checksum);
scratchByteBuffer.position(OGG_ID_HEADER_LENGTH + OGG_COMMENT_HEADER_LENGTH);
}
private void writeBuffer(ByteBuffer buffer) throws IOException {
RandomAccessFile randomAccessFile = Assertions.checkNotNull(this.randomAccessFile);
while (buffer.hasRemaining()) {
int bytesToWrite = min(buffer.remaining(), scratchBuffer.length);
buffer.get(scratchBuffer, /* offset= */ 0, bytesToWrite);
randomAccessFile.write(scratchBuffer, /* off= */ 0, bytesToWrite);
}
}
private void resetInternal() throws IOException {
@Nullable RandomAccessFile randomAccessFile = this.randomAccessFile;
if (randomAccessFile == null) {
return;
}
try {
randomAccessFile.close();
} finally {
this.randomAccessFile = null;
}
}
private String getNextOutputFileName() {
return Util.formatInvariant(
"%s/%s-%04d.ogg",
Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_MUSIC)
.getAbsolutePath(),
outputFileNamePrefix,
counter++);
}
}