loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Create InAppMuxer in transformer

Browse Source 
To use the InAppMuxer, the client needs to pass InAppMuxer Factory.

PiperOrigin-RevId: 530684007
This commit is contained in:
sheenachhabra 2023-05-09 19:36:07 +00:00 committed by Tofunmi Adigun-Hameed
parent 00d5031dcf
commit 65d5132f76
11 changed files with 374 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
libraries/muxer/src/main/java/androidx/media3/muxer/Boxes.java
View File

@ -562,7 +562,7 @@ import java.util.Locale;
contents.put(paspBox());
// Put in a "colr" box if any of the three color format parameters has a non-default (0) value.
// TODO(b/278101856): Only null check should be enough once we disallow invalid values.
// TODO: b/278101856 - Only null check should be enough once we disallow invalid values.
if (format.colorInfo != null
&& (format.colorInfo.colorSpace != 0
|| format.colorInfo.colorTransfer != 0
@ -588,6 +588,7 @@ import java.util.Locale;
* @param lastDurationBehavior The behaviour for the last sample duration.
* @return A list of all the sample durations.
*/
// TODO: b/280084657 - Add support for setting last sample duration.
public static List<Long> durationsVuForStts(
List<MediaCodec.BufferInfo> writtenSamples,
long minInputPresentationTimestampUs,
@ -671,7 +672,7 @@ import java.util.Locale;
contents.putInt(0x0); // version and flags.
// TODO(b/270583563): Consider optimizing for identically-sized samples.
// TODO: b/270583563 - Consider optimizing for identically-sized samples.
// sample_size; specifying the default sample size. Set to zero to indicate that the samples
// have different sizes and they are stored in the sample size table.
contents.putInt(0);
@ -697,7 +698,7 @@ import java.util.Locale;
int currentChunk = 1;
// TODO(b/270583563): Consider optimizing for consecutive chunks having same number of samples.
// TODO: b/270583563 - Consider optimizing for consecutive chunks having same number of samples.
for (int i = 0; i < writtenChunkSampleCounts.size(); i++) {
int samplesInChunk = writtenChunkSampleCounts.get(i);
contents.putInt(currentChunk); // first_chunk.

10
libraries/muxer/src/main/java/androidx/media3/muxer/Mp4Muxer.java
View File

@ -23,7 +23,9 @@ import androidx.annotation.FloatRange;
import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.UnstableApi;
import com.google.common.collect.ImmutableList;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.io.FileOutputStream;
import java.io.IOException;
@ -140,6 +142,14 @@ public final class Mp4Muxer {
}
}
/** A list of supported video sample mime types. */
public static final ImmutableList<String> SUPPORTED_VIDEO_SAMPLE_MIME_TYPES =
ImmutableList.of(MimeTypes.VIDEO_H264, MimeTypes.VIDEO_H265, MimeTypes.VIDEO_AV1);
/** A list of supported audio sample mime types. */
public static final ImmutableList<String> SUPPORTED_AUDIO_SAMPLE_MIME_TYPES =
ImmutableList.of(MimeTypes.AUDIO_AAC);
private final Mp4Writer mp4Writer;
private final MetadataCollector metadataCollector;

20
libraries/muxer/src/main/java/androidx/media3/muxer/Mp4Writer.java
View File

@ -319,7 +319,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
private void doInterleave() throws IOException {
for (int i = 0; i < tracks.size(); i++) {
Track track = tracks.get(i);
// TODO(b/270583563): check if we need to consider the global timestamp instead.
// TODO: b/270583563 - check if we need to consider the global timestamp instead.
if (track.pendingSamples.size() > 2) {
BufferInfo firstSampleInfo = checkNotNull(track.pendingSamples.peekFirst()).first;
BufferInfo lastSampleInfo = checkNotNull(track.pendingSamples.peekLast()).first;
@ -377,9 +377,21 @@ import java.util.concurrent.atomic.AtomicBoolean;
}
// Skip empty samples.
// TODO(b/279931840): Confirm whether muxer should throw when writing empty samples.
// TODO: b/279931840 - Confirm whether muxer should throw when writing empty samples.
if (byteBuffer.remaining() > 0) {
pendingSamples.addLast(Pair.create(bufferInfo, byteBuffer));
// Copy sample data and release the original buffer.
ByteBuffer byteBufferCopy = ByteBuffer.allocateDirect(byteBuffer.remaining());
byteBufferCopy.put(byteBuffer);
byteBufferCopy.rewind();
BufferInfo bufferInfoCopy = new BufferInfo();
bufferInfoCopy.set(
/* newOffset= */ byteBufferCopy.position(),
/* newSize= */ byteBufferCopy.remaining(),
bufferInfo.presentationTimeUs,
bufferInfo.flags);
pendingSamples.addLast(Pair.create(bufferInfoCopy, byteBufferCopy));
doInterleave();
}
}
@ -387,7 +399,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
@Override
public int videoUnitTimebase() {
return MimeTypes.isAudio(format.sampleMimeType)
? 48_000 // TODO(b/270583563): Update these with actual values from mediaFormat.
? 48_000 // TODO: b/270583563 - Update these with actual values from mediaFormat.
: 90_000;
}

BIN
libraries/test_data/src/test/assets/media/mp4/h265_with_metadata_track.mp4 Normal file
View File

Binary file not shown.

1
libraries/transformer/build.gradle
View File

@ -39,6 +39,7 @@ dependencies {
implementation project(modulePrefix + 'lib-datasource')
implementation project(modulePrefix + 'lib-exoplayer')
implementation project(modulePrefix + 'lib-effect')
implementation project(modulePrefix + 'lib-muxer')
compileOnly 'com.google.errorprone:error_prone_annotations:' + errorProneVersion
compileOnly 'org.checkerframework:checker-qual:' + checkerframeworkVersion
compileOnly 'org.checkerframework:checker-compat-qual:' + checkerframeworkCompatVersion

10
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java
View File

@ -15,6 +15,7 @@
*/
package androidx.media3.transformer;
import static androidx.media3.common.MimeTypes.VIDEO_AV1;
import static androidx.media3.common.MimeTypes.VIDEO_H264;
import static androidx.media3.common.MimeTypes.VIDEO_H265;
import static androidx.media3.common.util.Assertions.checkNotNull;
@ -66,6 +67,15 @@ public final class AndroidTestUtil {
.setCodecs("avc1.64001F")
.build();
public static final String MP4_ASSET_AV1_VIDEO_URI_STRING = "asset:///media/mp4/sample_av1.mp4";
public static final Format MP4_ASSET_AV1_VIDEO_FORMAT =
new Format.Builder()
.setSampleMimeType(VIDEO_AV1)
.setWidth(1080)
.setHeight(720)
.setFrameRate(30.0f)
.build();
public static final String MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING =
"asset:///media/mp4/sample_with_increasing_timestamps.mp4";
public static final Format MP4_ASSET_WITH_INCREASING_TIMESTAMPS_FORMAT =

102
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerWithInAppMuxerEndToEndTest.java Normal file
View File

@ -0,0 +1,102 @@
/*
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import static com.google.common.truth.Truth.assertThat;
import android.content.Context;
import android.net.Uri;
import androidx.media3.common.Effect;
import androidx.media3.common.MediaItem;
import androidx.media3.common.audio.ChannelMixingAudioProcessor;
import androidx.media3.common.audio.ChannelMixingMatrix;
import androidx.media3.effect.RgbFilter;
import androidx.test.core.app.ApplicationProvider;
import com.google.common.collect.ImmutableList;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameter;
import org.junit.runners.Parameterized.Parameters;
/** End-to-end instrumentation test for {@link Transformer} with {@link InAppMuxer}. */
@RunWith(Parameterized.class)
public class TransformerWithInAppMuxerEndToEndTest {
private static final String MP4_FILE_ASSET_DIRECTORY = "asset:///media/mp4/";
private static final String H264_MP4 = "sample.mp4";
private static final String H265_MP4 = "h265_with_metadata_track.mp4";
@Parameters(name = "{0}")
public static ImmutableList<String> mediaFiles() {
return ImmutableList.of(H264_MP4, H265_MP4);
}
@Parameter public @MonotonicNonNull String inputFile;
private final Context context = ApplicationProvider.getApplicationContext();
@Test
public void videoEditing_completesSuccessfully() throws Exception {
String testId = "videoEditing_completesSuccessfully";
Transformer transformer =
new Transformer.Builder(context)
.setMuxerFactory(
new InAppMuxer.Factory(DefaultMuxer.Factory.DEFAULT_MAX_DELAY_BETWEEN_SAMPLES_MS))
.build();
ImmutableList<Effect> videoEffects = ImmutableList.of(RgbFilter.createGrayscaleFilter());
MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_FILE_ASSET_DIRECTORY + inputFile));
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem)
.setEffects(new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects))
.build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
assertThat(result.exportResult.exportException).isNull();
}
@Test
public void audioEditing_completesSuccessfully() throws Exception {
String testId = "audioEditing_completesSuccessfully";
Transformer transformer =
new Transformer.Builder(context)
.setMuxerFactory(
new InAppMuxer.Factory(DefaultMuxer.Factory.DEFAULT_MAX_DELAY_BETWEEN_SAMPLES_MS))
.build();
ChannelMixingAudioProcessor channelMixingAudioProcessor = new ChannelMixingAudioProcessor();
channelMixingAudioProcessor.putChannelMixingMatrix(
ChannelMixingMatrix.create(/* inputChannelCount= */ 1, /* outputChannelCount= */ 2));
MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_FILE_ASSET_DIRECTORY + H264_MP4));
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem)
.setEffects(
new Effects(
ImmutableList.of(channelMixingAudioProcessor),
/* videoEffects= */ ImmutableList.of()))
.build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
assertThat(result.exportResult.exportException).isNull();
}
}

71
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformerWithInAppMuxerEndToEndTest.java Normal file
View File

@ -0,0 +1,71 @@
/*
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer.mh;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_AV1_VIDEO_FORMAT;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_AV1_VIDEO_URI_STRING;
import static com.google.common.truth.Truth.assertThat;
import android.content.Context;
import android.net.Uri;
import androidx.media3.common.Effect;
import androidx.media3.common.MediaItem;
import androidx.media3.effect.RgbFilter;
import androidx.media3.transformer.AndroidTestUtil;
import androidx.media3.transformer.DefaultMuxer;
import androidx.media3.transformer.EditedMediaItem;
import androidx.media3.transformer.Effects;
import androidx.media3.transformer.ExportTestResult;
import androidx.media3.transformer.InAppMuxer;
import androidx.media3.transformer.Transformer;
import androidx.media3.transformer.TransformerAndroidTestRunner;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import org.junit.Test;
import org.junit.runner.RunWith;
/** End-to-end instrumentation test for {@link Transformer} with {@link InAppMuxer}. */
@RunWith(AndroidJUnit4.class)
public class TransformerWithInAppMuxerEndToEndTest {
@Test
public void videoEditing_forAv1Video_completesSuccessfully() throws Exception {
String testId = "videoEditing_forAv1Video_completesSuccessfully";
Context context = ApplicationProvider.getApplicationContext();
if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
context, testId, /* inputFormat= */ MP4_ASSET_AV1_VIDEO_FORMAT, /* outputFormat= */ null)) {
return;
}
Transformer transformer =
new Transformer.Builder(context)
.setMuxerFactory(
new InAppMuxer.Factory(DefaultMuxer.Factory.DEFAULT_MAX_DELAY_BETWEEN_SAMPLES_MS))
.build();
ImmutableList<Effect> videoEffects = ImmutableList.of(RgbFilter.createGrayscaleFilter());
MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_ASSET_AV1_VIDEO_URI_STRING));
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem)
.setEffects(new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects))
.build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
assertThat(result.exportResult.exportException).isNull();
}
}

13
libraries/transformer/src/main/java/androidx/media3/transformer/FrameworkMuxer.java
View File

@ -162,7 +162,7 @@ import java.nio.ByteBuffer;
int offset = data.position();
int size = data.limit() - offset;
bufferInfo.set(offset, size, presentationTimeUs, getMediaMuxerFlags(flags));
bufferInfo.set(offset, size, presentationTimeUs, TransformerUtil.getMediaCodecFlags(flags));
long lastSamplePresentationTimeUs = trackIndexToLastPresentationTimeUs.get(trackIndex);
// writeSampleData blocks on old API versions, so check here to avoid calling the method.
checkState(
@ -231,17 +231,6 @@ import java.nio.ByteBuffer;
return maxDelayBetweenSamplesMs;
}
private static int getMediaMuxerFlags(@C.BufferFlags int flags) {
int mediaMuxerFlags = 0;
if ((flags & C.BUFFER_FLAG_KEY_FRAME) == C.BUFFER_FLAG_KEY_FRAME) {
mediaMuxerFlags |= MediaCodec.BUFFER_FLAG_KEY_FRAME;
}
if ((flags & C.BUFFER_FLAG_END_OF_STREAM) == C.BUFFER_FLAG_END_OF_STREAM) {
mediaMuxerFlags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM;
}
return mediaMuxerFlags;
}
// Accesses MediaMuxer state via reflection to ensure that muxer resources can be released even
// if stopping fails.
@SuppressLint("PrivateApi")

146
libraries/transformer/src/main/java/androidx/media3/transformer/InAppMuxer.java Normal file
View File

@ -0,0 +1,146 @@
/*
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import static androidx.media3.muxer.Mp4Muxer.SUPPORTED_AUDIO_SAMPLE_MIME_TYPES;
import static androidx.media3.muxer.Mp4Muxer.SUPPORTED_VIDEO_SAMPLE_MIME_TYPES;
import android.media.MediaCodec.BufferInfo;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.Metadata;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.extractor.metadata.mp4.Mp4LocationData;
import androidx.media3.muxer.Mp4Muxer;
import androidx.media3.muxer.Mp4Muxer.TrackToken;
import com.google.common.collect.ImmutableList;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
/** {@link Muxer} implementation that uses a {@link Mp4Muxer}. */
@UnstableApi
public final class InAppMuxer implements Muxer {
/** {@link Muxer.Factory} for {@link InAppMuxer}. */
public static final class Factory implements Muxer.Factory {
private final long maxDelayBetweenSamplesMs;
/** {@link Muxer.Factory} for {@link InAppMuxer}. */
public Factory(long maxDelayBetweenSamplesMs) {
this.maxDelayBetweenSamplesMs = maxDelayBetweenSamplesMs;
}
@Override
public InAppMuxer create(String path) throws MuxerException {
FileOutputStream outputStream;
try {
outputStream = new FileOutputStream(path);
} catch (FileNotFoundException e) {
throw new MuxerException("Error creating file output stream", e);
}
Mp4Muxer mp4Muxer = new Mp4Muxer.Builder(outputStream).build();
return new InAppMuxer(mp4Muxer, maxDelayBetweenSamplesMs);
}
@Override
public ImmutableList<String> getSupportedSampleMimeTypes(@C.TrackType int trackType) {
if (trackType == C.TRACK_TYPE_VIDEO) {
return SUPPORTED_VIDEO_SAMPLE_MIME_TYPES;
} else if (trackType == C.TRACK_TYPE_AUDIO) {
return SUPPORTED_AUDIO_SAMPLE_MIME_TYPES;
}
return ImmutableList.of();
}
}
private final Mp4Muxer mp4Muxer;
private final long maxDelayBetweenSamplesMs;
private final List<TrackToken> trackTokenList;
private final BufferInfo bufferInfo;
private InAppMuxer(Mp4Muxer mp4Muxer, long maxDelayBetweenSamplesMs) {
this.mp4Muxer = mp4Muxer;
this.maxDelayBetweenSamplesMs = maxDelayBetweenSamplesMs;
trackTokenList = new ArrayList<>();
bufferInfo = new BufferInfo();
}
@Override
public int addTrack(Format format) {
// Keep same sort key as no specific sort order is required.
TrackToken trackToken = mp4Muxer.addTrack(/* sortKey= */ 0, format);
trackTokenList.add(trackToken);
if (MimeTypes.isVideo(format.sampleMimeType)) {
mp4Muxer.setOrientation(format.rotationDegrees);
}
return trackTokenList.size() - 1;
}
@Override
public void writeSampleData(
int trackIndex, ByteBuffer data, long presentationTimeUs, @C.BufferFlags int flags)
throws MuxerException {
int size = data.remaining();
bufferInfo.set(
data.position(), size, presentationTimeUs, TransformerUtil.getMediaCodecFlags(flags));
try {
mp4Muxer.writeSampleData(trackTokenList.get(trackIndex), data, bufferInfo);
} catch (IOException e) {
throw new MuxerException(
"Failed to write sample for trackIndex="
+ trackIndex
+ ", presentationTimeUs="
+ presentationTimeUs
+ ", size="
+ size,
e);
}
}
@Override
public void addMetadata(Metadata metadata) {
for (int i = 0; i < metadata.length(); i++) {
Metadata.Entry entry = metadata.get(i);
if (entry instanceof Mp4LocationData) {
mp4Muxer.setLocation(
((Mp4LocationData) entry).latitude, ((Mp4LocationData) entry).longitude);
}
}
}
@Override
public void release(boolean forCancellation) throws MuxerException {
try {
mp4Muxer.close();
} catch (IOException e) {
throw new MuxerException("Error closing muxer", e);
}
}
@Override
public long getMaxDelayBetweenSamplesMs() {
return maxDelayBetweenSamplesMs;
}
}

13
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java
View File

@ -16,6 +16,7 @@
package androidx.media3.transformer;
import android.media.MediaCodec;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Effect;
@ -84,4 +85,16 @@ import com.google.common.collect.ImmutableList;
}
return false;
}
/** Returns {@link MediaCodec} flags corresponding to {@link C.BufferFlags}. */
public static int getMediaCodecFlags(@C.BufferFlags int flags) {
int mediaCodecFlags = 0;
if ((flags & C.BUFFER_FLAG_KEY_FRAME) == C.BUFFER_FLAG_KEY_FRAME) {
mediaCodecFlags |= MediaCodec.BUFFER_FLAG_KEY_FRAME;
}
if ((flags & C.BUFFER_FLAG_END_OF_STREAM) == C.BUFFER_FLAG_END_OF_STREAM) {
mediaCodecFlags |= MediaCodec.BUFFER_FLAG_END_OF_STREAM;
}
return mediaCodecFlags;
}
}