loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add the frame count to TransformationResult.

Browse Source 
Calculate throughputFps for TransformationTestResult.

PiperOrigin-RevId: 438817440
This commit is contained in:
samrobinson 2022-04-01 15:15:24 +01:00 committed by Ian Baker
parent 0c6882867b
commit af5386cbc1
6 changed files with 68 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameCountingMuxer.java
View File

@ -1,114 +0,0 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import android.os.ParcelFileDescriptor;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* An implementation of {@link Muxer} that forwards operations to another {@link Muxer}, counting
* the number of frames as they go past.
*/
/* package */ final class FrameCountingMuxer implements Muxer {
public static final class Factory implements Muxer.Factory {
private final Muxer.Factory muxerFactory;
private @MonotonicNonNull FrameCountingMuxer frameCountingMuxer;
public Factory(Muxer.Factory muxerFactory) {
this.muxerFactory = muxerFactory;
}
@Override
public Muxer create(String path, String outputMimeType) throws IOException {
frameCountingMuxer = new FrameCountingMuxer(muxerFactory.create(path, outputMimeType));
return frameCountingMuxer;
}
@Override
public Muxer create(ParcelFileDescriptor parcelFileDescriptor, String outputMimeType)
throws IOException {
frameCountingMuxer =
new FrameCountingMuxer(muxerFactory.create(parcelFileDescriptor, outputMimeType));
return frameCountingMuxer;
}
@Override
public boolean supportsOutputMimeType(String mimeType) {
return muxerFactory.supportsOutputMimeType(mimeType);
}
@Override
public boolean supportsSampleMimeType(@Nullable String sampleMimeType, String outputMimeType) {
return muxerFactory.supportsSampleMimeType(sampleMimeType, outputMimeType);
}
@Override
public ImmutableList<String> getSupportedSampleMimeTypes(
@C.TrackType int trackType, String containerMimeType) {
return muxerFactory.getSupportedSampleMimeTypes(trackType, containerMimeType);
}
@Nullable
public FrameCountingMuxer getLastFrameCountingMuxerCreated() {
return frameCountingMuxer;
}
}
private final Muxer muxer;
private int videoTrackIndex;
private int frameCount;
private FrameCountingMuxer(Muxer muxer) throws IOException {
this.muxer = muxer;
}
@Override
public int addTrack(Format format) throws MuxerException {
int trackIndex = muxer.addTrack(format);
if (MimeTypes.isVideo(format.sampleMimeType)) {
videoTrackIndex = trackIndex;
}
return trackIndex;
}
@Override
public void writeSampleData(
int trackIndex, ByteBuffer data, boolean isKeyFrame, long presentationTimeUs)
throws MuxerException {
muxer.writeSampleData(trackIndex, data, isKeyFrame, presentationTimeUs);
if (trackIndex == videoTrackIndex) {
frameCount++;
}
}
@Override
public void release(boolean forCancellation) throws MuxerException {
muxer.release(forCancellation);
}
/* Returns the number of frames written for the video track. */
public int getFrameCount() {
return frameCount;
}
}

17
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformationTestResult.java
View File

@ -31,7 +31,6 @@ public class TransformationTestResult {
@Nullable private String filePath;
@Nullable private Exception analysisException;
private long elapsedTimeMs;
private double ssim;
@ -103,8 +102,12 @@ public class TransformationTestResult {
}
public final TransformationResult transformationResult;
@Nullable public final String filePath;
/**
* The average rate (per second) at which frames are processed by the transformer, or {@link
* C#RATE_UNSET} if unset or unknown.
*/
public final float throughputFps;
/**
* The amount of time taken to perform the transformation in milliseconds. {@link C#TIME_UNSET} if
* unset.
@ -133,6 +136,12 @@ public class TransformationTestResult {
if (transformationResult.averageVideoBitrate != C.RATE_UNSET_INT) {
jsonObject.put("averageVideoBitrate", transformationResult.averageVideoBitrate);
}
if (transformationResult.videoFrameCount > 0) {
jsonObject.put("videoFrameCount", transformationResult.videoFrameCount);
}
if (throughputFps != C.RATE_UNSET) {
jsonObject.put("throughputFps", throughputFps);
}
if (elapsedTimeMs != C.TIME_UNSET) {
jsonObject.put("elapsedTimeMs", elapsedTimeMs);
}
@ -156,5 +165,9 @@ public class TransformationTestResult {
this.elapsedTimeMs = elapsedTimeMs;
this.ssim = ssim;
this.analysisException = analysisException;
this.throughputFps =
elapsedTimeMs != C.TIME_UNSET && transformationResult.videoFrameCount > 0
? 1000f * transformationResult.videoFrameCount / elapsedTimeMs
: C.RATE_UNSET;
}
}

18
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
View File

@ -15,7 +15,7 @@
*/
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING;
import static com.google.common.truth.Truth.assertThat;
import android.content.Context;
@ -31,18 +31,13 @@ import org.junit.runner.RunWith;
@RunWith(AndroidJUnit4.class)
public class TransformerEndToEndTest {
private static final String AVC_VIDEO_URI_STRING = "asset:///media/mp4/sample.mp4";
@Test
public void videoEditing_completesWithConsistentFrameCount() throws Exception {
Context context = ApplicationProvider.getApplicationContext();
FrameCountingMuxer.Factory muxerFactory =
new FrameCountingMuxer.Factory(new FrameworkMuxer.Factory());
Transformer transformer =
new Transformer.Builder(context)
.setTransformationRequest(
new TransformationRequest.Builder().setResolution(480).build())
.setMuxerFactory(muxerFactory)
.setEncoderFactory(
new DefaultEncoderFactory(EncoderSelector.DEFAULT, /* enableFallback= */ false))
.build();
@ -50,13 +45,14 @@ public class TransformerEndToEndTest {
// ffprobe -count_frames -select_streams v:0 -show_entries stream=nb_read_frames sample.mp4
int expectedFrameCount = 30;
TransformationTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(/* testId= */ "videoEditing_completesWithConsistentFrameCount", AVC_VIDEO_URI_STRING);
.run(
/* testId= */ "videoEditing_completesWithConsistentFrameCount",
MP4_ASSET_URI_STRING);
FrameCountingMuxer frameCountingMuxer =
checkNotNull(muxerFactory.getLastFrameCountingMuxerCreated());
assertThat(frameCountingMuxer.getFrameCount()).isEqualTo(expectedFrameCount);
assertThat(result.transformationResult.videoFrameCount).isEqualTo(expectedFrameCount);
}
@Test
@ -75,7 +71,7 @@ public class TransformerEndToEndTest {
TransformationTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(/* testId= */ "videoOnly_completesWithConsistentDuration", AVC_VIDEO_URI_STRING);
.run(/* testId= */ "videoOnly_completesWithConsistentDuration", MP4_ASSET_URI_STRING);
assertThat(result.transformationResult.durationMs).isEqualTo(expectedDurationMs);
}

20
libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java
View File

@ -48,6 +48,7 @@ import java.nio.ByteBuffer;
private final Muxer muxer;
private final Muxer.Factory muxerFactory;
private final SparseIntArray trackTypeToIndex;
private final SparseIntArray trackTypeToSampleCount;
private final SparseLongArray trackTypeToTimeUs;
private final SparseLongArray trackTypeToBytesWritten;
private final String containerMimeType;
@ -62,7 +63,9 @@ import java.nio.ByteBuffer;
this.muxer = muxer;
this.muxerFactory = muxerFactory;
this.containerMimeType = containerMimeType;
trackTypeToIndex = new SparseIntArray();
trackTypeToSampleCount = new SparseIntArray();
trackTypeToTimeUs = new SparseLongArray();
trackTypeToBytesWritten = new SparseLongArray();
previousTrackType = C.TRACK_TYPE_NONE;
@ -123,6 +126,7 @@ import java.nio.ByteBuffer;
int trackIndex = muxer.addTrack(format);
trackTypeToIndex.put(trackType, trackIndex);
trackTypeToSampleCount.put(trackType, 0);
trackTypeToTimeUs.put(trackType, 0L);
trackTypeToBytesWritten.put(trackType, 0L);
trackFormatCount++;
@ -158,6 +162,7 @@ import java.nio.ByteBuffer;
return false;
}
trackTypeToSampleCount.put(trackType, trackTypeToSampleCount.get(trackType) + 1);
trackTypeToBytesWritten.put(
trackType, trackTypeToBytesWritten.get(trackType) + data.remaining());
if (trackTypeToTimeUs.get(trackType) < presentationTimeUs) {
@ -218,6 +223,16 @@ import java.nio.ByteBuffer;
/* divisor= */ trackDurationUs);
}
/** Returns the number of samples written to the track of the provided {@code trackType}. */
public int getTrackSampleCount(@C.TrackType int trackType) {
return trackTypeToSampleCount.get(trackType, /* valueIfKeyNotFound= */ 0);
}
/** Returns the duration of the longest track in milliseconds. */
public long getDurationMs() {
return Util.usToMs(maxValue(trackTypeToTimeUs));
}
/**
* Returns whether the muxer can write a sample of the given track type.
*
@ -243,9 +258,4 @@ import java.nio.ByteBuffer;
}
return trackTimeUs - minTrackTimeUs <= MAX_TRACK_WRITE_AHEAD_US;
}
/** Returns the duration of the longest track in milliseconds. */
public long getDurationMs() {
return Util.usToMs(maxValue(trackTypeToTimeUs));
}
}

32
libraries/transformer/src/main/java/androidx/media3/transformer/TransformationResult.java
View File

@ -31,6 +31,7 @@ public final class TransformationResult {
private long fileSizeBytes;
private int averageAudioBitrate;
private int averageVideoBitrate;
private int videoFrameCount;
public Builder() {
durationMs = C.TIME_UNSET;
@ -83,13 +84,24 @@ public final class TransformationResult {
return this;
}
/**
* Sets the number of video frames.
*
* <p>Input must be positive or {@code 0}.
*/
public Builder setVideoFrameCount(int videoFrameCount) {
checkArgument(videoFrameCount >= 0);
this.videoFrameCount = videoFrameCount;
return this;
}
public TransformationResult build() {
return new TransformationResult(
durationMs, fileSizeBytes, averageAudioBitrate, averageVideoBitrate);
durationMs, fileSizeBytes, averageAudioBitrate, averageVideoBitrate, videoFrameCount);
}
}
/** The duration of the video in milliseconds, or {@link C#TIME_UNSET} if unset or unknown. */
/** The duration of the file in milliseconds, or {@link C#TIME_UNSET} if unset or unknown. */
public final long durationMs;
/** The size of the file in bytes, or {@link C#LENGTH_UNSET} if unset or unknown. */
public final long fileSizeBytes;
@ -101,13 +113,20 @@ public final class TransformationResult {
* The average bitrate of the video track data, or {@link C#RATE_UNSET_INT} if unset or unknown.
*/
public final int averageVideoBitrate;
/** The number of video frames. */
public final int videoFrameCount;
private TransformationResult(
long durationMs, long fileSizeBytes, int averageAudioBitrate, int averageVideoBitrate) {
long durationMs,
long fileSizeBytes,
int averageAudioBitrate,
int averageVideoBitrate,
int videoFrameCount) {
this.durationMs = durationMs;
this.fileSizeBytes = fileSizeBytes;
this.averageAudioBitrate = averageAudioBitrate;
this.averageVideoBitrate = averageVideoBitrate;
this.videoFrameCount = videoFrameCount;
}
public Builder buildUpon() {
@ -115,7 +134,8 @@ public final class TransformationResult {
.setDurationMs(durationMs)
.setFileSizeBytes(fileSizeBytes)
.setAverageAudioBitrate(averageAudioBitrate)
.setAverageVideoBitrate(averageVideoBitrate);
.setAverageVideoBitrate(averageVideoBitrate)
.setVideoFrameCount(videoFrameCount);
}
@Override
@ -130,7 +150,8 @@ public final class TransformationResult {
return durationMs == result.durationMs
&& fileSizeBytes == result.fileSizeBytes
&& averageAudioBitrate == result.averageAudioBitrate
&& averageVideoBitrate == result.averageVideoBitrate;
&& averageVideoBitrate == result.averageVideoBitrate
&& videoFrameCount == result.videoFrameCount;
}
@Override
@ -139,6 +160,7 @@ public final class TransformationResult {
result = 31 * result + (int) fileSizeBytes;
result = 31 * result + averageAudioBitrate;
result = 31 * result + averageVideoBitrate;
result = 31 * result + videoFrameCount;
return result;
}
}

3
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View File

@ -704,7 +704,6 @@ public final class Transformer {
if (player != null) {
throw new IllegalStateException("There is already a transformation in progress.");
}
MuxerWrapper muxerWrapper = new MuxerWrapper(muxer, muxerFactory, containerMimeType);
this.muxerWrapper = muxerWrapper;
DefaultTrackSelector trackSelector = new DefaultTrackSelector(context);
@ -1005,7 +1004,9 @@ public final class Transformer {
.setDurationMs(muxerWrapper.getDurationMs())
.setAverageAudioBitrate(muxerWrapper.getTrackAverageBitrate(C.TRACK_TYPE_AUDIO))
.setAverageVideoBitrate(muxerWrapper.getTrackAverageBitrate(C.TRACK_TYPE_VIDEO))
.setVideoFrameCount(muxerWrapper.getTrackSampleCount(C.TRACK_TYPE_VIDEO))
.build();
listeners.queueEvent(
/* eventFlag= */ C.INDEX_UNSET,
listener -> listener.onTransformationCompleted(mediaItem, result));