loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use microseconds not nanoseconds for GlFrameProcessor.

Browse Source 
This requires an additional nanos to micros conversion because
the SurfaceTexture uses nanos. But as the timestamps from the
MediaCodec decoder (propagated in DefaultCodec#releaseOutputBuffer) are
in microseconds no precision is lost here.

Also add test that checks output video duration.

PiperOrigin-RevId: 438010490
This commit is contained in:
hschlueter 2022-03-29 14:36:10 +01:00 committed by Ian Baker
parent 2adf0f67d8
commit d97de5b9f0
14 changed files with 106 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
libraries/common/src/main/java/androidx/media3/common/util/Util.java
View File

@ -1121,6 +1121,25 @@ public final class Util {
return min;
}
/**
* Returns the maximum value in the given {@link SparseLongArray}.
*
* @param sparseLongArray The {@link SparseLongArray}.
* @return The maximum value.
* @throws NoSuchElementException If the array is empty.
*/
@RequiresApi(18)
public static long maxValue(SparseLongArray sparseLongArray) {
if (sparseLongArray.size() == 0) {
throw new NoSuchElementException();
}
long max = Long.MIN_VALUE;
for (int i = 0; i < sparseLongArray.size(); i++) {
max = max(max, sparseLongArray.valueAt(i));
}
return max;
}
/**
* Converts a time in microseconds to the corresponding time in milliseconds, preserving {@link
* C#TIME_UNSET} and {@link C#TIME_END_OF_SOURCE} values.

16
libraries/common/src/test/java/androidx/media3/common/util/UtilTest.java
View File

@ -21,6 +21,7 @@ import static androidx.media3.common.util.Util.escapeFileName;
import static androidx.media3.common.util.Util.getCodecsOfType;
import static androidx.media3.common.util.Util.getStringForTime;
import static androidx.media3.common.util.Util.gzip;
import static androidx.media3.common.util.Util.maxValue;
import static androidx.media3.common.util.Util.minValue;
import static androidx.media3.common.util.Util.parseXsDateTime;
import static androidx.media3.common.util.Util.parseXsDuration;
@ -747,6 +748,21 @@ public class UtilTest {
assertThrows(NoSuchElementException.class, () -> minValue(new SparseLongArray()));
}
@Test
public void sparseLongArrayMaxValue_returnsMaxValue() {
SparseLongArray sparseLongArray = new SparseLongArray();
sparseLongArray.put(0, 2);
sparseLongArray.put(25, 10);
sparseLongArray.put(42, 1);
assertThat(maxValue(sparseLongArray)).isEqualTo(10);
}
@Test
public void sparseLongArrayMaxValue_emptyArray_throws() {
assertThrows(NoSuchElementException.class, () -> maxValue(new SparseLongArray()));
}
@Test
public void parseXsDuration_returnsParsedDurationInMillis() {
assertThat(parseXsDuration("PT150.279S")).isEqualTo(150279L);

8
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AdvancedFrameProcessorPixelTest.java
View File

@ -94,7 +94,7 @@ public final class AdvancedFrameProcessorPixelTest {
advancedFrameProcessor.initialize(inputTexId);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(FIRST_FRAME_PNG_ASSET_STRING);
advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0);
advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);
@ -118,7 +118,7 @@ public final class AdvancedFrameProcessorPixelTest {
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING);
advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0);
advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);
@ -141,7 +141,7 @@ public final class AdvancedFrameProcessorPixelTest {
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING);
advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0);
advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);
@ -163,7 +163,7 @@ public final class AdvancedFrameProcessorPixelTest {
advancedFrameProcessor.initialize(inputTexId);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING);
advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0);
advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeUs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);

3
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java
View File

@ -309,6 +309,9 @@ public class TransformerAndroidTestRunner {
TransformationResult transformationResult = testResult.transformationResult;
JSONObject transformationResultJson = new JSONObject();
if (transformationResult.durationMs != C.LENGTH_UNSET) {
transformationResultJson.put("durationMs", transformationResult.durationMs);
}
if (transformationResult.fileSizeBytes != C.LENGTH_UNSET) {
transformationResultJson.put("fileSizeBytes", transformationResult.fileSizeBytes);
}

21
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
View File

@ -58,4 +58,25 @@ public class TransformerEndToEndTest {
checkNotNull(muxerFactory.getLastFrameCountingMuxerCreated());
assertThat(frameCountingMuxer.getFrameCount()).isEqualTo(expectedFrameCount);
}
@Test
public void videoOnly_completesWithConsistentDuration() throws Exception {
Context context = ApplicationProvider.getApplicationContext();
Transformer transformer =
new Transformer.Builder(context)
.setRemoveAudio(true)
.setTransformationRequest(
new TransformationRequest.Builder().setResolution(480).build())
.setEncoderFactory(
new DefaultEncoderFactory(EncoderSelector.DEFAULT, /* enableFallback= */ false))
.build();
long expectedDurationMs = 967;
TransformationTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(/* testId= */ "videoOnly_completesWithConsistentDuration", AVC_VIDEO_URI_STRING);
assertThat(result.transformationResult.durationMs).isEqualTo(expectedDurationMs);
}
}

2
libraries/transformer/src/main/java/androidx/media3/transformer/AdvancedFrameProcessor.java
View File

@ -124,7 +124,7 @@ public final class AdvancedFrameProcessor implements GlFrameProcessor {
}
@Override
public void updateProgramAndDraw(long presentationTimeNs) {
public void updateProgramAndDraw(long presentationTimeUs) {
checkStateNotNull(glProgram);
glProgram.use();
glProgram.bindAttributesAndUniforms();

2
libraries/transformer/src/main/java/androidx/media3/transformer/ExternalCopyFrameProcessor.java
View File

@ -101,7 +101,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
@Override
public void updateProgramAndDraw(long presentationTimeNs) {
public void updateProgramAndDraw(long presentationTimeUs) {
checkStateNotNull(glProgram);
glProgram.use();
glProgram.bindAttributesAndUniforms();

7
libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java
View File

@ -412,7 +412,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
externalCopyFrameProcessor.setTextureTransformMatrix(textureTransformMatrix);
long presentationTimeNs = inputSurfaceTexture.getTimestamp();
externalCopyFrameProcessor.updateProgramAndDraw(presentationTimeNs);
long presentationTimeUs = presentationTimeNs / 1000;
externalCopyFrameProcessor.updateProgramAndDraw(presentationTimeUs);
for (int i = 0; i < frameProcessors.size() - 1; i++) {
Size outputSize = inputSizes.get(i + 1);
@ -423,11 +424,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
framebuffers[i + 1],
outputSize.getWidth(),
outputSize.getHeight());
frameProcessors.get(i).updateProgramAndDraw(presentationTimeNs);
frameProcessors.get(i).updateProgramAndDraw(presentationTimeUs);
}
if (!frameProcessors.isEmpty()) {
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
getLast(frameProcessors).updateProgramAndDraw(presentationTimeNs);
getLast(frameProcessors).updateProgramAndDraw(presentationTimeUs);
}
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeNs);

4
libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java
View File

@ -60,9 +60,9 @@ public interface GlFrameProcessor {
* <p>The frame processor must be {@linkplain #initialize(int) initialized}. The caller is
* responsible for focussing the correct render target before calling this method.
*
* @param presentationTimeNs The presentation timestamp of the current frame, in nanoseconds.
* @param presentationTimeUs The presentation timestamp of the current frame, in microseconds.
*/
void updateProgramAndDraw(long presentationTimeNs);
void updateProgramAndDraw(long presentationTimeUs);
/** Releases all resources. */
void release();

6
libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java
View File

@ -17,6 +17,7 @@
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Util.maxValue;
import static androidx.media3.common.util.Util.minValue;
import android.util.SparseIntArray;
@ -240,4 +241,9 @@ import java.nio.ByteBuffer;
}
return trackTimeUs - minTrackTimeUs <= MAX_TRACK_WRITE_AHEAD_US;
}
/** Returns the duration of the longest track in milliseconds. */
public long getDurationMs() {
return Util.usToMs(maxValue(trackTypeToTimeUs));
}
}

4
libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java
View File

@ -152,8 +152,8 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
}
@Override
public void updateProgramAndDraw(long presentationTimeNs) {
checkStateNotNull(advancedFrameProcessor).updateProgramAndDraw(presentationTimeNs);
public void updateProgramAndDraw(long presentationTimeUs) {
checkStateNotNull(advancedFrameProcessor).updateProgramAndDraw(presentationTimeUs);
}
@Override

4
libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java
View File

@ -176,8 +176,8 @@ public final class ScaleToFitFrameProcessor implements GlFrameProcessor {
}
@Override
public void updateProgramAndDraw(long presentationTimeNs) {
checkStateNotNull(advancedFrameProcessor).updateProgramAndDraw(presentationTimeNs);
public void updateProgramAndDraw(long presentationTimeUs) {
checkStateNotNull(advancedFrameProcessor).updateProgramAndDraw(presentationTimeUs);
}
@Override

28
libraries/transformer/src/main/java/androidx/media3/transformer/TransformationResult.java
View File

@ -27,16 +27,29 @@ public final class TransformationResult {
/** A builder for {@link TransformationResult} instances. */
public static final class Builder {
private long durationMs;
private long fileSizeBytes;
private int averageAudioBitrate;
private int averageVideoBitrate;
public Builder() {
durationMs = C.TIME_UNSET;
fileSizeBytes = C.LENGTH_UNSET;
averageAudioBitrate = C.RATE_UNSET_INT;
averageVideoBitrate = C.RATE_UNSET_INT;
}
/**
* Sets the duration of the video in milliseconds.
*
* <p>Input must be positive or {@link C#TIME_UNSET}.
*/
public Builder setDurationMs(long durationMs) {
checkArgument(durationMs > 0 || durationMs == C.TIME_UNSET);
this.durationMs = durationMs;
return this;
}
/**
* Sets the file size in bytes.
*
@ -71,10 +84,13 @@ public final class TransformationResult {
}
public TransformationResult build() {
return new TransformationResult(fileSizeBytes, averageAudioBitrate, averageVideoBitrate);
return new TransformationResult(
durationMs, fileSizeBytes, averageAudioBitrate, averageVideoBitrate);
}
}
/** The duration of the video in milliseconds, or {@link C#TIME_UNSET} if unset or unknown. */
public final long durationMs;
/** The size of the file in bytes, or {@link C#LENGTH_UNSET} if unset or unknown. */
public final long fileSizeBytes;
/**
@ -87,7 +103,8 @@ public final class TransformationResult {
public final int averageVideoBitrate;
private TransformationResult(
long fileSizeBytes, int averageAudioBitrate, int averageVideoBitrate) {
long durationMs, long fileSizeBytes, int averageAudioBitrate, int averageVideoBitrate) {
this.durationMs = durationMs;
this.fileSizeBytes = fileSizeBytes;
this.averageAudioBitrate = averageAudioBitrate;
this.averageVideoBitrate = averageVideoBitrate;
@ -95,6 +112,7 @@ public final class TransformationResult {
public Builder buildUpon() {
return new Builder()
.setDurationMs(durationMs)
.setFileSizeBytes(fileSizeBytes)
.setAverageAudioBitrate(averageAudioBitrate)
.setAverageVideoBitrate(averageVideoBitrate);
@ -109,14 +127,16 @@ public final class TransformationResult {
return false;
}
TransformationResult result = (TransformationResult) o;
return fileSizeBytes == result.fileSizeBytes
return durationMs == result.durationMs
&& fileSizeBytes == result.fileSizeBytes
&& averageAudioBitrate == result.averageAudioBitrate
&& averageVideoBitrate == result.averageVideoBitrate;
}
@Override
public int hashCode() {
int result = (int) fileSizeBytes;
int result = (int) durationMs;
result = 31 * result + (int) fileSizeBytes;
result = 31 * result + averageAudioBitrate;
result = 31 * result + averageVideoBitrate;
return result;

1
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View File

@ -1002,6 +1002,7 @@ public final class Transformer {
} else {
TransformationResult result =
new TransformationResult.Builder()
.setDurationMs(muxerWrapper.getDurationMs())
.setAverageAudioBitrate(muxerWrapper.getTrackAverageBitrate(C.TRACK_TYPE_AUDIO))
.setAverageVideoBitrate(muxerWrapper.getTrackAverageBitrate(C.TRACK_TYPE_VIDEO))
.build();