loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix SequenceAssetLoader signalling End Of Video Input twice

Browse Source 
PiperOrigin-RevId: 560170216
This commit is contained in:
tofunmi 2023-08-25 12:44:22 -07:00 committed by Copybara-Service
parent 99ae44efa4
commit 667103f2bd
8 changed files with 135 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
View File

@ -626,6 +626,33 @@ public class TransformerEndToEndTest {
assertThat(result.exportResult.videoFrameCount).isEqualTo(95);
}
@Test
public void loopingImage_loopingSequenceIsLongest_producesExpectedResult() throws Exception {
Transformer transformer = new Transformer.Builder(context).build();
String testId = "loopingImage_producesExpectedResult";
EditedMediaItem audioEditedMediaItem =
new EditedMediaItem.Builder(MediaItem.fromUri(MP3_ASSET_URI_STRING)).build();
EditedMediaItemSequence audioSequence = new EditedMediaItemSequence(audioEditedMediaItem);
EditedMediaItem imageEditedMediaItem =
new EditedMediaItem.Builder(MediaItem.fromUri(PNG_ASSET_URI_STRING))
.setDurationUs(1_050_000)
.setFrameRate(20)
.build();
EditedMediaItemSequence loopingImageSequence =
new EditedMediaItemSequence(
ImmutableList.of(imageEditedMediaItem, imageEditedMediaItem), /* isLooping= */ true);
Composition composition = new Composition.Builder(audioSequence, loopingImageSequence).build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, composition);
assertThat(result.exportResult.processedInputs).hasSize(3);
assertThat(result.exportResult.channelCount).isEqualTo(1);
assertThat(result.exportResult.durationMs).isEqualTo(1000);
}
@Test
public void audioTranscode_processesInInt16Pcm() throws Exception {
String testId = "audioTranscode_processesInInt16Pcm";

36
libraries/transformer/src/main/java/androidx/media3/transformer/ImageAssetLoader.java
View File

@ -20,6 +20,9 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.transformer.ExportException.ERROR_CODE_IO_UNSPECIFIED;
import static androidx.media3.transformer.ExportException.ERROR_CODE_UNSPECIFIED;
import static androidx.media3.transformer.SampleConsumer.INPUT_RESULT_END_OF_STREAM;
import static androidx.media3.transformer.SampleConsumer.INPUT_RESULT_SUCCESS;
import static androidx.media3.transformer.SampleConsumer.INPUT_RESULT_TRY_AGAIN_LATER;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
@ -42,6 +45,7 @@ import androidx.media3.common.util.Util;
import androidx.media3.datasource.DataSource;
import androidx.media3.datasource.DataSourceBitmapLoader;
import androidx.media3.datasource.DefaultDataSource;
import androidx.media3.transformer.SampleConsumer.InputResult;
import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
@ -174,20 +178,34 @@ public final class ImageAssetLoader implements AssetLoader {
try {
if (sampleConsumer == null) {
sampleConsumer = listener.onOutputFormat(format);
}
// TODO(b/262693274): consider using listener.onDurationUs() or the MediaItem change
// callback rather than setting duration here.
if (sampleConsumer == null
|| !sampleConsumer.queueInputBitmap(
bitmap,
new ConstantRateTimestampIterator(
editedMediaItem.durationUs, editedMediaItem.frameRate))) {
scheduledExecutorService.schedule(
() -> queueBitmapInternal(bitmap, format), QUEUE_BITMAP_INTERVAL_MS, MILLISECONDS);
return;
}
sampleConsumer.signalEndOfVideoInput();
// TODO(b/262693274): consider using listener.onDurationUs() or the MediaItem change
// callback rather than setting duration here.
@InputResult
int result =
sampleConsumer.queueInputBitmap(
bitmap,
new ConstantRateTimestampIterator(
editedMediaItem.durationUs, editedMediaItem.frameRate));
switch (result) {
case INPUT_RESULT_SUCCESS:
progress = 100;
sampleConsumer.signalEndOfVideoInput();
break;
case INPUT_RESULT_TRY_AGAIN_LATER:
scheduledExecutorService.schedule(
() -> queueBitmapInternal(bitmap, format), QUEUE_BITMAP_INTERVAL_MS, MILLISECONDS);
break;
case INPUT_RESULT_END_OF_STREAM:
progress = 100;
break;
default:
throw new IllegalStateException();
}
} catch (ExportException e) {
listener.onError(e);
} catch (RuntimeException e) {

49
libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java
View File

@ -15,19 +15,59 @@
*/
package androidx.media3.transformer;
import static java.lang.annotation.ElementType.TYPE_USE;
import android.graphics.Bitmap;
import android.view.Surface;
import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.OnInputFrameProcessedListener;
import androidx.media3.common.util.TimestampIterator;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.decoder.DecoderInputBuffer;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/** Consumer of encoded media samples, raw audio or raw video frames. */
@UnstableApi
public interface SampleConsumer {
/**
* Specifies the result of an input operation. One of {@link #INPUT_RESULT_SUCCESS}, {@link
* #INPUT_RESULT_TRY_AGAIN_LATER} or {@link #INPUT_RESULT_END_OF_STREAM}.
*/
@Documented
@Retention(RetentionPolicy.SOURCE)
@Target(TYPE_USE)
@IntDef({INPUT_RESULT_SUCCESS, INPUT_RESULT_TRY_AGAIN_LATER, INPUT_RESULT_END_OF_STREAM})
@interface InputResult {}
/**
* The operation of queueing input was successful.
*
* <p>The caller can queue more input or signal {@link #signalEndOfVideoInput() signal end of
* input}.
*/
int INPUT_RESULT_SUCCESS = 1;
/**
* The operation of queueing/registering input was unsuccessful.
*
* <p>The caller should queue try again later.
*/
int INPUT_RESULT_TRY_AGAIN_LATER = 2;
/**
* The operation of queueing input successful and end of input has been automatically signalled.
*
* <p>The caller should not {@link #signalEndOfVideoInput() signal end of input} as this has
* already been done internally.
*/
int INPUT_RESULT_END_OF_STREAM = 3;
/**
* Returns a {@link DecoderInputBuffer}, if available.
*
@ -74,8 +114,10 @@ public interface SampleConsumer {
* @param inputBitmap The {@link Bitmap} to queue to the consumer.
* @param inStreamOffsetsUs The times within the current stream that the bitmap should be
* displayed at. The timestamps should be monotonically increasing.
* @return The {@link InputResult} describing the result of the operation.
*/
default boolean queueInputBitmap(Bitmap inputBitmap, TimestampIterator inStreamOffsetsUs) {
default @InputResult int queueInputBitmap(
Bitmap inputBitmap, TimestampIterator inStreamOffsetsUs) {
throw new UnsupportedOperationException();
}
@ -99,10 +141,9 @@ public interface SampleConsumer {
*
* @param texId The ID of the texture to queue to the consumer.
* @param presentationTimeUs The presentation time for the texture, in microseconds.
* @return Whether the texture was successfully queued. If {@code false}, the caller should try
* again later.
* @return The {@link InputResult} describing the result of the operation.
*/
default boolean queueInputTexture(int texId, long presentationTimeUs) {
default @InputResult int queueInputTexture(int texId, long presentationTimeUs) {
throw new UnsupportedOperationException();
}

13
libraries/transformer/src/main/java/androidx/media3/transformer/SequenceAssetLoader.java
View File

@ -398,21 +398,23 @@ import java.util.concurrent.atomic.AtomicInteger;
}
@Override
public boolean queueInputBitmap(Bitmap inputBitmap, TimestampIterator inStreamOffsetsUs) {
public @InputResult int queueInputBitmap(
Bitmap inputBitmap, TimestampIterator inStreamOffsetsUs) {
if (isLooping) {
long lastOffsetUs = C.TIME_UNSET;
while (inStreamOffsetsUs.hasNext()) {
long offsetUs = inStreamOffsetsUs.next();
if (totalDurationUs + offsetUs > maxSequenceDurationUs) {
if (!isMaxSequenceDurationUsFinal) {
return false;
return INPUT_RESULT_TRY_AGAIN_LATER;
}
if (lastOffsetUs == C.TIME_UNSET) {
if (!videoLoopingEnded) {
videoLoopingEnded = true;
signalEndOfVideoInput();
return INPUT_RESULT_END_OF_STREAM;
}
return false;
return INPUT_RESULT_TRY_AGAIN_LATER;
}
inStreamOffsetsUs = new ClippingIterator(inStreamOffsetsUs.copyOf(), lastOffsetUs);
videoLoopingEnded = true;
@ -430,14 +432,15 @@ import java.util.concurrent.atomic.AtomicInteger;
}
@Override
public boolean queueInputTexture(int texId, long presentationTimeUs) {
public @InputResult int queueInputTexture(int texId, long presentationTimeUs) {
long globalTimestampUs = totalDurationUs + presentationTimeUs;
if (isLooping && globalTimestampUs >= maxSequenceDurationUs) {
if (isMaxSequenceDurationUsFinal && !videoLoopingEnded) {
videoLoopingEnded = true;
signalEndOfVideoInput();
return INPUT_RESULT_END_OF_STREAM;
}
return false;
return INPUT_RESULT_TRY_AGAIN_LATER;
}
return sampleConsumer.queueInputTexture(texId, presentationTimeUs);
}

13
libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java
View File

@ -215,8 +215,11 @@ import java.util.concurrent.atomic.AtomicLong;
}
@Override
public boolean queueInputBitmap(Bitmap inputBitmap, TimestampIterator inStreamOffsetsUs) {
return videoFrameProcessor.queueInputBitmap(inputBitmap, inStreamOffsetsUs);
public @InputResult int queueInputBitmap(
Bitmap inputBitmap, TimestampIterator inStreamOffsetsUs) {
return videoFrameProcessor.queueInputBitmap(inputBitmap, inStreamOffsetsUs)
? INPUT_RESULT_SUCCESS
: INPUT_RESULT_TRY_AGAIN_LATER;
}
@Override
@ -225,8 +228,10 @@ import java.util.concurrent.atomic.AtomicLong;
}
@Override
public boolean queueInputTexture(int texId, long presentationTimeUs) {
return videoFrameProcessor.queueInputTexture(texId, presentationTimeUs);
public @InputResult int queueInputTexture(int texId, long presentationTimeUs) {
return videoFrameProcessor.queueInputTexture(texId, presentationTimeUs)
? INPUT_RESULT_SUCCESS
: INPUT_RESULT_TRY_AGAIN_LATER;
}
@Override

13
libraries/transformer/src/main/java/androidx/media3/transformer/TextureAssetLoader.java
View File

@ -18,6 +18,8 @@ package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.transformer.ExportException.ERROR_CODE_UNSPECIFIED;
import static androidx.media3.transformer.SampleConsumer.INPUT_RESULT_END_OF_STREAM;
import static androidx.media3.transformer.SampleConsumer.INPUT_RESULT_TRY_AGAIN_LATER;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED;
import static java.lang.Math.round;
@ -28,6 +30,7 @@ import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.OnInputFrameProcessedListener;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.transformer.SampleConsumer.InputResult;
import com.google.common.collect.ImmutableMap;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@ -50,6 +53,7 @@ public final class TextureAssetLoader implements AssetLoader {
private @MonotonicNonNull SampleConsumer sampleConsumer;
private @Transformer.ProgressState int progressState;
private boolean isTrackAdded;
private boolean isEndOfStreamSignaled;
private volatile boolean isStarted;
private volatile long lastQueuedPresentationTimeUs;
@ -136,9 +140,13 @@ public final class TextureAssetLoader implements AssetLoader {
sampleConsumer.setOnInputFrameProcessedListener(frameProcessedListener);
}
}
if (!sampleConsumer.queueInputTexture(texId, presentationTimeUs)) {
@InputResult int result = sampleConsumer.queueInputTexture(texId, presentationTimeUs);
if (result == INPUT_RESULT_TRY_AGAIN_LATER) {
return false;
}
if (result == INPUT_RESULT_END_OF_STREAM) {
isEndOfStreamSignaled = true;
}
lastQueuedPresentationTimeUs = presentationTimeUs;
return true;
} catch (ExportException e) {
@ -156,7 +164,10 @@ public final class TextureAssetLoader implements AssetLoader {
*/
public void signalEndOfVideoInput() {
try {
if (!isEndOfStreamSignaled) {
isEndOfStreamSignaled = true;
checkNotNull(sampleConsumer).signalEndOfVideoInput();
}
} catch (RuntimeException e) {
assetLoaderListener.onError(ExportException.createForAssetLoader(e, ERROR_CODE_UNSPECIFIED));
}

5
libraries/transformer/src/test/java/androidx/media3/transformer/ImageAssetLoaderTest.java
View File

@ -127,8 +127,9 @@ public class ImageAssetLoaderTest {
private static final class FakeSampleConsumer implements SampleConsumer {
@Override
public boolean queueInputBitmap(Bitmap inputBitmap, TimestampIterator inStreamOffsetsUs) {
return true;
public @InputResult int queueInputBitmap(
Bitmap inputBitmap, TimestampIterator inStreamOffsetsUs) {
return INPUT_RESULT_SUCCESS;
}
@Override

4
libraries/transformer/src/test/java/androidx/media3/transformer/TextureAssetLoaderTest.java
View File

@ -141,8 +141,8 @@ public class TextureAssetLoaderTest {
public void setOnInputFrameProcessedListener(OnInputFrameProcessedListener listener) {}
@Override
public boolean queueInputTexture(int texId, long presentationTimeUs) {
return true;
public @InputResult int queueInputTexture(int texId, long presentationTimeUs) {
return INPUT_RESULT_SUCCESS;
}
@Override