loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add sample rate fallback to DefaultEncoderFactory

Browse Source 
After this change if a sample rate is requested that is not supported by the available encoders and enableFallback is true, DefaultEncoderFactory will fall back to the encoder with the closest matching sample rate.

In the case when an encoding profile is included in requestedAudioEncoderSettings, an encoder matching that profile will continue to be preferenced.

PiperOrigin-RevId: 708295869
This commit is contained in:
Googler 2024-12-20 05:47:18 -08:00 committed by Copybara-Service
parent 5c3c3b91f3
commit c12b1768a6
7 changed files with 171 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java
View File

@ -120,10 +120,11 @@ public final class ConfigurationActivity extends AppCompatActivity {
// Audio effect selections.
public static final int HIGH_PITCHED_INDEX = 0;
public static final int SAMPLE_RATE_INDEX = 1;
public static final int SKIP_SILENCE_INDEX = 2;
public static final int CHANNEL_MIXING_INDEX = 3;
public static final int VOLUME_SCALING_INDEX = 4;
public static final int SAMPLE_RATE_48K_INDEX = 1;
public static final int SAMPLE_RATE_96K_INDEX = 2;
public static final int SKIP_SILENCE_INDEX = 3;
public static final int CHANNEL_MIXING_INDEX = 4;
public static final int VOLUME_SCALING_INDEX = 5;
// Color filter options.
public static final int COLOR_FILTER_GRAYSCALE = 0;

8
demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java
View File

@ -393,14 +393,18 @@ public final class TransformerActivity extends AppCompatActivity {
ImmutableList.Builder<AudioProcessor> processors = new ImmutableList.Builder<>();
if (selectedAudioEffects[ConfigurationActivity.HIGH_PITCHED_INDEX]
|| selectedAudioEffects[ConfigurationActivity.SAMPLE_RATE_INDEX]) {
|| selectedAudioEffects[ConfigurationActivity.SAMPLE_RATE_48K_INDEX]
|| selectedAudioEffects[ConfigurationActivity.SAMPLE_RATE_96K_INDEX]) {
SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor();
if (selectedAudioEffects[ConfigurationActivity.HIGH_PITCHED_INDEX]) {
sonicAudioProcessor.setPitch(2f);
}
if (selectedAudioEffects[ConfigurationActivity.SAMPLE_RATE_INDEX]) {
if (selectedAudioEffects[ConfigurationActivity.SAMPLE_RATE_48K_INDEX]) {
sonicAudioProcessor.setOutputSampleRateHz(48_000);
}
if (selectedAudioEffects[ConfigurationActivity.SAMPLE_RATE_96K_INDEX]) {
sonicAudioProcessor.setOutputSampleRateHz(96_000);
}
processors.add(sonicAudioProcessor);
}

1
demos/transformer/src/main/res/values/arrays.xml
View File

@ -35,6 +35,7 @@
<string-array name="audio_effects_names">
<item>High pitched</item>
<item>Sample rate of 48000Hz</item>
<item>Sample rate of 96000Hz</item>
<item>Skip silence</item>
<item>Mix channels into mono</item>
<item>Scale volume to 50%</item>

1
libraries/transformer/src/main/java/androidx/media3/transformer/AudioSampleExporter.java
View File

@ -88,6 +88,7 @@ import org.checkerframework.dataflow.qual.Pure;
requestedEncoderFormat,
muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO)))
.build());
// TODO: b/324056144 - Fallback when sample rate is unsupported by encoder
encoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED);
encoderOutputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED);

78
libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java
View File

@ -33,7 +33,6 @@ import android.content.Context;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
import android.os.Build;
import android.util.Pair;
import android.util.Size;
import androidx.annotation.IntRange;
import androidx.annotation.Nullable;
@ -206,13 +205,14 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
}
MediaCodecInfo selectedEncoder = mediaCodecInfos.get(0);
boolean encoderSelectedForRequestedProfile = false;
if (requestedAudioEncoderSettings.profile != AudioEncoderSettings.NO_VALUE) {
for (int i = 0; i < mediaCodecInfos.size(); i++) {
MediaCodecInfo encoderInfo = mediaCodecInfos.get(i);
if (EncoderUtil.findSupportedEncodingProfiles(encoderInfo, format.sampleMimeType)
.contains(requestedAudioEncoderSettings.profile)) {
selectedEncoder = encoderInfo;
encoderSelectedForRequestedProfile = true;
if (format.sampleMimeType.equals(MimeTypes.AUDIO_AAC)) {
mediaFormat.setInteger(
MediaFormat.KEY_AAC_PROFILE, requestedAudioEncoderSettings.profile);
@ -223,7 +223,16 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
}
}
}
if (!encoderSelectedForRequestedProfile && enableFallback) {
@Nullable
EncoderQueryResult encoderQueryResult =
findAudioEncoderWithClosestSupportedFormat(format, mediaCodecInfos);
if (encoderQueryResult != null) {
selectedEncoder = encoderQueryResult.encoder;
format = encoderQueryResult.supportedFormat;
mediaFormat = createMediaFormatFromFormat(format);
}
}
if (requestedAudioEncoderSettings.bitrate != AudioEncoderSettings.NO_VALUE) {
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, requestedAudioEncoderSettings.bitrate);
}
@ -261,7 +270,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
@Nullable
VideoEncoderQueryResult encoderAndClosestFormatSupport =
findEncoderWithClosestSupportedFormat(
findVideoEncoderWithClosestSupportedFormat(
format, requestedVideoEncoderSettings, videoEncoderSelector, enableFallback);
if (encoderAndClosestFormatSupport == null) {
@ -402,15 +411,14 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
}
/**
* Finds an {@linkplain MediaCodecInfo encoder} that supports a format closest to the requested
* format.
* Finds a video {@linkplain MediaCodecInfo encoder} that supports a format closest to the
* requested format.
*
* <p>Returns the {@linkplain MediaCodecInfo encoder} and the supported {@link Format} in a {@link
* Pair}, or {@code null} if none is found.
* <p>Returns a {@link VideoEncoderQueryResult}, or {@code null} if no encoder is found.
*/
@RequiresNonNull("#1.sampleMimeType")
@Nullable
private static VideoEncoderQueryResult findEncoderWithClosestSupportedFormat(
private static VideoEncoderQueryResult findVideoEncoderWithClosestSupportedFormat(
Format requestedFormat,
VideoEncoderSettings videoEncoderSettings,
EncoderSelector encoderSelector,
@ -576,17 +584,63 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
: Integer.MAX_VALUE); // Drops encoder.
}
private static final class VideoEncoderQueryResult {
/**
* Finds an audio {@linkplain MediaCodecInfo encoder} that supports a format closest to the
* requested format.
*
* <p>Returns a {@link EncoderQueryResult}, or {@code null} if no encoder is found.
*/
@RequiresNonNull("#1.sampleMimeType")
@Nullable
private static EncoderQueryResult findAudioEncoderWithClosestSupportedFormat(
Format requestedFormat, ImmutableList<MediaCodecInfo> filteredEncoderInfos) {
String mimeType = checkNotNull(requestedFormat.sampleMimeType);
if (filteredEncoderInfos.isEmpty()) {
return null;
}
MediaCodecInfo filteredEncoderInfo =
filterEncodersBySampleRate(filteredEncoderInfos, mimeType, requestedFormat.sampleRate)
.get(0);
int sampleRate =
EncoderUtil.getClosestSupportedSampleRate(
filteredEncoderInfo, mimeType, requestedFormat.sampleRate);
Format encoderFormat = requestedFormat.buildUpon().setSampleRate(sampleRate).build();
return new EncoderQueryResult(filteredEncoderInfo, encoderFormat);
}
/**
* Returns a list of {@linkplain MediaCodecInfo encoders} that support the requested sample rate
* most closely.
*/
private static ImmutableList<MediaCodecInfo> filterEncodersBySampleRate(
List<MediaCodecInfo> encoders, String mimeType, int requestedSampleRate) {
return filterEncoders(
encoders,
/* cost= */ (encoderInfo) -> {
int closestSupportedSampleRate =
EncoderUtil.getClosestSupportedSampleRate(encoderInfo, mimeType, requestedSampleRate);
return Math.abs(closestSupportedSampleRate - requestedSampleRate);
});
}
private static class EncoderQueryResult {
public final MediaCodecInfo encoder;
public final Format supportedFormat;
public EncoderQueryResult(MediaCodecInfo encoder, Format supportedFormat) {
this.encoder = encoder;
this.supportedFormat = supportedFormat;
}
}
private static final class VideoEncoderQueryResult extends EncoderQueryResult {
public final VideoEncoderSettings supportedEncoderSettings;
public VideoEncoderQueryResult(
MediaCodecInfo encoder,
Format supportedFormat,
VideoEncoderSettings supportedEncoderSettings) {
this.encoder = encoder;
this.supportedFormat = supportedFormat;
super(encoder, supportedFormat);
this.supportedEncoderSettings = supportedEncoderSettings;
}
}

32
libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java
View File

@ -384,6 +384,38 @@ public final class EncoderUtil {
Ints.asList(encoderInfo.getCapabilitiesForType(mimeType).colorFormats));
}
/**
* Returns the sample rate supported by the provided {@linkplain MediaCodecInfo encoder} that is
* closest to the provided sample rate.
*/
public static int getClosestSupportedSampleRate(
MediaCodecInfo encoderInfo, String mimeType, int requestedSampleRate) {
MediaCodecInfo.AudioCapabilities audioCapabilities =
encoderInfo.getCapabilitiesForType(mimeType).getAudioCapabilities();
@Nullable int[] supportedSampleRates = audioCapabilities.getSupportedSampleRates();
int closestSampleRate = Integer.MAX_VALUE;
if (supportedSampleRates != null) {
// The codec supports only discrete values.
for (int supportedSampleRate : supportedSampleRates) {
if (Math.abs(supportedSampleRate - requestedSampleRate)
< Math.abs(closestSampleRate - requestedSampleRate)) {
closestSampleRate = supportedSampleRate;
}
}
return closestSampleRate;
} else {
Range<Integer>[] ranges = audioCapabilities.getSupportedSampleRateRanges();
for (Range<Integer> range : ranges) {
int supportedSampleRate = range.clamp(requestedSampleRate);
if (Math.abs(supportedSampleRate - requestedSampleRate)
< Math.abs(closestSampleRate - requestedSampleRate)) {
closestSampleRate = supportedSampleRate;
}
}
}
return closestSampleRate;
}
/** Checks if a {@linkplain MediaCodecInfo codec} is hardware-accelerated. */
public static boolean isHardwareAccelerated(MediaCodecInfo encoderInfo, String mimeType) {
// TODO(b/214964116): Merge into MediaCodecUtil.

69
libraries/transformer/src/test/java/androidx/media3/transformer/DefaultEncoderFactoryTest.java
View File

@ -45,6 +45,7 @@ public class DefaultEncoderFactoryTest {
@Before
public void setUp() {
createShadowH264Encoder();
createShadowAacEncoder();
}
@After
@ -66,24 +67,40 @@ public class DefaultEncoderFactoryTest {
createShadowVideoEncoder(avcFormat, profileLevel, "test.transformer.avc.encoder");
}
private static void createShadowAacEncoder() {
MediaFormat format = new MediaFormat();
format.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_AUDIO_AAC);
MediaCodecInfo.CodecCapabilities capabilities =
MediaCodecInfoBuilder.CodecCapabilitiesBuilder.newBuilder()
.setMediaFormat(format)
.setIsEncoder(true)
.build();
createShadowEncoder("test.transformer.aac.encoder", capabilities);
}
private static void createShadowVideoEncoder(
MediaFormat supportedFormat,
MediaCodecInfo.CodecProfileLevel supportedProfileLevel,
String name) {
MediaCodecInfo.CodecCapabilities capabilities =
MediaCodecInfoBuilder.CodecCapabilitiesBuilder.newBuilder()
.setMediaFormat(supportedFormat)
.setIsEncoder(true)
.setColorFormats(
new int[] {MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible})
.setProfileLevels(new MediaCodecInfo.CodecProfileLevel[] {supportedProfileLevel})
.build();
createShadowEncoder(name, capabilities);
}
private static void createShadowEncoder(
String name, MediaCodecInfo.CodecCapabilities... capabilities) {
// ShadowMediaCodecList is static. The added encoders will be visible for every test.
ShadowMediaCodecList.addCodec(
MediaCodecInfoBuilder.newBuilder()
.setName(name)
.setIsEncoder(true)
.setCapabilities(
MediaCodecInfoBuilder.CodecCapabilitiesBuilder.newBuilder()
.setMediaFormat(supportedFormat)
.setIsEncoder(true)
.setColorFormats(
new int[] {MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible})
.setProfileLevels(
new MediaCodecInfo.CodecProfileLevel[] {supportedProfileLevel})
.build())
.setCapabilities(capabilities)
.build());
}
@ -247,6 +264,36 @@ public class DefaultEncoderFactoryTest {
.createForVideoEncoding(requestedVideoFormat));
}
@Test
public void createForAudioEncoding_unsupportedSampleRateWithFallback() throws Exception {
Format requestedAudioFormat = createAudioFormat(MimeTypes.AUDIO_AAC, /* sampleRate= */ 192_000);
Format actualAudioFormat =
new DefaultEncoderFactory.Builder(context)
.setEnableFallback(true)
.build()
.createForAudioEncoding(requestedAudioFormat)
.getConfigurationFormat();
assertThat(actualAudioFormat.sampleMimeType).isEqualTo(MimeTypes.AUDIO_AAC);
assertThat(actualAudioFormat.sampleRate).isEqualTo(96_000);
}
@Test
public void createForAudioEncoding_unsupportedSampleRateWithoutFallback() throws Exception {
Format requestedAudioFormat = createAudioFormat(MimeTypes.AUDIO_AAC, /* sampleRate= */ 192_000);
Format actualAudioFormat =
new DefaultEncoderFactory.Builder(context)
.setEnableFallback(false)
.build()
.createForAudioEncoding(requestedAudioFormat)
.getConfigurationFormat();
assertThat(actualAudioFormat.sampleMimeType).isEqualTo(MimeTypes.AUDIO_AAC);
assertThat(actualAudioFormat.sampleRate).isEqualTo(192_000);
}
private static Format createVideoFormat(String mimeType, int width, int height, int frameRate) {
return new Format.Builder()
.setWidth(width)
@ -256,4 +303,8 @@ public class DefaultEncoderFactoryTest {
.setSampleMimeType(mimeType)
.build();
}
private static Format createAudioFormat(String mimeType, int sampleRate) {
return new Format.Builder().setSampleRate(sampleRate).setSampleMimeType(mimeType).build();
}
}