diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlaybackTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlaybackTest.java
index 753a0a9653..9696706ddb 100644
--- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlaybackTest.java
+++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlaybackTest.java
@@ -374,6 +374,44 @@ public class CompositionPlaybackTest {
     assertThat(inputTimestampRecordingShaderProgram.getInputTimestampsUs()).isEmpty();
   }
 
+  @Test
+  public void playback_compositionWithSecondSequenceRemoveVideo_rendersVideoFromFirstSequence()
+      throws Exception {
+    InputTimestampRecordingShaderProgram inputTimestampRecordingShaderProgram =
+        new InputTimestampRecordingShaderProgram();
+
+    EditedMediaItem videoEditedMediaItem =
+        new EditedMediaItem.Builder(VIDEO_MEDIA_ITEM)
+            .setDurationUs(VIDEO_DURATION_US)
+            .setEffects(
+                new Effects(
+                    /* audioProcessors= */ ImmutableList.of(),
+                    /* videoEffects= */ ImmutableList.of(
+                        (GlEffect) (context, useHdr) -> inputTimestampRecordingShaderProgram)))
+            .build();
+    EditedMediaItem videoEditedMediaItemRemoveVideo =
+        videoEditedMediaItem.buildUpon().setRemoveVideo(true).build();
+    Composition composition =
+        new Composition.Builder(
+                new EditedMediaItemSequence.Builder(videoEditedMediaItem).build(),
+                new EditedMediaItemSequence.Builder(videoEditedMediaItemRemoveVideo).build())
+            .build();
+
+    getInstrumentation()
+        .runOnMainSync(
+            () -> {
+              player = new CompositionPlayer.Builder(context).build();
+              player.addListener(playerTestListener);
+              player.setComposition(composition);
+              player.prepare();
+              player.play();
+            });
+    playerTestListener.waitUntilPlayerEnded();
+
+    assertThat(inputTimestampRecordingShaderProgram.getInputTimestampsUs())
+        .isEqualTo(VIDEO_TIMESTAMPS_US);
+  }
+
   @Test
   public void playback_withCompositionEffect_effectIsApplied() throws Exception {
     EditedMediaItem editedMediaItem =
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java
index afc8f11ee4..90975e30b0 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java
@@ -682,7 +682,6 @@ public final class CompositionPlayer extends SimpleBasePlayer
     long primarySequenceDurationUs =
         getSequenceDurationUs(checkNotNull(composition.sequences.get(0)));
     // Video playback is disabled when one EditedMediaItem removes video.
-    boolean disableVideoPlayback = shouldDisableVideoPlayback(composition);
     for (int i = 0; i < composition.sequences.size(); i++) {
       EditedMediaItemSequence editedMediaItemSequence = composition.sequences.get(i);
       SequenceRenderersFactory sequenceRenderersFactory =
@@ -704,9 +703,15 @@ public final class CompositionPlayer extends SimpleBasePlayer
               .setHandleAudioBecomingNoisy(true)
               .setClock(clock);
 
-      if (i == 0) {
-        playerBuilder.setTrackSelector(new CompositionTrackSelector(context, disableVideoPlayback));
+      boolean disableVideoPlayback = false;
+      for (int j = 0; j < editedMediaItemSequence.editedMediaItems.size(); j++) {
+        if (editedMediaItemSequence.editedMediaItems.get(j).removeVideo) {
+          disableVideoPlayback = true;
+          break;
+        }
       }
+      playerBuilder.setTrackSelector(
+          new CompositionTrackSelector(context, /* sequenceIndex= */ i, disableVideoPlayback));
 
       ExoPlayer player = playerBuilder.build();
       player.addListener(new PlayerListener(i));
@@ -992,19 +997,6 @@ public final class CompositionPlayer extends SimpleBasePlayer
     return compositionDurationUs;
   }
 
-  private static boolean shouldDisableVideoPlayback(Composition composition) {
-    for (int i = 0; i < composition.sequences.size(); i++) {
-      EditedMediaItemSequence editedMediaItemSequence = composition.sequences.get(i);
-      for (int j = 0; j < editedMediaItemSequence.editedMediaItems.size(); j++) {
-        EditedMediaItem editedMediaItem = editedMediaItemSequence.editedMediaItems.get(j);
-        if (editedMediaItem.removeVideo) {
-          return true;
-        }
-      }
-    }
-    return false;
-  }
-
   /**
    * A {@link VideoFrameReleaseControl.FrameTimingEvaluator} for composition frames.
    *
@@ -1095,10 +1087,13 @@ public final class CompositionPlayer extends SimpleBasePlayer
   private static final class CompositionTrackSelector extends DefaultTrackSelector {
 
     private static final String SILENCE_AUDIO_TRACK_GROUP_ID = "1:";
+    private final int sequenceIndex;
     private final boolean disableVideoPlayback;
 
-    public CompositionTrackSelector(Context context, boolean disableVideoPlayback) {
+    public CompositionTrackSelector(
+        Context context, int sequenceIndex, boolean disableVideoPlayback) {
       super(context);
+      this.sequenceIndex = sequenceIndex;
       this.disableVideoPlayback = disableVideoPlayback;
     }
 
@@ -1110,41 +1105,44 @@ public final class CompositionPlayer extends SimpleBasePlayer
         @RendererCapabilities.AdaptiveSupport int[] rendererMixedMimeTypeAdaptationSupports,
         Parameters params)
         throws ExoPlaybackException {
-      int audioRenderIndex = C.INDEX_UNSET;
-      for (int i = 0; i < mappedTrackInfo.getRendererCount(); i++) {
-        if (mappedTrackInfo.getRendererType(i) == C.TRACK_TYPE_AUDIO) {
-          audioRenderIndex = i;
-          break;
-        }
-      }
-      checkState(audioRenderIndex != C.INDEX_UNSET);
-
-      TrackGroupArray audioTrackGroups = mappedTrackInfo.getTrackGroups(audioRenderIndex);
-      // If there's only one audio TrackGroup, it'll be silence, there's no need to override track
-      // selection.
-      if (audioTrackGroups.length > 1) {
-        boolean mediaAudioIsPlayable = false;
-        int silenceAudioTrackGroupIndex = C.INDEX_UNSET;
-        for (int i = 0; i < audioTrackGroups.length; i++) {
-          if (audioTrackGroups.get(i).id.startsWith(SILENCE_AUDIO_TRACK_GROUP_ID)) {
-            silenceAudioTrackGroupIndex = i;
-            continue;
-          }
-          // For non-silence tracks
-          for (int j = 0; j < audioTrackGroups.get(i).length; j++) {
-            mediaAudioIsPlayable |=
-                RendererCapabilities.getFormatSupport(
-                        rendererFormatSupports[audioRenderIndex][i][j])
-                    == C.FORMAT_HANDLED;
+      if (sequenceIndex == 0) {
+        // Currently silence is only generated for the zero-indexed sequence.
+        int audioRenderIndex = C.INDEX_UNSET;
+        for (int i = 0; i < mappedTrackInfo.getRendererCount(); i++) {
+          if (mappedTrackInfo.getRendererType(i) == C.TRACK_TYPE_AUDIO) {
+            audioRenderIndex = i;
+            break;
           }
         }
-        checkState(silenceAudioTrackGroupIndex != C.INDEX_UNSET);
+        checkState(audioRenderIndex != C.INDEX_UNSET);
 
-        if (mediaAudioIsPlayable) {
-          // Disable silence if the media's audio track is playable.
-          int silenceAudioTrackIndex = audioTrackGroups.length - 1;
-          rendererFormatSupports[audioRenderIndex][silenceAudioTrackIndex][0] =
-              RendererCapabilities.create(C.FORMAT_UNSUPPORTED_TYPE);
+        TrackGroupArray audioTrackGroups = mappedTrackInfo.getTrackGroups(audioRenderIndex);
+        // If there's only one audio TrackGroup, it'll be silence, there's no need to override track
+        // selection.
+        if (audioTrackGroups.length > 1) {
+          boolean mediaAudioIsPlayable = false;
+          int silenceAudioTrackGroupIndex = C.INDEX_UNSET;
+          for (int i = 0; i < audioTrackGroups.length; i++) {
+            if (audioTrackGroups.get(i).id.startsWith(SILENCE_AUDIO_TRACK_GROUP_ID)) {
+              silenceAudioTrackGroupIndex = i;
+              continue;
+            }
+            // For non-silence tracks
+            for (int j = 0; j < audioTrackGroups.get(i).length; j++) {
+              mediaAudioIsPlayable |=
+                  RendererCapabilities.getFormatSupport(
+                          rendererFormatSupports[audioRenderIndex][i][j])
+                      == C.FORMAT_HANDLED;
+            }
+          }
+          checkState(silenceAudioTrackGroupIndex != C.INDEX_UNSET);
+
+          if (mediaAudioIsPlayable) {
+            // Disable silence if the media's audio track is playable.
+            int silenceAudioTrackIndex = audioTrackGroups.length - 1;
+            rendererFormatSupports[audioRenderIndex][silenceAudioTrackIndex][0] =
+                RendererCapabilities.create(C.FORMAT_UNSUPPORTED_TYPE);
+          }
         }
       }