diff --git a/libraries/common/src/main/java/androidx/media3/common/Format.java b/libraries/common/src/main/java/androidx/media3/common/Format.java
index e54ec79246..3db7583314 100644
--- a/libraries/common/src/main/java/androidx/media3/common/Format.java
+++ b/libraries/common/src/main/java/androidx/media3/common/Format.java
@@ -953,10 +953,11 @@ public final class Format implements Bundleable {
     language = Util.normalizeLanguageCode(builder.language);
     @Nullable String tmpLabel = builder.label;
     labels = builder.labels == null ? new ArrayList<>() : builder.labels;
-    if (labels.isEmpty() && tmpLabel != null && !tmpLabel.isEmpty()) {
+    if (labels.isEmpty() && !TextUtils.isEmpty(tmpLabel)) {
       labels.add(new Label(language, tmpLabel));
     }
     label = makeLabelIfNeeded(tmpLabel, labels);
+    checkLabels(label, labels);
     selectionFlags = builder.selectionFlags;
     roleFlags = builder.roleFlags;
     averageBitrate = builder.averageBitrate;
@@ -1005,7 +1006,7 @@ public final class Format implements Bundleable {
   }
 
   private @Nullable String makeLabelIfNeeded(@Nullable String label, List<Label> labels) {
-    if (label == null || label.isEmpty()) {
+    if (TextUtils.isEmpty(label)) {
       for (Label l : labels) {
         if (TextUtils.equals(l.lang, language)) {
           return l.value;
@@ -1018,6 +1019,15 @@ public final class Format implements Bundleable {
     return label;
   }
 
+  private void checkLabels(@Nullable String label, List<Label> labels)
+      throws IllegalStateException {
+    if (!TextUtils.isEmpty(label)
+        && !labels.isEmpty()
+        && labels.stream().noneMatch(l -> TextUtils.equals(l.value, label))) {
+      throw new IllegalStateException();
+    }
+  }
+
   /** Returns a {@link Format.Builder} initialized with the values of this instance. */
   @UnstableApi
   public Builder buildUpon() {
diff --git a/libraries/common/src/test/java/androidx/media3/common/FormatTest.java b/libraries/common/src/test/java/androidx/media3/common/FormatTest.java
index 1fb71abb7c..40ab4fc560 100644
--- a/libraries/common/src/test/java/androidx/media3/common/FormatTest.java
+++ b/libraries/common/src/test/java/androidx/media3/common/FormatTest.java
@@ -57,6 +57,48 @@ public final class FormatTest {
     assertThat(formatWithMetadataExcluded).isEqualTo(format.buildUpon().setMetadata(null).build());
   }
 
+  @Test
+  public void formatBuild_withLabelAndWithoutLabels_labelIsInLabels() {
+    Format format = createTestFormat();
+    format = format.buildUpon().setLabels(null).build();
+
+    assertThat(format.labels.size()).isEqualTo(1);
+    assertThat(format.labels.get(0).value).isEqualTo(format.label);
+  }
+
+  @Test
+  public void formatBuild_withLabelsAndLanguageMatchingAndWithoutLabel_theLanguageMatchIsInLabel() {
+    Format format = createTestFormat();
+    format.labels.add(new Label("language", "matchingLabel"));
+    format = format.buildUpon().setLabel(null).build();
+
+    assertThat(format.label).isEqualTo("matchingLabel");
+  }
+
+  @Test
+  public void formatBuild_withLabelsAndNoLanguageMatchingAndWithoutLabel_theFirstIsInLabel() {
+    Format format = createTestFormat();
+    format.labels.add(new Label("otherLanguage", "otherLabel"));
+    format = format.buildUpon().setLabel(null).build();
+
+    assertThat(format.label).isEqualTo("label");
+  }
+
+  @Test
+  public void formatBuild_withoutLabelsOrLabel_theyAreEmpty() {
+    Format format = createTestFormat();
+    format = format.buildUpon().setLabel(null).setLabels(null).build();
+
+    assertThat(format.label).isEqualTo(null);
+    assertThat(format.labels.size()).isEqualTo(0);
+  }
+
+  @Test(expected = IllegalStateException.class)
+  public void formatBuild_withLabelAndLabelsSetButNotMatching_throwsException() {
+    Format format = createTestFormat();
+    format.buildUpon().setLabel("otherLabel").build();
+  }
+
   private static Format createTestFormat() {
     byte[] initData1 = new byte[] {1, 2, 3};
     byte[] initData2 = new byte[] {4, 5, 6};