diff --git a/RELEASENOTES.md b/RELEASENOTES.md
index 1171d35411..1c05a2375f 100644
--- a/RELEASENOTES.md
+++ b/RELEASENOTES.md
@@ -81,6 +81,8 @@
 * CacheDataSource: Check periodically if it's possible to read from/write to
   cache after deciding to bypass cache.
 * IMA extension:
+    * Fix the player getting stuck when an ad group fails to load
+      ([#3584](https://github.com/google/ExoPlayer/issues/3584)).
     * Work around loadAd not being called beore the LOADED AdEvent arrives
       ([#3552](https://github.com/google/ExoPlayer/issues/3552)).
     * Add support for playing non-Extractor content MediaSources in
diff --git a/demos/main/src/main/assets/media.exolist.json b/demos/main/src/main/assets/media.exolist.json
index 15183a4a8b..7052e7c436 100644
--- a/demos/main/src/main/assets/media.exolist.json
+++ b/demos/main/src/main/assets/media.exolist.json
@@ -566,6 +566,16 @@
         "name": "VMAP pre-roll single ad, mid-roll standard pods with 5 ads every 10 seconds for 1:40, post-roll single ad",
         "uri": "https://storage.googleapis.com/exoplayer-test-media-1/mkv/android-screens-lavf-56.36.100-aac-avc-main-1280x720.mkv",
         "ad_tag_uri": "https://pubads.g.doubleclick.net/gampad/ads?sz=640x480&iu=/124319096/external/ad_rule_samples&ciu_szs=300x250&ad_rule=1&impl=s&gdfp_req=1&env=vp&output=vmap&unviewed_position_start=1&cust_params=deployment%3Ddevsite%26sample_ar%3Dpremidpostlongpod&cmsid=496&vid=short_tencue&correlator="
+      },
+      {
+        "name": "VMAP empty midroll",
+        "uri": "https://storage.googleapis.com/exoplayer-test-media-1/mkv/android-screens-lavf-56.36.100-aac-avc-main-1280x720.mkv",
+        "ad_tag_uri": "http://vastsynthesizer.appspot.com/empty-midroll"
+      },
+      {
+        "name": "VMAP full, empty, full midrolls",
+        "uri": "https://storage.googleapis.com/exoplayer-test-media-1/mkv/android-screens-lavf-56.36.100-aac-avc-main-1280x720.mkv",
+        "ad_tag_uri": "http://vastsynthesizer.appspot.com/empty-midroll-2"
       }
     ]
   }
diff --git a/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaAdsLoader.java b/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaAdsLoader.java
index b632e7ba84..493deed4ad 100644
--- a/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaAdsLoader.java
+++ b/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaAdsLoader.java
@@ -516,6 +516,9 @@ public final class ImaAdsLoader extends Player.DefaultEventListener implements A
       case LOG:
         Map<String, String> adData = adEvent.getAdData();
         Log.i(TAG, "Log AdEvent: " + adData);
+        if ("adLoadError".equals(adData.get("type"))) {
+          handleAdGroupLoadError();
+        }
         break;
       case ALL_ADS_COMPLETED:
       default:
@@ -894,6 +897,23 @@ public final class ImaAdsLoader extends Player.DefaultEventListener implements A
     }
   }
 
+  private void handleAdGroupLoadError() {
+    AdPlaybackState.AdGroup adGroup = adPlaybackState.adGroups[expectedAdGroupIndex];
+    // Ad group load error can be notified more than once, so check if it was already handled.
+    // TODO: Update the expected ad group index based on the position returned by
+    // getContentProgress so that it's possible to detect when more than one ad group fails to load
+    // consecutively.
+    if (adGroup.count == C.LENGTH_UNSET
+        || adGroup.states[0] == AdPlaybackState.AD_STATE_UNAVAILABLE) {
+      if (DEBUG) {
+        Log.d(TAG, "Removing ad group " + expectedAdGroupIndex + " as it failed to load");
+      }
+      adPlaybackState = adPlaybackState.withAdCount(expectedAdGroupIndex, 1);
+      adPlaybackState = adPlaybackState.withAdLoadError(expectedAdGroupIndex, 0);
+      updateAdPlaybackState();
+    }
+  }
+
   private void checkForContentComplete() {
     if (contentDurationMs != C.TIME_UNSET && pendingContentPositionMs == C.TIME_UNSET
         && player.getContentPosition() + END_OF_CONTENT_POSITION_THRESHOLD_MS >= contentDurationMs
@@ -939,6 +959,21 @@ public final class ImaAdsLoader extends Player.DefaultEventListener implements A
     return adGroupTimesUs.length == 0 ? C.INDEX_UNSET : (adGroupTimesUs.length - 1);
   }
 
+  /**
+   * Returns the next ad index in the specified ad group to load, or {@link C#INDEX_UNSET} if all
+   * ads in the ad group have loaded.
+   */
+  private int getAdIndexInAdGroupToLoad(int adGroupIndex) {
+    @AdState int[] states = adPlaybackState.adGroups[adGroupIndex].states;
+    int adIndexInAdGroup = 0;
+    // IMA loads ads in order.
+    while (adIndexInAdGroup < states.length
+        && states[adIndexInAdGroup] != AdPlaybackState.AD_STATE_UNAVAILABLE) {
+      adIndexInAdGroup++;
+    }
+    return adIndexInAdGroup == states.length ? C.INDEX_UNSET : adIndexInAdGroup;
+  }
+
   private static long[] getAdGroupTimesUs(List<Float> cuePoints) {
     if (cuePoints.isEmpty()) {
       // If no cue points are specified, there is a preroll ad.
@@ -955,18 +990,4 @@ public final class ImaAdsLoader extends Player.DefaultEventListener implements A
     return adGroupTimesUs;
   }
 
-  /**
-   * Returns the next ad index in the specified ad group to load, or {@link C#INDEX_UNSET} if all
-   * ads in the ad group have loaded.
-   */
-  private int getAdIndexInAdGroupToLoad(int adGroupIndex) {
-    @AdState int[] states = adPlaybackState.adGroups[adGroupIndex].states;
-    int adIndexInAdGroup = 0;
-    // IMA loads ads in order.
-    while (adIndexInAdGroup < states.length
-        && states[adIndexInAdGroup] != AdPlaybackState.AD_STATE_UNAVAILABLE) {
-      adIndexInAdGroup++;
-    }
-    return adIndexInAdGroup == states.length ? C.INDEX_UNSET : adIndexInAdGroup;
-  }
 }