diff --git a/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/LibvpxVideoRenderer.java b/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/LibvpxVideoRenderer.java
index e3081cd2d2..e61030a2e1 100644
--- a/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/LibvpxVideoRenderer.java
+++ b/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/LibvpxVideoRenderer.java
@@ -127,8 +127,8 @@ public class LibvpxVideoRenderer extends BaseRenderer {
   private VpxDecoder decoder;
   private VpxInputBuffer inputBuffer;
   private VpxOutputBuffer outputBuffer;
-  private DrmSession<ExoMediaCrypto> drmSession;
-  private DrmSession<ExoMediaCrypto> pendingDrmSession;
+  @Nullable private DrmSession<ExoMediaCrypto> decoderDrmSession;
+  @Nullable private DrmSession<ExoMediaCrypto> sourceDrmSession;
 
   private @ReinitializationState int decoderReinitializationState;
   private boolean decoderReceivedBuffers;
@@ -364,24 +364,10 @@ public class LibvpxVideoRenderer extends BaseRenderer {
     clearReportedVideoSize();
     clearRenderedFirstFrame();
     try {
+      setSourceDrmSession(null);
       releaseDecoder();
     } finally {
-      try {
-        if (drmSession != null) {
-          drmSessionManager.releaseSession(drmSession);
-        }
-      } finally {
-        try {
-          if (pendingDrmSession != null && pendingDrmSession != drmSession) {
-            drmSessionManager.releaseSession(pendingDrmSession);
-          }
-        } finally {
-          drmSession = null;
-          pendingDrmSession = null;
-          decoderCounters.ensureUpdated();
-          eventDispatcher.disabled(decoderCounters);
-        }
-      }
+      eventDispatcher.disabled(decoderCounters);
     }
   }
 
@@ -433,18 +419,35 @@ public class LibvpxVideoRenderer extends BaseRenderer {
   /** Releases the decoder. */
   @CallSuper
   protected void releaseDecoder() {
-    if (decoder == null) {
-      return;
-    }
-
     inputBuffer = null;
     outputBuffer = null;
-    decoder.release();
-    decoder = null;
-    decoderCounters.decoderReleaseCount++;
     decoderReinitializationState = REINITIALIZATION_STATE_NONE;
     decoderReceivedBuffers = false;
     buffersInCodecCount = 0;
+    if (decoder != null) {
+      decoder.release();
+      decoder = null;
+      decoderCounters.decoderReleaseCount++;
+    }
+    setDecoderDrmSession(null);
+  }
+
+  private void setSourceDrmSession(@Nullable DrmSession<ExoMediaCrypto> session) {
+    DrmSession<ExoMediaCrypto> previous = sourceDrmSession;
+    sourceDrmSession = session;
+    releaseDrmSessionIfUnused(previous);
+  }
+
+  private void setDecoderDrmSession(@Nullable DrmSession<ExoMediaCrypto> session) {
+    DrmSession<ExoMediaCrypto> previous = decoderDrmSession;
+    decoderDrmSession = session;
+    releaseDrmSessionIfUnused(previous);
+  }
+
+  private void releaseDrmSessionIfUnused(@Nullable DrmSession<ExoMediaCrypto> session) {
+    if (session != null && session != decoderDrmSession && session != sourceDrmSession) {
+      drmSessionManager.releaseSession(session);
+    }
   }
 
   /**
@@ -467,16 +470,20 @@ public class LibvpxVideoRenderer extends BaseRenderer {
           throw ExoPlaybackException.createForRenderer(
               new IllegalStateException("Media requires a DrmSessionManager"), getIndex());
         }
-        pendingDrmSession = drmSessionManager.acquireSession(Looper.myLooper(), format.drmInitData);
-        if (pendingDrmSession == drmSession) {
-          drmSessionManager.releaseSession(pendingDrmSession);
+        DrmSession<ExoMediaCrypto> session =
+            drmSessionManager.acquireSession(Looper.myLooper(), newFormat.drmInitData);
+        if (session == decoderDrmSession || session == sourceDrmSession) {
+          // We already had this session. The manager must be reference counting, so release it once
+          // to get the count attributed to this renderer back down to 1.
+          drmSessionManager.releaseSession(session);
         }
+        setSourceDrmSession(session);
       } else {
-        pendingDrmSession = null;
+        setSourceDrmSession(null);
       }
     }
 
-    if (pendingDrmSession != drmSession) {
+    if (sourceDrmSession != decoderDrmSession) {
       if (decoderReceivedBuffers) {
         // Signal end of stream and wait for any final output buffers before re-initialization.
         decoderReinitializationState = REINITIALIZATION_STATE_SIGNAL_END_OF_STREAM;
@@ -704,12 +711,13 @@ public class LibvpxVideoRenderer extends BaseRenderer {
       return;
     }
 
-    drmSession = pendingDrmSession;
+    setDecoderDrmSession(sourceDrmSession);
+
     ExoMediaCrypto mediaCrypto = null;
-    if (drmSession != null) {
-      mediaCrypto = drmSession.getMediaCrypto();
+    if (decoderDrmSession != null) {
+      mediaCrypto = decoderDrmSession.getMediaCrypto();
       if (mediaCrypto == null) {
-        DrmSessionException drmError = drmSession.getError();
+        DrmSessionException drmError = decoderDrmSession.getError();
         if (drmError != null) {
           // Continue for now. We may be able to avoid failure if the session recovers, or if a new
           // input format causes the session to be replaced before it's used.
@@ -922,12 +930,12 @@ public class LibvpxVideoRenderer extends BaseRenderer {
   }
 
   private boolean shouldWaitForKeys(boolean bufferEncrypted) throws ExoPlaybackException {
-    if (drmSession == null || (!bufferEncrypted && playClearSamplesWithoutKeys)) {
+    if (decoderDrmSession == null || (!bufferEncrypted && playClearSamplesWithoutKeys)) {
       return false;
     }
-    @DrmSession.State int drmSessionState = drmSession.getState();
+    @DrmSession.State int drmSessionState = decoderDrmSession.getState();
     if (drmSessionState == DrmSession.STATE_ERROR) {
-      throw ExoPlaybackException.createForRenderer(drmSession.getError(), getIndex());
+      throw ExoPlaybackException.createForRenderer(decoderDrmSession.getError(), getIndex());
     }
     return drmSessionState != DrmSession.STATE_OPENED_WITH_KEYS;
   }
diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioRendererEventListener.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioRendererEventListener.java
index eff7bc8de2..48fbea75b4 100644
--- a/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioRendererEventListener.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioRendererEventListener.java
@@ -147,6 +147,7 @@ public interface AudioRendererEventListener {
      * Invokes {@link AudioRendererEventListener#onAudioDisabled(DecoderCounters)}.
      */
     public void disabled(final DecoderCounters counters) {
+      counters.ensureUpdated();
       if (listener != null) {
         handler.post(
             () -> {
diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java
index 49c391c4cc..7fc6c16db8 100644
--- a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java
@@ -548,7 +548,6 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
       try {
         super.onDisabled();
       } finally {
-        decoderCounters.ensureUpdated();
         eventDispatcher.disabled(decoderCounters);
       }
     }
diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/SimpleDecoderAudioRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/SimpleDecoderAudioRenderer.java
index bfd7bbc165..f2e8a23811 100644
--- a/library/core/src/main/java/com/google/android/exoplayer2/audio/SimpleDecoderAudioRenderer.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/SimpleDecoderAudioRenderer.java
@@ -106,8 +106,8 @@ public abstract class SimpleDecoderAudioRenderer extends BaseRenderer implements
         ? extends AudioDecoderException> decoder;
   private DecoderInputBuffer inputBuffer;
   private SimpleOutputBuffer outputBuffer;
-  private DrmSession<ExoMediaCrypto> drmSession;
-  private DrmSession<ExoMediaCrypto> pendingDrmSession;
+  @Nullable private DrmSession<ExoMediaCrypto> decoderDrmSession;
+  @Nullable private DrmSession<ExoMediaCrypto> sourceDrmSession;
 
   @ReinitializationState private int decoderReinitializationState;
   private boolean decoderReceivedBuffers;
@@ -462,12 +462,12 @@ public abstract class SimpleDecoderAudioRenderer extends BaseRenderer implements
   }
 
   private boolean shouldWaitForKeys(boolean bufferEncrypted) throws ExoPlaybackException {
-    if (drmSession == null || (!bufferEncrypted && playClearSamplesWithoutKeys)) {
+    if (decoderDrmSession == null || (!bufferEncrypted && playClearSamplesWithoutKeys)) {
       return false;
     }
-    @DrmSession.State int drmSessionState = drmSession.getState();
+    @DrmSession.State int drmSessionState = decoderDrmSession.getState();
     if (drmSessionState == DrmSession.STATE_ERROR) {
-      throw ExoPlaybackException.createForRenderer(drmSession.getError(), getIndex());
+      throw ExoPlaybackException.createForRenderer(decoderDrmSession.getError(), getIndex());
     }
     return drmSessionState != DrmSession.STATE_OPENED_WITH_KEYS;
   }
@@ -568,25 +568,11 @@ public abstract class SimpleDecoderAudioRenderer extends BaseRenderer implements
     audioTrackNeedsConfigure = true;
     waitingForKeys = false;
     try {
+      setSourceDrmSession(null);
       releaseDecoder();
       audioSink.reset();
     } finally {
-      try {
-        if (drmSession != null) {
-          drmSessionManager.releaseSession(drmSession);
-        }
-      } finally {
-        try {
-          if (pendingDrmSession != null && pendingDrmSession != drmSession) {
-            drmSessionManager.releaseSession(pendingDrmSession);
-          }
-        } finally {
-          drmSession = null;
-          pendingDrmSession = null;
-          decoderCounters.ensureUpdated();
-          eventDispatcher.disabled(decoderCounters);
-        }
-      }
+      eventDispatcher.disabled(decoderCounters);
     }
   }
 
@@ -615,12 +601,13 @@ public abstract class SimpleDecoderAudioRenderer extends BaseRenderer implements
       return;
     }
 
-    drmSession = pendingDrmSession;
+    setDecoderDrmSession(sourceDrmSession);
+
     ExoMediaCrypto mediaCrypto = null;
-    if (drmSession != null) {
-      mediaCrypto = drmSession.getMediaCrypto();
+    if (decoderDrmSession != null) {
+      mediaCrypto = decoderDrmSession.getMediaCrypto();
       if (mediaCrypto == null) {
-        DrmSessionException drmError = drmSession.getError();
+        DrmSessionException drmError = decoderDrmSession.getError();
         if (drmError != null) {
           // Continue for now. We may be able to avoid failure if the session recovers, or if a new
           // input format causes the session to be replaced before it's used.
@@ -646,17 +633,34 @@ public abstract class SimpleDecoderAudioRenderer extends BaseRenderer implements
   }
 
   private void releaseDecoder() {
-    if (decoder == null) {
-      return;
-    }
-
     inputBuffer = null;
     outputBuffer = null;
-    decoder.release();
-    decoder = null;
-    decoderCounters.decoderReleaseCount++;
     decoderReinitializationState = REINITIALIZATION_STATE_NONE;
     decoderReceivedBuffers = false;
+    if (decoder != null) {
+      decoder.release();
+      decoder = null;
+      decoderCounters.decoderReleaseCount++;
+    }
+    setDecoderDrmSession(null);
+  }
+
+  private void setSourceDrmSession(@Nullable DrmSession<ExoMediaCrypto> session) {
+    DrmSession<ExoMediaCrypto> previous = sourceDrmSession;
+    sourceDrmSession = session;
+    releaseDrmSessionIfUnused(previous);
+  }
+
+  private void setDecoderDrmSession(@Nullable DrmSession<ExoMediaCrypto> session) {
+    DrmSession<ExoMediaCrypto> previous = decoderDrmSession;
+    decoderDrmSession = session;
+    releaseDrmSessionIfUnused(previous);
+  }
+
+  private void releaseDrmSessionIfUnused(@Nullable DrmSession<ExoMediaCrypto> session) {
+    if (session != null && session != decoderDrmSession && session != sourceDrmSession) {
+      drmSessionManager.releaseSession(session);
+    }
   }
 
   private void onInputFormatChanged(Format newFormat) throws ExoPlaybackException {
@@ -671,13 +675,16 @@ public abstract class SimpleDecoderAudioRenderer extends BaseRenderer implements
           throw ExoPlaybackException.createForRenderer(
               new IllegalStateException("Media requires a DrmSessionManager"), getIndex());
         }
-        pendingDrmSession = drmSessionManager.acquireSession(Looper.myLooper(),
-            inputFormat.drmInitData);
-        if (pendingDrmSession == drmSession) {
-          drmSessionManager.releaseSession(pendingDrmSession);
+        DrmSession<ExoMediaCrypto> session =
+            drmSessionManager.acquireSession(Looper.myLooper(), newFormat.drmInitData);
+        if (session == decoderDrmSession || session == sourceDrmSession) {
+          // We already had this session. The manager must be reference counting, so release it once
+          // to get the count attributed to this renderer back down to 1.
+          drmSessionManager.releaseSession(session);
         }
+        setSourceDrmSession(session);
       } else {
-        pendingDrmSession = null;
+        setSourceDrmSession(null);
       }
     }
 
diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java
index b578467933..a538bca4b2 100644
--- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java
@@ -287,13 +287,13 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   private final DecoderInputBuffer flagsOnlyBuffer;
   private final FormatHolder formatHolder;
   private final TimedValueQueue<Format> formatQueue;
-  private final List<Long> decodeOnlyPresentationTimestamps;
+  private final ArrayList<Long> decodeOnlyPresentationTimestamps;
   private final MediaCodec.BufferInfo outputBufferInfo;
 
   @Nullable private Format inputFormat;
   private Format outputFormat;
-  private DrmSession<FrameworkMediaCrypto> drmSession;
-  private DrmSession<FrameworkMediaCrypto> pendingDrmSession;
+  @Nullable private DrmSession<FrameworkMediaCrypto> codecDrmSession;
+  @Nullable private DrmSession<FrameworkMediaCrypto> sourceDrmSession;
   private long renderTimeLimitMs;
   private float rendererOperatingRate;
   @Nullable private MediaCodec codec;
@@ -457,14 +457,15 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
       return;
     }
 
-    drmSession = pendingDrmSession;
+    setCodecDrmSession(sourceDrmSession);
+
     String mimeType = inputFormat.sampleMimeType;
     MediaCrypto wrappedMediaCrypto = null;
     boolean drmSessionRequiresSecureDecoder = false;
-    if (drmSession != null) {
-      FrameworkMediaCrypto mediaCrypto = drmSession.getMediaCrypto();
+    if (codecDrmSession != null) {
+      FrameworkMediaCrypto mediaCrypto = codecDrmSession.getMediaCrypto();
       if (mediaCrypto == null) {
-        DrmSessionException drmError = drmSession.getError();
+        DrmSessionException drmError = codecDrmSession.getError();
         if (drmError != null) {
           // Continue for now. We may be able to avoid failure if the session recovers, or if a new
           // input format causes the session to be replaced before it's used.
@@ -477,9 +478,9 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
         drmSessionRequiresSecureDecoder = mediaCrypto.requiresSecureDecoderComponent(mimeType);
       }
       if (deviceNeedsDrmKeysToConfigureCodecWorkaround()) {
-        @DrmSession.State int drmSessionState = drmSession.getState();
+        @DrmSession.State int drmSessionState = codecDrmSession.getState();
         if (drmSessionState == DrmSession.STATE_ERROR) {
-          throw ExoPlaybackException.createForRenderer(drmSession.getError(), getIndex());
+          throw ExoPlaybackException.createForRenderer(codecDrmSession.getError(), getIndex());
         } else if (drmSessionState != DrmSession.STATE_OPENED_WITH_KEYS) {
           // Wait for keys.
           return;
@@ -552,7 +553,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   @Override
   protected void onDisabled() {
     inputFormat = null;
-    if (drmSession != null || pendingDrmSession != null) {
+    if (codecDrmSession != null || sourceDrmSession != null) {
       // TODO: Do something better with this case.
       onReset();
     } else {
@@ -565,51 +566,32 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     try {
       releaseCodec();
     } finally {
-      try {
-        if (drmSession != null) {
-          drmSessionManager.releaseSession(drmSession);
-        }
-      } finally {
-        try {
-          if (pendingDrmSession != null && pendingDrmSession != drmSession) {
-            drmSessionManager.releaseSession(pendingDrmSession);
-          }
-        } finally {
-          drmSession = null;
-          pendingDrmSession = null;
-        }
-      }
+      setSourceDrmSession(null);
     }
   }
 
   protected void releaseCodec() {
     availableCodecInfos = null;
-    if (codec != null) {
-      codecInfo = null;
-      codecFormat = null;
-      resetInputBuffer();
-      resetOutputBuffer();
-      resetCodecBuffers();
-      waitingForKeys = false;
-      codecHotswapDeadlineMs = C.TIME_UNSET;
-      decodeOnlyPresentationTimestamps.clear();
-      decoderCounters.decoderReleaseCount++;
-      try {
-        codec.stop();
-      } finally {
+    codecInfo = null;
+    codecFormat = null;
+    resetInputBuffer();
+    resetOutputBuffer();
+    resetCodecBuffers();
+    waitingForKeys = false;
+    codecHotswapDeadlineMs = C.TIME_UNSET;
+    decodeOnlyPresentationTimestamps.clear();
+    try {
+      if (codec != null) {
+        decoderCounters.decoderReleaseCount++;
         try {
-          codec.release();
+          codec.stop();
         } finally {
-          codec = null;
-          if (drmSession != null && pendingDrmSession != drmSession) {
-            try {
-              drmSessionManager.releaseSession(drmSession);
-            } finally {
-              drmSession = null;
-            }
-          }
+          codec.release();
         }
       }
+    } finally {
+      codec = null;
+      setCodecDrmSession(null);
     }
   }
 
@@ -928,6 +910,24 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     outputBuffer = null;
   }
 
+  private void setSourceDrmSession(@Nullable DrmSession<FrameworkMediaCrypto> session) {
+    DrmSession<FrameworkMediaCrypto> previous = sourceDrmSession;
+    sourceDrmSession = session;
+    releaseDrmSessionIfUnused(previous);
+  }
+
+  private void setCodecDrmSession(@Nullable DrmSession<FrameworkMediaCrypto> session) {
+    DrmSession<FrameworkMediaCrypto> previous = codecDrmSession;
+    codecDrmSession = session;
+    releaseDrmSessionIfUnused(previous);
+  }
+
+  private void releaseDrmSessionIfUnused(@Nullable DrmSession<FrameworkMediaCrypto> session) {
+    if (session != null && session != codecDrmSession && session != sourceDrmSession) {
+      drmSessionManager.releaseSession(session);
+    }
+  }
+
   /**
    * @return Whether it may be possible to feed more input data.
    * @throws ExoPlaybackException If an error occurs feeding the input buffer.
@@ -1082,12 +1082,12 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   }
 
   private boolean shouldWaitForKeys(boolean bufferEncrypted) throws ExoPlaybackException {
-    if (drmSession == null || (!bufferEncrypted && playClearSamplesWithoutKeys)) {
+    if (codecDrmSession == null || (!bufferEncrypted && playClearSamplesWithoutKeys)) {
       return false;
     }
-    @DrmSession.State int drmSessionState = drmSession.getState();
+    @DrmSession.State int drmSessionState = codecDrmSession.getState();
     if (drmSessionState == DrmSession.STATE_ERROR) {
-      throw ExoPlaybackException.createForRenderer(drmSession.getError(), getIndex());
+      throw ExoPlaybackException.createForRenderer(codecDrmSession.getError(), getIndex());
     }
     return drmSessionState != DrmSession.STATE_OPENED_WITH_KEYS;
   }
@@ -1126,13 +1126,16 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
           throw ExoPlaybackException.createForRenderer(
               new IllegalStateException("Media requires a DrmSessionManager"), getIndex());
         }
-        pendingDrmSession =
+        DrmSession<FrameworkMediaCrypto> session =
             drmSessionManager.acquireSession(Looper.myLooper(), newFormat.drmInitData);
-        if (pendingDrmSession == drmSession) {
-          drmSessionManager.releaseSession(pendingDrmSession);
+        if (session == codecDrmSession || session == sourceDrmSession) {
+          // We already had this session. The manager must be reference counting, so release it once
+          // to get the count attributed to this renderer back down to 1.
+          drmSessionManager.releaseSession(session);
         }
+        setSourceDrmSession(session);
       } else {
-        pendingDrmSession = null;
+        setSourceDrmSession(null);
       }
     }
 
@@ -1143,7 +1146,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
 
     // We have an existing codec that we may need to reconfigure or re-initialize. If the existing
     // codec instance is being kept then its operating rate may need to be updated.
-    if (pendingDrmSession != drmSession) {
+    if (sourceDrmSession != codecDrmSession) {
       drainAndReinitializeCodec();
     } else {
       switch (canKeepCodec(codec, codecInfo, codecFormat, newFormat)) {
diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java
index 7c4287710d..b92dd44eb2 100644
--- a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java
@@ -375,7 +375,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     try {
       super.onDisabled();
     } finally {
-      decoderCounters.ensureUpdated();
       eventDispatcher.disabled(decoderCounters);
     }
   }
diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/VideoRendererEventListener.java b/library/core/src/main/java/com/google/android/exoplayer2/video/VideoRendererEventListener.java
index 7d78ba03c7..f2f451b3d0 100644
--- a/library/core/src/main/java/com/google/android/exoplayer2/video/VideoRendererEventListener.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/video/VideoRendererEventListener.java
@@ -179,6 +179,7 @@ public interface VideoRendererEventListener {
 
     /** Invokes {@link VideoRendererEventListener#onVideoDisabled(DecoderCounters)}. */
     public void disabled(DecoderCounters counters) {
+      counters.ensureUpdated();
       if (listener != null) {
         handler.post(
             () -> {