diff --git a/library/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java b/library/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java
index e621e19a48..4b673d3750 100644
--- a/library/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java
+++ b/library/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java
@@ -108,6 +108,7 @@ public final class SimpleExoPlayer implements ExoPlayer {
   private Format audioFormat;
 
   private Surface surface;
+  private boolean ownsSurface;
   private SurfaceHolder surfaceHolder;
   private TextureView textureView;
   private TextRenderer.Output textOutput;
@@ -206,7 +207,7 @@ public final class SimpleExoPlayer implements ExoPlayer {
    */
   public void setVideoSurface(Surface surface) {
     removeSurfaceCallbacks();
-    setVideoSurfaceInternal(surface);
+    setVideoSurfaceInternal(surface, false);
   }
 
   /**
@@ -219,9 +220,9 @@ public final class SimpleExoPlayer implements ExoPlayer {
     removeSurfaceCallbacks();
     this.surfaceHolder = surfaceHolder;
     if (surfaceHolder == null) {
-      setVideoSurfaceInternal(null);
+      setVideoSurfaceInternal(null, false);
     } else {
-      setVideoSurfaceInternal(surfaceHolder.getSurface());
+      setVideoSurfaceInternal(surfaceHolder.getSurface(), false);
       surfaceHolder.addCallback(componentListener);
     }
   }
@@ -246,13 +247,13 @@ public final class SimpleExoPlayer implements ExoPlayer {
     removeSurfaceCallbacks();
     this.textureView = textureView;
     if (textureView == null) {
-      setVideoSurfaceInternal(null);
+      setVideoSurfaceInternal(null, true);
     } else {
       if (textureView.getSurfaceTextureListener() != null) {
         Log.w(TAG, "Replacing existing SurfaceTextureListener.");
       }
       SurfaceTexture surfaceTexture = textureView.getSurfaceTexture();
-      setVideoSurfaceInternal(surfaceTexture == null ? null : new Surface(surfaceTexture));
+      setVideoSurfaceInternal(surfaceTexture == null ? null : new Surface(surfaceTexture), true);
       textureView.setSurfaceTextureListener(componentListener);
     }
   }
@@ -468,6 +469,12 @@ public final class SimpleExoPlayer implements ExoPlayer {
   public void release() {
     player.release();
     removeSurfaceCallbacks();
+    if (surface != null) {
+      if (ownsSurface) {
+        surface.release();
+      }
+      surface = null;
+    }
   }
 
   @Override
@@ -618,7 +625,7 @@ public final class SimpleExoPlayer implements ExoPlayer {
     }
   }
 
-  private void setVideoSurfaceInternal(Surface surface) {
+  private void setVideoSurfaceInternal(Surface surface, boolean ownsSurface) {
     // Note: We don't turn this method into a no-op if the surface is being replaced with itself
     // so as to ensure onRenderedFirstFrame callbacks are still called in this case.
     ExoPlayerMessage[] messages = new ExoPlayerMessage[videoRendererCount];
@@ -629,12 +636,17 @@ public final class SimpleExoPlayer implements ExoPlayer {
       }
     }
     if (this.surface != null && this.surface != surface) {
+      // If we created this surface, we are responsible for releasing it.
+      if (this.ownsSurface) {
+        this.surface.release();
+      }
       // We're replacing a surface. Block to ensure that it's not accessed after the method returns.
       player.blockingSendMessages(messages);
     } else {
       player.sendMessages(messages);
     }
     this.surface = surface;
+    this.ownsSurface = ownsSurface;
   }
 
   private final class ComponentListener implements VideoRendererEventListener,
@@ -783,7 +795,7 @@ public final class SimpleExoPlayer implements ExoPlayer {
 
     @Override
     public void surfaceCreated(SurfaceHolder holder) {
-      setVideoSurfaceInternal(holder.getSurface());
+      setVideoSurfaceInternal(holder.getSurface(), false);
     }
 
     @Override
@@ -793,14 +805,14 @@ public final class SimpleExoPlayer implements ExoPlayer {
 
     @Override
     public void surfaceDestroyed(SurfaceHolder holder) {
-      setVideoSurfaceInternal(null);
+      setVideoSurfaceInternal(null, false);
     }
 
     // TextureView.SurfaceTextureListener implementation
 
     @Override
     public void onSurfaceTextureAvailable(SurfaceTexture surfaceTexture, int width, int height) {
-      setVideoSurfaceInternal(new Surface(surfaceTexture));
+      setVideoSurfaceInternal(new Surface(surfaceTexture), true);
     }
 
     @Override
@@ -810,7 +822,7 @@ public final class SimpleExoPlayer implements ExoPlayer {
 
     @Override
     public boolean onSurfaceTextureDestroyed(SurfaceTexture surfaceTexture) {
-      setVideoSurface(null);
+      setVideoSurfaceInternal(null, true);
       return true;
     }