Bug 1291163 - Make sure TextureHosts are read-unlock'ed if Compositor…

…::EndFrame is not called. r=sotaro

gfx/layers/PersistentBufferProvider.cpp

@@ -272,6 +272,15 @@ PersistentBufferProviderShared::BorrowDrawTarget(const gfx::IntRect& aPersistedR
     if (mTextures.length() >= 4) {
       // We should never need to buffer that many textures, something's wrong.
       MOZ_ASSERT(false);
+      // In theory we throttle the main thread when the compositor can't keep up,
+      // so we shoud never get in a situation where we sent 4 textures to the
+      // compositor and the latter as not released any of them.
+      // This seems to happen, however, in some edge cases such as just after a
+      // device reset (cf. Bug 1291163).
+      // It would be pretty bad to keep piling textures up at this point so we
+      // call NotifyInactive to remove some of our textures.
+      NotifyInactive();
+      // Give up now. The caller can fall-back to a non-shared buffer provider.
       return nullptr;
     }
 

gfx/layers/d3d11/CompositorD3D11.cpp

@@ -1148,6 +1148,9 @@ CompositorD3D11::BeginFrame(const nsIntRegion& aInvalidRegion,
   // cause a crash when we're restored.
   NS_ASSERTION(mHwnd, "Couldn't find an HWND when initialising?");
   if (::IsIconic(mHwnd) || mDevice->GetDeviceRemovedReason() != S_OK) {
+    // We are not going to render, and not going to call EndFrame so we have to
+    // read-unlock our textures to prevent them from accumulating.
+    ReadUnlockTextures();
     *aRenderBoundsOut = IntRect();
     return;
   }
@@ -1157,6 +1160,7 @@ CompositorD3D11::BeginFrame(const nsIntRegion& aInvalidRegion,
   // Failed to create a render target or the view.
   if (!UpdateRenderTarget() || !mDefaultRT || !mDefaultRT->mRTView ||
       mSize.width <= 0 || mSize.height <= 0) {
+    ReadUnlockTextures();
     *aRenderBoundsOut = IntRect();
     return;
   }