Vulkan framebuffer and texture handling enhancements

libraries/display-plugins/src/display-plugins/VulkanDisplayPlugin.cpp

@@ -846,10 +846,10 @@ void VulkanDisplayPlugin::present(const std::shared_ptr<RefreshRateController>&
                 vkBackend->_outputTexture->attachments[0].image,
                 VK_ACCESS_TRANSFER_READ_BIT,
                 0,
-                VK_IMAGE_LAYOUT_UNDEFINED,
+                VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
                 VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
-                VK_PIPELINE_STAGE_TRANSFER_BIT,
-                VK_PIPELINE_STAGE_TRANSFER_BIT,
+                VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, // VKTODO
+                VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, // VKTODO
                 mipSubRange);
 
             vks::tools::insertImageMemoryBarrier(
@@ -859,8 +859,8 @@ void VulkanDisplayPlugin::present(const std::shared_ptr<RefreshRateController>&
                 VK_ACCESS_TRANSFER_WRITE_BIT,
                 VK_IMAGE_LAYOUT_UNDEFINED,
                 VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
-                VK_PIPELINE_STAGE_TRANSFER_BIT,
-                VK_PIPELINE_STAGE_TRANSFER_BIT,
+                VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, // VKTODO
+                VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, // VKTODO
                 mipSubRange);
 
             vkCmdBlitImage(
@@ -880,8 +880,19 @@ void VulkanDisplayPlugin::present(const std::shared_ptr<RefreshRateController>&
                 VK_ACCESS_TRANSFER_WRITE_BIT,
                 VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
                 VK_IMAGE_LAYOUT_PRESENT_SRC_KHR,
-                VK_PIPELINE_STAGE_TRANSFER_BIT,
-                VK_PIPELINE_STAGE_TRANSFER_BIT,
+                VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, // VKTODO
+                VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, // VKTODO
+                mipSubRange);
+
+            vks::tools::insertImageMemoryBarrier(
+                commandBuffer,
+                vkBackend->_outputTexture->attachments[0].image,
+                0,
+                VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT,
+                VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
+                VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
+                VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, // VKTODO
+                VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, // VKTODO
                 mipSubRange);
 
             cmdEndLabel(commandBuffer);

libraries/gpu-vk/src/gpu/vk/VKBackend.h

@@ -260,9 +260,12 @@ class VKBackend : public Backend, public std::enable_shared_from_this<VKBackend>
 
     void resetQueryStage();
 
-    VkRenderPass _currentRenderPass{ VK_NULL_HANDLE };
+    VkRenderPass _currentVkRenderPass{ VK_NULL_HANDLE };
+    gpu::FramebufferReference _currentFramebuffer{ nullptr }; // Framebuffer used in currently happening render pass
+    VkFramebuffer _currentVkFramebuffer{ VK_NULL_HANDLE }; // Framebuffer used in currently happening render pass
     // Checks if renderpass change is needed and changes it if required
     void updateRenderPass();
+    void updateAttachmentLayoutsAfterRenderPass();
     void resetRenderPass();
 
     // Contains objects that are created per frame and need to be deleted after the frame is rendered
@@ -334,6 +337,7 @@ class VKBackend : public Backend, public std::enable_shared_from_this<VKBackend>
     VkDescriptorImageInfo getDefaultTextureDescriptorInfo() ;
     // Used by GPU frame player to move camera around
     void enableContextViewCorrectionForFramePlayer() { _transform._viewCorrectionEnabledForFramePlayer = true; };
+    void setIsFramePlayer(bool isFramePlayer) { _isFramePlayer = isFramePlayer; };
 
     static gpu::Primitive getPrimitiveTopologyFromCommand(Batch::Command command, const Batch& batch, size_t paramOffset);
 
@@ -448,7 +452,8 @@ class VKBackend : public Backend, public std::enable_shared_from_this<VKBackend>
     // VKTODO: quick hack
     VKFramebuffer *_outputTexture{ nullptr };
 protected:
-    void transitionImageLayouts();
+    void transitionInputImageLayouts();
+    void transitionAttachmentImageLayouts(gpu::Framebuffer &framebuffer);
 
     // These are filled by syncGPUObject() calls, and are needed to track backend objects so that they can be destroyed before
     // destroying backend.
@@ -487,16 +492,14 @@ class VKBackend : public Backend, public std::enable_shared_from_this<VKBackend>
     // Frame for which command buffer is already generated and it's currently being rendered.
     std::shared_ptr<FrameData> _currentlyRenderedFrame;
 
-    std::vector<VKAttachmentTexture*> _attachmentTexturesToTransitionToRead;
-    std::vector<VKAttachmentTexture*> _attachmentTexturesToTransitionToWrite;
-
     // Safety check to ensure that shutdown was completed before destruction.
     std::atomic<bool> isBackendShutdownComplete{ false };
 
     typedef void (VKBackend::*CommandCall)(const Batch&, size_t);
     static std::array<VKBackend::CommandCall, Batch::NUM_COMMANDS> _commandCalls;
     static const size_t INVALID_OFFSET = (size_t)-1;
     static size_t UNIFORM_BUFFER_OFFSET_ALIGNMENT;
+    bool _isFramePlayer {false};
     bool _isInitialized {false};
 };
 

libraries/gpu-vk/src/gpu/vk/VKFramebuffer.cpp

@@ -317,6 +317,7 @@ uint32_t gpu::vk::VKFramebuffer::addAttachment(VKAttachmentCreateInfo createinfo
     // Color attachment
     if (createinfo.usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT)
     {
+        Q_ASSERT(attachment.format != VK_FORMAT_D24_UNORM_S8_UINT);
         aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
     }
 

libraries/gpu-vk/src/gpu/vk/VKPipelineCache.cpp

@@ -4,6 +4,7 @@
 
 #include "VKShared.h"
 #include <vk/Pipelines.h>
+#include "VKTexture.h"
 
 using namespace gpu;
 using namespace gpu::vk;
@@ -149,17 +150,35 @@ Cache::Pipeline::PipelineLayout Cache::Pipeline::getPipelineAndDescriptorLayout(
 Cache::Pipeline::RenderpassKey Cache::Pipeline::getRenderPassKey(gpu::Framebuffer* framebuffer) const {
     RenderpassKey result;
     if (!framebuffer) {
-        result.push_back(VK_FORMAT_R8G8B8A8_SRGB);
+        result.emplace_back(VK_FORMAT_R8G8B8A8_SRGB, VK_IMAGE_LAYOUT_UNDEFINED);
     } else {
         for (const auto& attachment : framebuffer->getRenderBuffers()) {
             if (attachment.isValid()) {
                 // VKTODO: why _element often has different format than texture's pixel format, and seemingly wrong one?
                 //result.push_back(evalTexelFormatInternal(attachment._element));
-                result.push_back(evalTexelFormatInternal(attachment._texture->getTexelFormat()));
+                VkImageLayout layout = VK_IMAGE_LAYOUT_UNDEFINED;
+                auto *gpuTexture = Backend::getGPUObject<VKTexture>(*attachment._texture);
+                if (gpuTexture) {
+                    auto attachmentTexture = dynamic_cast<VKAttachmentTexture*>(gpuTexture);
+                    if (attachmentTexture) {
+                        layout = attachmentTexture->getVkImageLayout();
+                    }
+                }
+                result.emplace_back(evalTexelFormatInternal(attachment._texture->getTexelFormat()), layout);
             }
         }
         if (framebuffer->hasDepthStencil()) {
-            result.push_back(evalTexelFormatInternal(framebuffer->getDepthStencilBufferFormat()));
+            VkImageLayout layout = VK_IMAGE_LAYOUT_UNDEFINED;
+            if (framebuffer->getDepthStencilBuffer()) {
+                auto* gpuTexture = Backend::getGPUObject<VKTexture>(*framebuffer->getDepthStencilBuffer());
+                if (gpuTexture) {
+                    auto attachmentTexture = dynamic_cast<VKAttachmentTexture*>(gpuTexture);
+                    if (attachmentTexture) {
+                        layout = attachmentTexture->getVkImageLayout();
+                    }
+                }
+            }
+            result.emplace_back(evalTexelFormatInternal(framebuffer->getDepthStencilBufferFormat()), layout);
         }
     }
     return result;
@@ -171,28 +190,57 @@ VkRenderPass Cache::Pipeline::getRenderPass(const vks::Context& context) {
     RenderpassKey key = getRenderPassKey(framebuffer);
     auto itr = _renderPassMap.find(key);
     if (itr == _renderPassMap.end()) {
+        auto &renderBuffers = framebuffer->getRenderBuffers();
+        size_t renderBufferIndex = 0;
         std::vector<VkAttachmentDescription> attachments;
         attachments.reserve(key.size());
         std::vector<VkAttachmentReference> colorAttachmentReferences;
         VkAttachmentReference depthReference{};
-        for (const auto& format : key) {
+        for (const auto& formatAndLayout : key) {
+            Q_ASSERT(renderBufferIndex < renderBuffers.size());
+            std::shared_ptr<gpu::Texture> texture = renderBuffers[renderBufferIndex]._texture;
+            if (isDepthStencilFormat(formatAndLayout.first)) {
+                texture = framebuffer->getDepthStencilBuffer();
+            } else {
+                texture = renderBuffers[renderBufferIndex]._texture;
+                renderBufferIndex++;
+            }
+            VKAttachmentTexture *attachmentTexture = nullptr;
+            if (texture) {
+                auto gpuObject = Backend::getGPUObject<VKTexture>(*texture);
+                if (gpuObject) {
+                    attachmentTexture = dynamic_cast<VKAttachmentTexture*>(gpuObject);
+                }
+            }
             VkAttachmentDescription attachment{};
-            attachment.format = format;
-            //attachment.loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
-            attachment.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
+            attachment.format = formatAndLayout.first;
+            // Framebuffers are always cleared with a separate command in the renderer/
+            if (!attachmentTexture || attachmentTexture->getVkImageLayout() == VK_IMAGE_LAYOUT_UNDEFINED) {
+                attachment.loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
+                attachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
+            } else {
+                attachment.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
+                attachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
+            }
             attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE;
-            //attachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
-            attachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_LOAD;
             attachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_STORE;
             //attachment.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
             attachment.samples = VK_SAMPLE_COUNT_1_BIT;
-            if (isDepthStencilFormat(format)) {
-                attachment.initialLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
+            if (isDepthStencilFormat(formatAndLayout.first)) {
+                if (!attachmentTexture || attachmentTexture->getVkImageLayout() == VK_IMAGE_LAYOUT_UNDEFINED) {
+                    attachment.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
+                } else {
+                    attachment.initialLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
+                }
                 attachment.finalLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
                 depthReference.attachment = (uint32_t)(attachments.size());
                 depthReference.layout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
             } else {
-                attachment.initialLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
+                if (!attachmentTexture || attachmentTexture->getVkImageLayout() == VK_IMAGE_LAYOUT_UNDEFINED) {
+                    attachment.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
+                } else {
+                    attachment.initialLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
+                }
                 attachment.finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
                 VkAttachmentReference reference;
                 reference.attachment = (uint32_t)(attachments.size());
@@ -228,14 +276,15 @@ VkRenderPass Cache::Pipeline::getRenderPass(const vks::Context& context) {
     return itr->second;
 }
 
-std::string Cache::Pipeline::getRenderpassKeyString(const RenderpassKey& renderpassKey) {
+// VKTODO:
+/*std::string Cache::Pipeline::getRenderpassKeyString(const RenderpassKey& renderpassKey) {
     std::string result;
 
     for (const auto& e : renderpassKey) {
         result += hex((uint32_t)e);
     }
     return result;
-}
+}*/
 
 std::string Cache::Pipeline::getStridesKey() const {
     std::string key;

libraries/gpu-vk/src/gpu/vk/VKPipelineCache.h

@@ -19,17 +19,29 @@ struct hash<gpu::Element> {
     size_t operator()(const gpu::Element& a) const { return std::hash<uint16_t>()(a.getRaw()); }
 };
 
+template <>
+struct hash<::VkImageLayout> {
+    size_t operator()(const VkImageLayout& a) const { return std::hash<uint16_t>()((uint16_t)a); }
+};
+
+
+
 }  // namespace std
 namespace gpu { namespace vk {
 
+inline size_t hashRenderPassPair(const std::pair<VkFormat,VkImageLayout>& a) {
+    size_t seed = 0;
+    std::hash_combine(seed, a.first);
+    std::hash_combine(seed, a.second);
+    return seed;
+}
 
-
-template <typename Container>  // we can make this generic for any container [1]
-struct container_hash {
+template <typename Container>
+struct RenderPassHash {
     std::size_t operator()(const Container& c) const {
         size_t seed = 0;
         for (const auto& e : c) {
-            std::hash_combine(seed, e);
+            std::hash_combine(seed, hashRenderPassPair(e));
         }
         return seed;
     }
@@ -39,7 +51,7 @@ struct Cache {
     std::unordered_map<uint32_t, VkShaderModule> moduleMap;
 
     struct Pipeline {
-        using RenderpassKey = std::vector<VkFormat>;
+        using RenderpassKey = std::vector<std::pair<VkFormat, VkImageLayout>>;
         using BindingMap = std::unordered_map<uint32_t, VkShaderStageFlags>;
         using LocationMap = shader::Reflection::LocationMap;
 
@@ -55,8 +67,8 @@ struct Cache {
             VkDescriptorSetLayout storageLayout;
         };
 
-        std::unordered_map<gpu::PipelineReference, PipelineLayout> _layoutMap;
-        std::unordered_map<RenderpassKey, VkRenderPass, container_hash<RenderpassKey>> _renderPassMap;
+        std::unordered_map<gpu::PipelineReference, PipelineLayout> _layoutMap; // VKTODO: make sure pipelines are retrieved from here
+        std::unordered_map<RenderpassKey, VkRenderPass, RenderPassHash<std::vector<std::pair<VkFormat, VkImageLayout>>>> _renderPassMap; // VKTODO: make sure render passes are retrieved from here
 
         // These get set when stride gets set by setInputBuffer
         std::array<Offset, MAX_NUM_INPUT_BUFFERS> _bufferStrides{ 0 };

libraries/gpu-vk/src/gpu/vk/VKTexture.cpp

@@ -205,7 +205,7 @@ void VKAttachmentTexture::createTexture(VKBackend &backend) {
         || _gpuObject.getTexelFormat().getSemantic() == gpu::SRGB
         || _gpuObject.getTexelFormat().getSemantic() == gpu::SRGBA) {*/
     if (_gpuObject.isDepthStencilRenderTarget()) {
-        imageCI.usage = VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT;
+        imageCI.usage = VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT;
     } else {
         imageCI.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT;
     }
@@ -281,7 +281,13 @@ VkDescriptorImageInfo VKAttachmentTexture::getDescriptorImageInfo() {
         viewCreateInfo.pNext = nullptr;
         viewCreateInfo.viewType = getVKTextureType(_gpuObject);
         viewCreateInfo.format = evalTexelFormatInternal(_gpuObject.getTexelFormat());
-        viewCreateInfo.subresourceRange = { VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1 };
+        if (viewCreateInfo.format == VK_FORMAT_D24_UNORM_S8_UINT) {
+            //viewCreateInfo.subresourceRange = { VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT, 0, 1, 0, 1 };
+            // VKTODO: both VK_IMAGE_ASPECT_DEPTH_BIT and VK_IMAGE_ASPECT_STENCIL_BIT cannot be set at the same time, but I'm not sure which one to set.
+            viewCreateInfo.subresourceRange = { VK_IMAGE_ASPECT_DEPTH_BIT, 0, 1, 0, 1 };
+        } else {
+            viewCreateInfo.subresourceRange = { VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1 };
+        }
         viewCreateInfo.subresourceRange.levelCount = 1;
         viewCreateInfo.image = _vkImage;
         VK_CHECK_RESULT(vkCreateImageView(device->logicalDevice, &viewCreateInfo, nullptr, &_vkImageView));

libraries/gpu-vk/src/gpu/vk/VKTexture.h

@@ -267,6 +267,9 @@ class VKFixedAllocationTexture : public VKTexture {
 class VKAttachmentTexture : public VKFixedAllocationTexture {
     friend class VKBackend;
 
+public:
+    VkImageLayout getVkImageLayout() { return _vkImageLayout; };
+
 protected:
     VKAttachmentTexture(const std::weak_ptr<VKBackend>& backend, const Texture& texture) :
         VKFixedAllocationTexture(backend, texture, false) {

tools/gpu-frame-player/src/RenderThread.cpp

@@ -117,6 +117,8 @@ void RenderThread::initialize(QWindow* window) {
     gpu::Context::init<gpu::vk::VKBackend>();
     _gpuContext = std::make_shared<gpu::Context>();
     _backend = _gpuContext->getBackend();
+    auto vkBackend = std::dynamic_pointer_cast<gpu::vk::VKBackend>(_gpuContext->getBackend());
+    vkBackend->setIsFramePlayer(true);
 #endif
 }
 
@@ -297,7 +299,7 @@ void RenderThread::renderFrame(gpu::FramePointer& frame) {
         vkBackend->_outputTexture->attachments[0].image,
         VK_ACCESS_TRANSFER_READ_BIT,
         0,
-        VK_IMAGE_LAYOUT_UNDEFINED,
+        VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
         VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
         VK_PIPELINE_STAGE_TRANSFER_BIT,
         VK_PIPELINE_STAGE_TRANSFER_BIT,