From 4346c8b1fb52364c23f583c7e5fd3f7017a47af8 Mon Sep 17 00:00:00 2001 From: Fei Yang Date: Tue, 9 Apr 2019 14:50:23 +0800 Subject: [PATCH] Vulkan: pipeline cache not populated as blob cache is not set 1. Use vkMergePipelineCaches to recreate pipeline cache after blob cache callbacks are set after eglInititalize. 2. Use a more proper way to save the cache data to disk. Bug: angleproject:3318 Change-Id: Ied1fb572813198b51e02ed9629cbf34e2d9159b2 Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/1683807 Commit-Queue: Shahbaz Youssefi Reviewed-by: Shahbaz Youssefi Reviewed-by: Jamie Madill --- src/libANGLE/renderer/vulkan/ProgramVk.h | 4 +- src/libANGLE/renderer/vulkan/RendererVk.cpp | 56 +++++++++++++++---- src/libANGLE/renderer/vulkan/RendererVk.h | 11 +++- src/libANGLE/renderer/vulkan/UtilsVk.cpp | 11 ++-- .../renderer/vulkan/vk_cache_utils.cpp | 1 + src/libANGLE/renderer/vulkan/vk_helpers.cpp | 4 +- src/libANGLE/renderer/vulkan/vk_wrapper.h | 13 +++++ 7 files changed, 80 insertions(+), 20 deletions(-) diff --git a/src/libANGLE/renderer/vulkan/ProgramVk.h b/src/libANGLE/renderer/vulkan/ProgramVk.h index 90ef46f23..48e9e26f4 100644 --- a/src/libANGLE/renderer/vulkan/ProgramVk.h +++ b/src/libANGLE/renderer/vulkan/ProgramVk.h @@ -141,8 +141,10 @@ class ProgramVk : public ProgramImpl ANGLE_TRY(initGraphicsShaders(contextVk, mode, &shaderProgram)); ASSERT(shaderProgram->isGraphicsProgram()); RendererVk *renderer = contextVk->getRenderer(); + vk::PipelineCache *pipelineCache = nullptr; + ANGLE_TRY(renderer->getPipelineCache(&pipelineCache)); return shaderProgram->getGraphicsPipeline( - contextVk, &contextVk->getRenderPassCache(), renderer->getPipelineCache(), + contextVk, &contextVk->getRenderPassCache(), *pipelineCache, contextVk->getCurrentQueueSerial(), mPipelineLayout.get(), desc, activeAttribLocations, mState.getAttributesTypeMask(), descPtrOut, pipelineOut); } diff --git a/src/libANGLE/renderer/vulkan/RendererVk.cpp b/src/libANGLE/renderer/vulkan/RendererVk.cpp index 4f274043d..f09b29102 100644 --- a/src/libANGLE/renderer/vulkan/RendererVk.cpp +++ b/src/libANGLE/renderer/vulkan/RendererVk.cpp @@ -47,8 +47,8 @@ namespace rx namespace { -// Update the pipeline cache every this many swaps (if 60fps, this means every 10 minutes) -constexpr uint32_t kPipelineCacheVkUpdatePeriod = 10 * 60 * 60; +// Update the pipeline cache every this many swaps. +constexpr uint32_t kPipelineCacheVkUpdatePeriod = 60; // Wait a maximum of 10s. If that times out, we declare it a failure. constexpr uint64_t kMaxFenceWaitTimeNs = 10'000'000'000llu; // Per the Vulkan specification, as long as Vulkan 1.1+ is returned by vkEnumerateInstanceVersion, @@ -496,7 +496,9 @@ RendererVk::RendererVk() mMaxVertexAttribDivisor(1), mDevice(VK_NULL_HANDLE), mDeviceLost(false), - mPipelineCacheVkUpdateTimeout(kPipelineCacheVkUpdatePeriod) + mPipelineCacheVkUpdateTimeout(kPipelineCacheVkUpdatePeriod), + mPipelineCacheDirty(false), + mPipelineCacheInitialized(false) { VkFormatProperties invalid = {0, 0, kInvalidFormatFeatureFlags}; mFormatProperties.fill(invalid); @@ -1026,7 +1028,8 @@ angle::Result RendererVk::initializeDevice(DisplayVk *displayVk, uint32_t queueF vkGetDeviceQueue(mDevice, mCurrentQueueFamilyIndex, 0, &mQueue); // Initialize the vulkan pipeline cache. - ANGLE_TRY(initPipelineCache(displayVk)); + bool success = false; + ANGLE_TRY(initPipelineCache(displayVk, &mPipelineCache, &success)); return angle::Result::Continue; } @@ -1315,22 +1318,49 @@ void RendererVk::initPipelineCacheVkKey() hashString.length(), mPipelineCacheVkBlobKey.data()); } -angle::Result RendererVk::initPipelineCache(DisplayVk *display) +angle::Result RendererVk::initPipelineCache(DisplayVk *display, + vk::PipelineCache *pipelineCache, + bool *success) { initPipelineCacheVkKey(); egl::BlobCache::Value initialData; - bool success = display->getBlobCache()->get(display->getScratchBuffer(), - mPipelineCacheVkBlobKey, &initialData); + *success = display->getBlobCache()->get(display->getScratchBuffer(), mPipelineCacheVkBlobKey, + &initialData); VkPipelineCacheCreateInfo pipelineCacheCreateInfo = {}; pipelineCacheCreateInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO; pipelineCacheCreateInfo.flags = 0; - pipelineCacheCreateInfo.initialDataSize = success ? initialData.size() : 0; - pipelineCacheCreateInfo.pInitialData = success ? initialData.data() : nullptr; + pipelineCacheCreateInfo.initialDataSize = *success ? initialData.size() : 0; + pipelineCacheCreateInfo.pInitialData = *success ? initialData.data() : nullptr; - ANGLE_VK_TRY(display, mPipelineCache.init(mDevice, pipelineCacheCreateInfo)); + ANGLE_VK_TRY(display, pipelineCache->init(mDevice, pipelineCacheCreateInfo)); + + return angle::Result::Continue; +} + +angle::Result RendererVk::getPipelineCache(vk::PipelineCache **pipelineCache) +{ + if (mPipelineCacheInitialized) + { + *pipelineCache = &mPipelineCache; + return angle::Result::Continue; + } + + // We should now recreate the pipeline cache with the blob cache pipeline data. + vk::PipelineCache pCache; + bool success = false; + ANGLE_TRY(initPipelineCache(vk::GetImpl(mDisplay), &pCache, &success)); + if (success) + { + // Merge the newly created pipeline cache into the existing one. + mPipelineCache.merge(mDevice, mPipelineCache.getHandle(), 1, pCache.ptr()); + } + mPipelineCacheInitialized = true; + pCache.destroy(mDevice); + + *pipelineCache = &mPipelineCache; return angle::Result::Continue; } @@ -1387,6 +1417,11 @@ angle::Result RendererVk::syncPipelineCacheVk(DisplayVk *displayVk) { return angle::Result::Continue; } + if (!mPipelineCacheDirty) + { + mPipelineCacheVkUpdateTimeout = kPipelineCacheVkUpdatePeriod; + return angle::Result::Continue; + } mPipelineCacheVkUpdateTimeout = kPipelineCacheVkUpdatePeriod; @@ -1420,6 +1455,7 @@ angle::Result RendererVk::syncPipelineCacheVk(DisplayVk *displayVk) } displayVk->getBlobCache()->putApplication(mPipelineCacheVkBlobKey, *pipelineCacheData); + mPipelineCacheDirty = false; return angle::Result::Continue; } diff --git a/src/libANGLE/renderer/vulkan/RendererVk.h b/src/libANGLE/renderer/vulkan/RendererVk.h index d5e3c575f..813c62874 100644 --- a/src/libANGLE/renderer/vulkan/RendererVk.h +++ b/src/libANGLE/renderer/vulkan/RendererVk.h @@ -128,8 +128,6 @@ class RendererVk : angle::NonCopyable bool isMockICDEnabled() const { return mEnableMockICD; } - const vk::PipelineCache &getPipelineCache() const { return mPipelineCache; } - // Query the format properties for select bits (linearTilingFeatures, optimalTilingFeatures and // bufferFeatures). Looks through mandatory features first, and falls back to querying the // device (first time only). @@ -160,13 +158,18 @@ class RendererVk : angle::NonCopyable static constexpr size_t kMaxExtensionNames = 200; using ExtensionNameList = angle::FixedVector; + angle::Result getPipelineCache(vk::PipelineCache **pipelineCache); + void onNewGraphicsPipeline() { mPipelineCacheDirty = true; } + private: angle::Result initializeDevice(DisplayVk *displayVk, uint32_t queueFamilyIndex); void ensureCapsInitialized() const; void initFeatures(const ExtensionNameList &extensions); void initPipelineCacheVkKey(); - angle::Result initPipelineCache(DisplayVk *display); + angle::Result initPipelineCache(DisplayVk *display, + vk::PipelineCache *pipelineCache, + bool *success); template VkFormatFeatureFlags getFormatFeatureBits(VkFormat format, @@ -223,6 +226,8 @@ class RendererVk : angle::NonCopyable vk::PipelineCache mPipelineCache; egl::BlobCache::Key mPipelineCacheVkBlobKey; uint32_t mPipelineCacheVkUpdateTimeout; + bool mPipelineCacheDirty; + bool mPipelineCacheInitialized; // A cache of VkFormatProperties as queried from the device over time. std::array mFormatProperties; diff --git a/src/libANGLE/renderer/vulkan/UtilsVk.cpp b/src/libANGLE/renderer/vulkan/UtilsVk.cpp index 7f96e2b23..d95b7564f 100644 --- a/src/libANGLE/renderer/vulkan/UtilsVk.cpp +++ b/src/libANGLE/renderer/vulkan/UtilsVk.cpp @@ -587,11 +587,12 @@ angle::Result UtilsVk::setupProgram(ContextVk *contextVk, // This value is not used but is passed to getGraphicsPipeline to avoid a nullptr check. const vk::GraphicsPipelineDesc *descPtr; vk::PipelineHelper *helper; - - ANGLE_TRY(program->getGraphicsPipeline( - contextVk, &contextVk->getRenderPassCache(), renderer->getPipelineCache(), serial, - pipelineLayout.get(), *pipelineDesc, gl::AttributesMask(), gl::ComponentTypeMask(), - &descPtr, &helper)); + vk::PipelineCache *pipelineCache = nullptr; + ANGLE_TRY(renderer->getPipelineCache(&pipelineCache)); + ANGLE_TRY(program->getGraphicsPipeline(contextVk, &contextVk->getRenderPassCache(), + *pipelineCache, serial, pipelineLayout.get(), + *pipelineDesc, gl::AttributesMask(), + gl::ComponentTypeMask(), &descPtr, &helper)); helper->updateSerial(serial); commandBuffer->bindGraphicsPipeline(helper->getPipeline()); } diff --git a/src/libANGLE/renderer/vulkan/vk_cache_utils.cpp b/src/libANGLE/renderer/vulkan/vk_cache_utils.cpp index afac23d83..b0ab8b452 100644 --- a/src/libANGLE/renderer/vulkan/vk_cache_utils.cpp +++ b/src/libANGLE/renderer/vulkan/vk_cache_utils.cpp @@ -1705,6 +1705,7 @@ angle::Result GraphicsPipelineCache::insertPipeline( // This "if" is left here for the benefit of VulkanPipelineCachePerfTest. if (context != nullptr) { + context->getRenderer()->onNewGraphicsPipeline(); ANGLE_TRY(desc.initializePipeline(context, pipelineCacheVk, compatibleRenderPass, pipelineLayout, activeAttribLocationsMask, programAttribsTypeMask, vertexModule, fragmentModule, diff --git a/src/libANGLE/renderer/vulkan/vk_helpers.cpp b/src/libANGLE/renderer/vulkan/vk_helpers.cpp index e8d280318..1f8eb7c29 100644 --- a/src/libANGLE/renderer/vulkan/vk_helpers.cpp +++ b/src/libANGLE/renderer/vulkan/vk_helpers.cpp @@ -2752,8 +2752,10 @@ angle::Result ShaderProgramHelper::getComputePipeline(Context *context, createInfo.basePipelineHandle = VK_NULL_HANDLE; createInfo.basePipelineIndex = 0; + vk::PipelineCache *pipelineCache = nullptr; + ANGLE_TRY(renderer->getPipelineCache(&pipelineCache)); ANGLE_VK_TRY(context, mComputePipeline.get().initCompute(context->getDevice(), createInfo, - renderer->getPipelineCache())); + *pipelineCache)); *pipelineOut = &mComputePipeline; return angle::Result::Continue; diff --git a/src/libANGLE/renderer/vulkan/vk_wrapper.h b/src/libANGLE/renderer/vulkan/vk_wrapper.h index bfaf894d8..8ffa1b64b 100644 --- a/src/libANGLE/renderer/vulkan/vk_wrapper.h +++ b/src/libANGLE/renderer/vulkan/vk_wrapper.h @@ -478,6 +478,10 @@ class PipelineCache final : public WrappedObject VkResult init(VkDevice device, const VkPipelineCacheCreateInfo &createInfo); VkResult getCacheData(VkDevice device, size_t *cacheSize, void *cacheData); + VkResult merge(VkDevice device, + VkPipelineCache dstCache, + uint32_t srcCacheCount, + const VkPipelineCache *srcCaches); }; class DescriptorSetLayout final : public WrappedObject @@ -1237,6 +1241,15 @@ ANGLE_INLINE VkResult PipelineCache::init(VkDevice device, return vkCreatePipelineCache(device, &createInfo, nullptr, &mHandle); } +ANGLE_INLINE VkResult PipelineCache::merge(VkDevice device, + VkPipelineCache dstCache, + uint32_t srcCacheCount, + const VkPipelineCache *srcCaches) +{ + ASSERT(valid()); + return vkMergePipelineCaches(device, dstCache, srcCacheCount, srcCaches); +} + ANGLE_INLINE VkResult PipelineCache::getCacheData(VkDevice device, size_t *cacheSize, void *cacheData)