Don't save resources to be released later when the GPU is already done with them.

onnxruntime/core/providers/dml/DmlExecutionProvider/src/CommandQueue.cpp

@@ -55,7 +55,8 @@ namespace Dml
         // for example, an allocation from BucketizedBufferAllocator attempts to queue a reference
         // to its underlying D3D resource when freed. Furthermore, these references are unnecessary
         // since Close() already blocks for scheduled GPU work before clearing m_queuedReferences.
-        if (!m_closing)
+        // If the CommandQueue is releasing completed references, we don't need to queue the reference up again.
+        if (!m_closing && !m_releasing)
         {
             QueuedReference queuedReference = {GetLastFenceValue(), object};
 
@@ -66,7 +67,13 @@ namespace Dml
                 ++queuedReference.fenceValue;
             }
 
-            m_queuedReferences.push_back(queuedReference);
+            // We don't need to queue references whose work has already completed on the GPU by the time the CPU queues
+            // the reference. Just let it go out of scope.
+            uint64_t completedValue = GetFence()->GetCompletedValue();
+            if (queuedReference.fenceValue > completedValue)
+            {
+                m_queuedReferences.push_back(queuedReference);
+            }
         }
     }
 
@@ -84,10 +91,12 @@ namespace Dml
     void CommandQueue::ReleaseCompletedReferences()
     {
         uint64_t completedValue = GetFence()->GetCompletedValue();
+        m_releasing = true;
         while (!m_queuedReferences.empty() && m_queuedReferences.front().fenceValue <= completedValue)
         {
             m_queuedReferences.pop_front();
         }
+        m_releasing = false;
     }
 
 } // namespace Dml

onnxruntime/core/providers/dml/DmlExecutionProvider/src/CommandQueue.h

@@ -62,6 +62,7 @@ namespace Dml
         ComPtr<ID3D12Fence> m_fence;
         uint64_t m_lastFenceValue = 0;
         bool m_closing = false;
+        bool m_releasing = false;
         bool m_cpuSyncSpinningEnabled = false;
     };
 

onnxruntime/core/providers/dml/DmlExecutionProvider/src/ExecutionContext.cpp

@@ -162,12 +162,19 @@ namespace Dml
     {
         assert(!m_closed);
 
-        if (!m_currentRecorder || !m_currentRecorder->HasUnsubmittedWork())
+        if (!m_currentRecorder)
         {
             // Nothing to flush
             return;
         }
 
+        if (!m_currentRecorder->HasUnsubmittedWork())
+        {
+            // Even if we have no queued work, we can still release resources that may no longer be needed.
+            ReleaseCompletedReferences();
+            return;
+        }
+
         m_currentRecorder->CloseAndExecute();
         ReleaseCompletedReferences();
 
@@ -183,7 +190,7 @@ namespace Dml
         assert(!m_closed);
         // If something has been recorded into a command list but not submitted yet, it means that the *next* fence
         // value is the one to signal completion.
-        bool waitForUnsubmittedWork = (m_currentRecorder != nullptr);
+        bool waitForUnsubmittedWork = ((m_currentRecorder != nullptr) && (m_currentRecorder->HasUnsubmittedWork()));
         m_queue->QueueReference(object, waitForUnsubmittedWork);
     }