diff --git a/src/llmq/instantsend.cpp b/src/llmq/instantsend.cpp
index 663a863f34..037dd2f1f6 100644
--- a/src/llmq/instantsend.cpp
+++ b/src/llmq/instantsend.cpp
@@ -1202,6 +1202,8 @@ void CInstantSendManager::RemoveNonLockedTx(const uint256& txid, bool retryChild
             retryChildrenCount++;
         }
     }
+    // don't try to lock it anymore
+    WITH_LOCK(cs_pendingRetry, pendingRetryTxs.erase(txid));
 
     if (info.tx) {
         for (const auto& in : info.tx->vin) {