fix: avoid memory overhead when there is large number of LMQ ConsumeQ…

…ueue (#8956)
apache · Nov 20, 2024 · 8505482 · 8505482
1 parent c13f051
commit 8505482
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 2 deletions.
diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java
@@ -39,7 +39,11 @@ public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInte
     public AbstractConsumeQueueStore(DefaultMessageStore messageStore) {
         this.messageStore = messageStore;
         this.messageStoreConfig = messageStore.getMessageStoreConfig();
-        this.consumeQueueTable = new ConcurrentHashMap<>(32);
+        if (messageStoreConfig.isEnableLmq()) {
+            this.consumeQueueTable = new ConcurrentHashMap<>(32_768);
+        } else {
+            this.consumeQueueTable = new ConcurrentHashMap<>(32);
+        }
     }
 
     @Override

diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java
@@ -480,7 +480,13 @@ public long getMaxPhyOffsetInConsumeQueue() throws RocksDBException {
     public ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId) {
         ConcurrentMap<Integer, ConsumeQueueInterface> map = this.consumeQueueTable.get(topic);
         if (null == map) {
-            ConcurrentMap<Integer, ConsumeQueueInterface> newMap = new ConcurrentHashMap<>(128);
+            ConcurrentMap<Integer, ConsumeQueueInterface> newMap;
+            if (MixAll.isLmq(topic)) {
+                // For LMQ, no need to over allocate internal hashtable
+                newMap = new ConcurrentHashMap<>(1, 1.0F);
+            } else {
+                newMap = new ConcurrentHashMap<>(8);
+            }
             ConcurrentMap<Integer, ConsumeQueueInterface> oldMap = this.consumeQueueTable.putIfAbsent(topic, newMap);
             if (oldMap != null) {
                 map = oldMap;