diff --git a/common/src/main/java/org/apache/comet/vector/CometDictionary.java b/common/src/main/java/org/apache/comet/vector/CometDictionary.java
index 24a6d6d8c..fd5c2410b 100644
--- a/common/src/main/java/org/apache/comet/vector/CometDictionary.java
+++ b/common/src/main/java/org/apache/comet/vector/CometDictionary.java
@@ -35,7 +35,6 @@ public class CometDictionary implements AutoCloseable {
   public CometDictionary(CometPlainVector values) {
     this.values = values;
     this.numValues = values.numValues();
-    initialize();
   }
 
   public void setDictionaryVector(CometPlainVector values) {
@@ -83,6 +82,19 @@ public byte[] decodeToBinary(int index) {
       case FIXEDSIZEBINARY:
         return values.getBinary(index);
       case DECIMAL:
+        if (binaries == null) {
+          // We only need to copy values for decimal 128 type as random access
+          // to the dictionary is not efficient for decimal (it needs to copy
+          // the value to a new byte array everytime).
+          ByteArrayWrapper[] binaries = new ByteArrayWrapper[numValues];
+          for (int i = 0; i < numValues; i++) {
+            // Need copying here since we re-use byte array for decimal
+            byte[] bytes = new byte[DECIMAL_BYTE_WIDTH];
+            bytes = values.copyBinaryDecimal(i, bytes);
+            binaries[i] = new ByteArrayWrapper(bytes);
+          }
+          this.binaries = binaries;
+        }
         return binaries[index].bytes;
       default:
         throw new IllegalArgumentException(
@@ -99,23 +111,6 @@ public void close() {
     values.close();
   }
 
-  private void initialize() {
-    switch (values.getValueVector().getMinorType()) {
-      case DECIMAL:
-        // We only need to copy values for decimal type as random access
-        // to the dictionary is not efficient for decimal (it needs to copy
-        // the value to a new byte array everytime).
-        binaries = new ByteArrayWrapper[numValues];
-        for (int i = 0; i < numValues; i++) {
-          // Need copying here since we re-use byte array for decimal
-          byte[] bytes = new byte[DECIMAL_BYTE_WIDTH];
-          bytes = values.copyBinaryDecimal(i, bytes);
-          binaries[i] = new ByteArrayWrapper(bytes);
-        }
-        break;
-    }
-  }
-
   private static class ByteArrayWrapper {
     private final byte[] bytes;