diff --git a/agent_gateway/gateway/gateway.py b/agent_gateway/gateway/gateway.py
index 485a02f..ebc704f 100644
--- a/agent_gateway/gateway/gateway.py
+++ b/agent_gateway/gateway/gateway.py
@@ -468,8 +468,8 @@ async def acall(
             inputs["context"] = formatted_contexts
 
         max_memory = 3  # TODO consider exposing this to users
-
-        if len(self.memory_context) <= max_memory:
-            self.memory_context.append({"Question:": input, "Answer": answer})
+        if self.memory:
+            if len(self.memory_context) <= max_memory:
+                self.memory_context.append({"Question:": input, "Answer": answer})
 
         return answer