From 9daf368a0f4fa0d21e314d8bfda958b405dd415e Mon Sep 17 00:00:00 2001
From: sd109 <scott@stackhpc.com>
Date: Wed, 13 Dec 2023 14:48:19 +0000
Subject: [PATCH] Add explicit model sampling parameters to requests

---
 web-app-utils/example_app_vanilla.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/web-app-utils/example_app_vanilla.py b/web-app-utils/example_app_vanilla.py
index 356c625..e096d64 100644
--- a/web-app-utils/example_app_vanilla.py
+++ b/web-app-utils/example_app_vanilla.py
@@ -29,6 +29,12 @@ def inference(message, history):
         "prompt": prompt.format(prompt=context),
         "stream": True,
         "max_tokens": 1000,
+        # Parameters requested by HU
+        "sampling_params": {
+            "temperature": 0.7,
+            "top_p": 0.4,
+            "top_k": 40,
+        }
     }
     response = requests.post(
         f"{backend_url}/generate", headers=headers, json=pload, stream=True