Add explicit model sampling parameters to requests

stackhpc · Dec 13, 2023 · 9daf368 · 9daf368
1 parent ee0c0eb
commit 9daf368
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/web-app-utils/example_app_vanilla.py b/web-app-utils/example_app_vanilla.py
@@ -29,6 +29,12 @@ def inference(message, history):
         "prompt": prompt.format(prompt=context),
         "stream": True,
         "max_tokens": 1000,
+        # Parameters requested by HU
+        "sampling_params": {
+            "temperature": 0.7,
+            "top_p": 0.4,
+            "top_k": 40,
+        }
     }
     response = requests.post(
         f"{backend_url}/generate", headers=headers, json=pload, stream=True