Flags for varying NIM seed and temperature every call (#808)

* add support for configured random seed (as per API spec) * add support for varying temp, seed per call - enabled by default * use instance-based seed, set from config in constructor; rely on None safeguards * prune unused import * let config do its work instead of accessing config_root directly * prefer `hasattr` to `dir` Signed-off-by: Jeffrey Martin <[email protected]> * default `seed` to `None` Signed-off-by: Jeffrey Martin <[email protected]> --------- Signed-off-by: Jeffrey Martin <[email protected]> Co-authored-by: Jeffrey Martin <[email protected]>
NVIDIA · Jul 30, 2024 · b6792a4 · b6792a4
1 parent 9df0d8e
commit b6792a4
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 3 deletions.
diff --git a/garak/generators/nim.py b/garak/generators/nim.py
@@ -3,6 +3,7 @@
 
 """NVIDIA Inference Microservice LLM interface"""
 
+import random
 from typing import List, Union
 
 import openai
@@ -38,6 +39,8 @@ class NVOpenAIChat(OpenAICompatible):
         "top_p": 0.7,
         "top_k": 0,  # top_k is hard set to zero as of 24.04.30
         "uri": "https://integrate.api.nvidia.com/v1/",
+        "vary_seed_each_call": True,  # encourage variation when generations>1. not respected by all NIMs
+        "vary_temp_each_call": True,  # encourage variation when generations>1. not respected by all NIMs
         "suppressed_params": {"n", "frequency_penalty", "presence_penalty"},
     }
     active = True
@@ -67,6 +70,13 @@ def _call_model(
         assert (
             generations_this_call == 1
         ), "generations_per_call / n > 1 is not supported"
+
+        if self.vary_seed_each_call:
+            self.seed = random.randint(0, 65535)
+
+        if self.vary_temp_each_call:
+            self.temperature = random.random()
+
         return super()._call_model(prompt, generations_this_call)
 
 

diff --git a/garak/generators/openai.py b/garak/generators/openai.py
@@ -25,9 +25,9 @@
 # lists derived from https://platform.openai.com/docs/models
 chat_models = (
     "gpt-4",  # links to latest version
-    "gpt-4-turbo", # links to latest version
-    "gpt-4o", # links to latest version
-    "gpt-4o-mini", # links to latest version
+    "gpt-4-turbo",  # links to latest version
+    "gpt-4o",  # links to latest version
+    "gpt-4o-mini",  # links to latest version
     "gpt-4-turbo-preview",
     "gpt-3.5-turbo",  # links to latest version
     "gpt-4-32k",
@@ -102,6 +102,7 @@ class OpenAICompatible(Generator):
         "top_p": 1.0,
         "frequency_penalty": 0.0,
         "presence_penalty": 0.0,
+        "seed": None,
         "stop": ["#", ";"],
         "suppressed_params": set(),
         "retry_json": True,
@@ -183,6 +184,7 @@ def _call_model(
             "frequency_penalty": self.frequency_penalty,
             "presence_penalty": self.presence_penalty,
             "stop": self.stop,
+            "seed": self.seed,
         }
 
         create_args = {