Fix/flaky tools

tools/optimization_by_prompting.py

@@ -102,7 +102,7 @@
    - "info_utility": Utility of the information provided in "ADDITIONAL_INFORMATION" to help you make the prediction.
      0 indicates lowest utility; 1 maximum utility.
 * The sum of "p_yes" and "p_no" must equal 1.
-* Output only the JSON object. Do not include any other contents in your response."""
+"""
 
 URL_QUERY_PROMPT = """
 You are an LLM inside a multi-agent system that takes in a prompt of a user requesting a probability estimation
@@ -150,14 +150,17 @@
 PROMPT_INSTRUCTOR = PromptTemplate(
     input_variables=["instructions", "score"], template=TEMPLATE_INSTRUCTOR
 )
-
+OUTPUT_FORMAT = """
+Your output response must be only a single JSON object to be parsed by Python's "json.loads()".
+The JSON must contain a field "p_yes" which marks the probability of the event happening. 
+"""
 
 def evaluate_prompt(prompt, df, llm):
     chain = LLMChain(llm=llm, prompt=prompt)
     probas = []
 
     for row in df.itertuples():
-        pred_chain = chain.run({"user_prompt": row.query, "additional_information": ""})
+        pred_chain = chain.run({"user_prompt": row.query, "additional_information": OUTPUT_FORMAT})
         try:
             dictionary_match = float(eval(pred_chain)["p_yes"])
         except:
@@ -187,10 +190,17 @@ def prompt_engineer(openai_api_key, init_instructions, instructions_format, iter
     template = init_instructions
 
     for _ in range(iterations):
-        prompt = PromptTemplate(
-            input_variables=["user_prompt", "additional_information"],
-            template=template + instructions_format,
-        )
+        generated_template = template + instructions_format
+        try:
+            prompt = PromptTemplate(
+                input_variables=["user_prompt", "additional_information"],
+                template=generated_template,
+            )
+        except Exception as e:
+            # it may happen that the generated prompt is not valid
+            # in that case, we just skip it
+            print(f"Failed to parse template {generated_template}: {e}")
+            continue
 
         df["probability"] = evaluate_prompt(prompt=prompt, llm=llm, df=df)
 

tools/prediction_request_claude.py

@@ -110,7 +110,9 @@
    - "queries": An array of strings of size between 1 and 5. Each string must be a search engine query that can help obtain relevant information to estimate
      the probability that the event in "USER_PROMPT" occurs. You must provide original information in each query, and they should not overlap
      or lead to obtain the same set of results.
-* Output only the JSON object to be parsed by Python's "json.loads()". Do not include any other contents in your response.
+* Output only the JSON object to be parsed by Python's "json.loads()". Do not include any other contents in your response. This means the output MUST be a stringified JSON object. 
+* A valid output format is: {{"queries": ["q1", "q2"]}}
+* You MUST NOT include any extra characters, like end-lines, quotes etc.
 """