diff --git a/docs/04_Evaluating_OpenAI_Functions.md b/docs/04_Evaluating_OpenAI_Functions.md new file mode 100644 index 0000000..d11fd6f --- /dev/null +++ b/docs/04_Evaluating_OpenAI_Functions.md @@ -0,0 +1,23199 @@ +[ + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide any additional information about the patient's symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide more information about the patient's history.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information to extract the patient's history. Can you provide additional details such as gender, age, symptoms, and medicines?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": -1,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"head\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"body\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "unknown", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's history?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide more information about the patient's gender, age, and any other symptoms or medications they are experiencing.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "unknown", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "Chronic macular rash to face & hair", + "duration": "Unknown", + "body_parts_or_organs": "Face, Hair" + }, + { + "description": "Worse in beard, eyebrows & nares", + "duration": "Unknown", + "body_parts_or_organs": "Beard, Eyebrows, Nares" + }, + { + "description": "Itchy, flaky, slightly scaly", + "duration": "Unknown", + "body_parts_or_organs": "Face, Hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "Unknown", + "frequency": "Unknown", + "effect": "Moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].description", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency", + "medicines[0].effect" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information to extract the patient's history. Can you provide the patient's gender, age, symptoms, and medicines they are taking?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide more information about the patient's history.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "unknown", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "unknown", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "unknown", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "unknown", + "body_parts_or_organs": "lungs" + }, + { + "description": "fever", + "duration": "unknown", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Please provide more information about the patient's history.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": -1,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"head\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face, hair, beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information from the patient's history. Can you provide the patient's gender, age, symptoms, and medicines?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"ongoing\",\n \"body_parts_or_organs\": \"head\"\n }\n ]\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "", + "body_parts_or_organs": "" + }, + { + "description": "wet cough", + "duration": "", + "body_parts_or_organs": "" + }, + { + "description": "fever", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "macular rash to face & hair", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "chronic", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information about the patient's history. Can you please provide the patient's gender, age, and any other relevant symptoms or medications?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you provide more information about the patient's history? For example, their gender, age, any other symptoms they are experiencing, and any medications they are currently taking.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [ + "symptoms[0].duration" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"\",\n \"body_parts_or_organs\": \"\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + }, + { + "description": "low fever", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "use daily", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": null,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": null,\n \"body_parts_or_organs\": null\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": 0,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"head\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"whole body\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as directed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more details about the patient's condition?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"present\",\n \"body_parts_or_organs\": \"head\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "unknown", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "unknown", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "unknown", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "unknown", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "unknown", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Please provide more information about the patient's history.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you provide more information about the patient's gender and age?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "unknown", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information about the patient. Could you please provide the patient's gender, age, and any other symptoms or medicines they have?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": 0,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"unknown\",\n \"body_parts_or_organs\": \"head\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"unknown\",\n \"body_parts_or_organs\": \"body\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "unknown", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "unknown", + "body_parts_or_organs": "lungs" + }, + { + "description": "fever", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Please provide more information about the patient's history, such as their gender, age, and any other symptoms they are experiencing. This will help me extract the relevant information.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Could you please provide more details about the patient's gender, age, and any other symptoms or conditions they might have?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "chronic", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + }, + { + "description": "moderate response to OTC steroid cream", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Sure, I will need some more information about the patient. Can you please provide the patient's gender and age?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"\",\n \"age\": 0,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"\",\n \"body_parts_or_organs\": \"\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "unknown", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "moderate response to OTC steroid cream", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I will need more information to extract the patient's history. Please provide the patient's gender and age.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n\"history\": {\n \"gender\": \"Unknown\",\n \"age\": null,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": null,\n \"body_parts_or_organs\": null\n },\n {\n \"description\": \"low fever\",\n \"duration\": null,\n \"body_parts_or_organs\": null\n }\n ],\n \"medicines\": []\n}\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "chronic", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Please provide more information about the patient's history.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": null,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": null,\n \"body_parts_or_organs\": null\n },\n {\n \"description\": \"low fever\",\n \"duration\": null,\n \"body_parts_or_organs\": null\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "chest" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Please provide more information about the patient's history.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you provide more details about the patient's symptoms? How long has the patient had the low fever and any other symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "unknown", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information to create the patient's history. Can you provide the patient's gender, age, symptoms, and any medicines they have been taking?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Could you please provide more details about the symptoms and any other relevant information?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "unknown", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "unknown", + "body_parts_or_organs": "lungs" + }, + { + "description": "fever", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information to extract the patient's history. Can you please provide the patient's gender and age?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide the gender and age of the patient.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "sudden", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "recent", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Please provide additional information about the patient.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you provide more information about the patient's gender, age, and any other symptoms they are experiencing?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "chronic", + "body_parts_or_organs": "face & hair" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "chronic", + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "chronic", + "body_parts_or_organs": "beard, eyebrows, nares" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].description", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more details about the symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide more information about the patient. What is the patient's gender and age?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "Chronic macular rash to face & hair", + "duration": "N/A", + "body_parts_or_organs": "Face, hair" + }, + { + "description": "Worse in beard, eyebrows & nares", + "duration": "N/A", + "body_parts_or_organs": "Beard, eyebrows, nares" + }, + { + "description": "Itchy, flaky, slightly scaly", + "duration": "N/A", + "body_parts_or_organs": "Face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "N/A", + "frequency": "N/A", + "effect": "Moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].description", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency", + "medicines[0].effect" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I have noted that the patient presents with acute headache. Is there any other information about the patient's symptoms or medical history that you can provide?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide the patient's gender and age.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + }, + { + "description": "low fever", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information to extract the patient's history. Can you provide more details about the patient's symptoms, gender, age, and any medications they are currently taking?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": NaN,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"head\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"body\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "", + "body_parts_or_organs": "" + }, + { + "description": "wet cough", + "duration": "", + "body_parts_or_organs": "" + }, + { + "description": "fever", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "macular rash to face & hair", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "chronic", + "body_parts_or_organs": "" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "", + "frequency": "", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration" + ], + "incorrect": [ + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide any additional information about the patient's symptoms or history?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you provide any additional symptoms or information about the patient?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "unknown", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "unknown", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Please provide more information about the patient's symptoms.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you please provide more details about the patient's gender, age, and any other symptoms they are experiencing?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information to extract the patient's history. Can you provide additional details about the patient's gender, age, symptoms, and medicines they are taking?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": -1,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"unknown\",\n \"body_parts_or_organs\": \"head\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"unknown\",\n \"body_parts_or_organs\": \"whole body\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "apply a thin layer", + "frequency": "once daily", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's symptoms, such as the duration of the headache and any other accompanying symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Could you please provide more details about the patient's gender, age, and any other symptoms they are experiencing?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "apply a thin layer", + "frequency": "twice daily", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [ + "symptoms[2].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide any additional information about the patient's history?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you please provide me with the gender, age, symptoms, and medicines of the patient?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "unknown", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "unknown", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "unknown", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair, worse in beard, eyebrows & nares", + "duration": "unknown", + "body_parts_or_organs": "face, hair, beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide any additional information about the patient's symptoms or history?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you provide any additional information about the patient's symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[1].duration", + "symptoms[2].duration" + ], + "incorrect": [ + "symptoms[0].body_parts_or_organs", + "symptoms[1].body_parts_or_organs", + "symptoms[2].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "unknown", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I understand that the patient is experiencing acute headache. Is there any additional information about the symptoms or any other relevant details?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you please provide more details about the patient's symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "macular rash to face & hair", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "chronic", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + }, + { + "description": "slightly scaly", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].description", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "When did the headache start?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"unknown\",\n \"body_parts_or_organs\": \"head\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[1].duration", + "symptoms[2].duration" + ], + "incorrect": [ + "symptoms[0].body_parts_or_organs", + "symptoms[1].body_parts_or_organs", + "symptoms[2].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "N/A", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "N/A", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "N/A", + "body_parts_or_organs": "N/A" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "N/A", + "frequency": "N/A", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Got it. Could you please provide more information about the patient's gender, age, symptoms, and any medications they are currently taking?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide more information about the patient's gender, age, duration of symptoms, body parts or organs affected, and any medicines that have been prescribed.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[1].duration", + "symptoms[2].duration" + ], + "incorrect": [ + "symptoms[0].body_parts_or_organs", + "symptoms[1].body_parts_or_organs", + "symptoms[2].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "unknown", + "body_parts_or_organs": "beard, eyebrows & nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you provide any more information about the patient's condition?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "unknown", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's history? Specifically, I need to know the patient's gender, age, and any other relevant symptoms or medications.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": null,\n \"age\": null,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": null,\n \"body_parts_or_organs\": null\n },\n {\n \"description\": \"low fever\",\n \"duration\": null,\n \"body_parts_or_organs\": null\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [ + "symptoms[0].duration" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Please provide more information about the patient's history.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": -1,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"head\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"whole body\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "apply a thin layer", + "frequency": "twice a day", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [ + "symptoms[2].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide more information about the patient's gender and age.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "throat/lungs" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "macular rash to face and hair", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I see that the patient is experiencing acute headache. Can you provide any additional information about the symptoms, duration, or any other relevant details?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Ok, I got it. The patient was admitted with acute headache and now also has a low fever. Is there any additional information that I should know about the patient's condition?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "", + "body_parts_or_organs": "" + }, + { + "description": "moderate response to OTC steroid cream", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [ + "symptoms[0].duration", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I will need more information in order to extract the patient's history. Can you provide additional details such as the patient's gender, age, and any other symptoms they are experiencing?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": null,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"unknown\",\n \"body_parts_or_organs\": \"head\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"unknown\",\n \"body_parts_or_organs\": \"whole body\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": null,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"head\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": null\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "recently", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "apply a thin layer", + "frequency": "twice daily", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's symptoms and any other relevant details?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": null,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"head\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"whole body\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + }, + { + "description": "low fever", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "Chronic macular rash to face & hair", + "duration": "N/A", + "body_parts_or_organs": "Face, hair" + }, + { + "description": "Worse in beard, eyebrows & nares", + "duration": "N/A", + "body_parts_or_organs": "Beard, eyebrows, nares" + }, + { + "description": "Itchy, flaky, slightly scaly", + "duration": "N/A", + "body_parts_or_organs": "Face" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "N/A", + "frequency": "N/A", + "effect": "Moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].description", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency", + "medicines[0].effect" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide any additional information about the patient's history?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": -1,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"head\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "Present", + "body_parts_or_organs": "Head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Okay, I will need more information to extract the patient's history. Can you provide the patient's gender, age, and any other symptoms or medicines?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": 0,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"head\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"whole body\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "unknown", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "unknown", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "unknown", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "unknown", + "body_parts_or_organs": "lungs" + }, + { + "description": "fever", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face & hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide any additional information about the patient's symptoms or history?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": 0,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"head\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "current", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "Unknown", + "body_parts_or_organs": "" + }, + { + "description": "wet cough", + "duration": "Unknown", + "body_parts_or_organs": "" + }, + { + "description": "fever", + "duration": "Unknown", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[1].duration", + "symptoms[2].duration" + ], + "incorrect": [ + "symptoms[0].body_parts_or_organs", + "symptoms[1].body_parts_or_organs", + "symptoms[2].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "unknown", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Please provide more information about the patient's history.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide more information about the patient's history.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "Chronic macular rash to face & hair", + "duration": "Unknown", + "body_parts_or_organs": "Face, Hair" + }, + { + "description": "Worse in beard, eyebrows & nares", + "duration": "Unknown", + "body_parts_or_organs": "Beard, Eyebrows, Nares" + }, + { + "description": "Itchy, flaky, slightly scaly", + "duration": "Unknown", + "body_parts_or_organs": "Unknown" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "Unknown", + "frequency": "Unknown", + "effect": "Moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].description", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency", + "medicines[0].effect" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "What is the patient's gender?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide more information about the patient's gender, age, symptoms, and any medicines that they are currently taking.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "ongoing", + "body_parts_or_organs": "body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[1].duration", + "symptoms[2].duration" + ], + "incorrect": [ + "symptoms[0].body_parts_or_organs", + "symptoms[1].body_parts_or_organs", + "symptoms[2].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "affected areas" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information to extract the patient's history. Can you provide the patient's gender, age, and any other symptoms they are experiencing?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Could you please provide more details about the patient's symptoms and any other relevant information?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "chest" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "Chronic macular rash to face & hair", + "duration": "Unknown", + "body_parts_or_organs": "Face, hair" + }, + { + "description": "Worse in beard, eyebrows & nares", + "duration": "Unknown", + "body_parts_or_organs": "Beard, eyebrows, nares" + }, + { + "description": "Itchy, flaky, slightly scaly", + "duration": "Unknown", + "body_parts_or_organs": "Face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "Unknown", + "frequency": "Unknown", + "effect": "Moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].description", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency", + "medicines[0].effect" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I understand that the patient is presenting with an acute headache. Is there any additional information about the patient's symptoms or medical history that you can provide?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"ongoing\",\n \"body_parts_or_organs\": \"head\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"ongoing\",\n \"body_parts_or_organs\": \"whole body\"\n }\n ]\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "throat/lungs" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as directed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information to extract the patient's history. Can you provide the patient's gender, age, symptoms, and medicines?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you please provide more information about the patient? Specifically, can you tell me the patient's gender and age?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": "Thank you for the information. Has the patient experienced any other symptoms besides the acute headache and low fever?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [ + "content" + ] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair, worse in beard, eyebrows & nares", + "duration": "unknown", + "body_parts_or_organs": "face, hair, beard, eyebrows, nares" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide me with more information about the patient's symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "I see that the patient was admitted with acute headache and now also has a low fever. Is there any other information about the patient's symptoms or medical history that you can provide?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "Chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "Itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": null,\n \"symptoms\": [\n {\n \"description\": \"headache\",\n \"duration\": \"acute\",\n \"body_parts_or_organs\": \"head\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you provide more information about the patient's gender, age, and any other symptoms they may be experiencing?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "persistent", + "body_parts_or_organs": "face, hair" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "persistent", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I have extracted the patient's symptom, which is acute headache. Is there any additional information you would like to provide about the patient's history?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": -1,\n \"symptoms\": [{\n \"description\": \"acute headache\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"Unknown\"\n }],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I will need more information to extract the patient's history. Can you please provide the following details?\n1. Gender of the patient\n2. Age of the patient\n3. Duration and description of the symptoms\n4. Any specific body parts or organs affected\n5. Any medications taken and their details (name, dosage, frequency, and effect)", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you provide more information about the patient's gender, age, symptoms duration, and body parts affected?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": "Has the patient reported any other symptoms besides the headache and low fever?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [ + "content" + ] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "chronic", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + }, + { + "description": "moderate response to OTC steroid cream", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": null,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": null,\n \"body_parts_or_organs\": null\n },\n {\n \"description\": \"low fever\",\n \"duration\": null,\n \"body_parts_or_organs\": null\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "unknown", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "unknown", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "Moderate response to OTC steroid cream", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's symptoms and any other relevant details?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide more information about the patient's gender, age, symptoms, and any medicines they are currently taking.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's gender, age, and any other symptoms they are experiencing?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "I have noted that the patient was admitted with acute headache and now also has a low fever.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[1].duration", + "symptoms[2].duration" + ], + "incorrect": [ + "symptoms[0].body_parts_or_organs", + "symptoms[1].body_parts_or_organs", + "symptoms[2].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "unknown", + "body_parts_or_organs": "beard, eyebrows & nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Okay, I have noted that the patient was admitted with acute headache and now also has a low fever. Is there any additional information you have about the patient's condition?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].body_parts_or_organs", + "symptoms[1].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Please provide more information about the patient's history, such as their gender, age, and any other relevant symptoms or medicines they have taken.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you please provide the gender and age of the patient?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"present\",\n \"body_parts_or_organs\": \"head\"\n }\n ]\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide the gender and age of the patient.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "present", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "Chronic macular rash to face and hair", + "duration": "Unknown", + "body_parts_or_organs": "Face, hair" + }, + { + "description": "Worse in beard, eyebrows, and nares", + "duration": "Unknown", + "body_parts_or_organs": "Beard, eyebrows, nares" + }, + { + "description": "Itchy, flaky, slightly scaly", + "duration": "Unknown", + "body_parts_or_organs": "Face, hair, beard, eyebrows, nares" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "Unknown", + "frequency": "Unknown", + "effect": "Moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].description", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency", + "medicines[0].effect" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's gender, age, and other symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you provide more details about the patient's condition? How long has the patient been experiencing the headache and low fever?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ] + }, + "result": { + "missing_keys": [ + "medicines" + ], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "moderate response to OTC steroid cream", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information to extract the patient's history. Please provide the gender, age, symptoms, and medicines.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide more information about the patient's history.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "unknown", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "unknown", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[1].duration", + "symptoms[2].duration" + ], + "incorrect": [ + "symptoms[0].body_parts_or_organs", + "symptoms[1].body_parts_or_organs", + "symptoms[2].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "unknown", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you provide more information about the patient's history? For example, their gender, age, and any other relevant symptoms or medications they may be taking.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "acute", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "unknown", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "unknown", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "chronic", + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "chronic", + "body_parts_or_organs": "beard, eyebrows & nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "chronic", + "body_parts_or_organs": "face & hair" + }, + { + "description": "moderate response to OTC steroid cream", + "duration": "chronic", + "body_parts_or_organs": "face & hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as directed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information about the patient's history. Can you please provide the patient's gender, age, symptoms, and any medicines they are currently taking?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide more information about the patient. What is their gender and age?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + }, + { + "description": "low fever", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "itchy", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "flaky", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[2].description", + "symptoms[2].duration", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [ + "symptoms[1].body_parts_or_organs", + "symptoms[2].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide any additional information about the patient's symptoms or medical history?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"\",\n \"body_parts_or_organs\": \"\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"\",\n \"body_parts_or_organs\": \"\"\n }\n ],\n \"gender\": \"Unknown\",\n \"age\": -1\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "ongoing", + "body_parts_or_organs": "body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "unknown", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "unknown" + }, + { + "description": "moderate response to OTC steroid cream", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information. How long has the patient been experiencing the headache?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"admitted\",\n \"body_parts_or_organs\": \"\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"\",\n \"body_parts_or_organs\": \"\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [ + "symptoms[0].duration" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "Macular rash to face & hair", + "duration": "Chronic", + "body_parts_or_organs": "Face, hair" + }, + { + "description": "Worse in beard, eyebrows & nares", + "duration": "Chronic", + "body_parts_or_organs": "Beard, eyebrows, nares" + }, + { + "description": "Itchy, flaky, slightly scaly", + "duration": "Chronic", + "body_parts_or_organs": "Face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "Unknown", + "frequency": "Unknown", + "effect": "Moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].description", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency", + "medicines[0].effect" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need some more information to extract the patient's history. Could you please provide the patient's gender, age, symptoms, and any medicines they have been taking?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": 0,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"Head\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "daily", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information to extract the patient's history. Can you please provide the patient's gender, age, symptoms, and any medicines taken?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide more information about the patient's gender and age.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "unknown", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "unknown", + "body_parts_or_organs": "lungs" + }, + { + "description": "fever", + "duration": "unknown", + "body_parts_or_organs": "body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "persistent", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "persistent", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "persistent", + "body_parts_or_organs": "affected area" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information to extract the patient's history. Could you please provide the following details:\n1. Gender of the patient?\n2. Age of the patient?\n3. Any other symptoms the patient is experiencing?\n4. Any medications the patient is currently taking?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide more information about the patient's gender, age, duration of symptoms, and any medicines they have been taking.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "unknown", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's history?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you provide more information about the patient's symptoms? How long have they been experiencing the headache and fever?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "unknown", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as instructed", + "frequency": "as instructed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide any additional information about the patient's symptoms or medical history?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"unknown\",\n \"body_parts_or_organs\": \"head\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"unknown\",\n \"body_parts_or_organs\": \"whole body\"\n }\n ]\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "low fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information to extract the patient's history. Can you provide the patient's gender, age, and any other symptoms they are experiencing?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": 0,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"unknown\",\n \"body_parts_or_organs\": \"head\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"unknown\",\n \"body_parts_or_organs\": \"whole body\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face & hair" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows & nares" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's history?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you provide more details about the patient's symptoms? Are there any other symptoms or changes in their condition that you are aware of?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "ongoing", + "body_parts_or_organs": "body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[1].duration", + "symptoms[2].duration" + ], + "incorrect": [ + "symptoms[0].body_parts_or_organs", + "symptoms[1].body_parts_or_organs", + "symptoms[2].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Please provide additional information about the patient's history.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": null,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"head\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"whole body\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "flaky", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].description", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information from the patient to build their history. Can you please provide the following details:\n\n1. Gender\n2. Age\n3. Description of symptoms\n4. Duration of symptoms\n5. Body parts or organs affected (if any)", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide more details about the patient's gender, age, and any other relevant information about their symptoms and medicines.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[1].duration", + "symptoms[2].duration" + ], + "incorrect": [ + "symptoms[0].body_parts_or_organs", + "symptoms[1].body_parts_or_organs", + "symptoms[2].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more details about the headache?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": null,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": null,\n \"body_parts_or_organs\": null\n },\n {\n \"description\": \"low fever\",\n \"duration\": null,\n \"body_parts_or_organs\": null\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + }, + { + "description": "low fever", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "macular rash to face & hair", + "duration": "chronic", + "body_parts_or_organs": "beard, eyebrows & nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "chronic", + "body_parts_or_organs": "face & hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Do you have any other information about the patient's symptoms or medical history?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Could you please provide more information about the patient's gender and age?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "present", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "present", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": null, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's history?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": null,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": null,\n \"body_parts_or_organs\": null\n },\n {\n \"description\": \"low fever\",\n \"duration\": null,\n \"body_parts_or_organs\": null\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + }, + { + "description": "low fever", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, beard, eyebrows, nares" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Please provide more details about the patient's symptoms.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide more information about the patient's gender, age, and any other symptoms or medications they have.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "unknown", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "unknown", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I will need more information to extract the patient's history. Can you provide me with the patient's gender, age, symptoms, and medicines?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Understood. Is there any other information about the patient's condition that I should know?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "unknown", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "unknown", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "unknown", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more details about the headache? For example, how long has the patient been experiencing the headache and any specific location or characteristics of the pain?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n\"history\": {\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"\",\n \"body_parts_or_organs\": \"head\"\n }\n ],\n \"medicines\": []\n}\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "present", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "present", + "body_parts_or_organs": "body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "unknown", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "unknown" + }, + { + "description": "moderate response to OTC steroid cream", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Sure, I will need some more information. Can you please provide me with the patient's gender and age?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Could you please provide more details about the patient's symptoms and any other relevant information?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "unknown", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Could you please provide more information about the patient's history? Specifically, I need to know the patient's gender, age, any related symptoms, and any medications they are currently taking.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Could you please provide more information about the patient's gender, age, and any other symptoms or medicines?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "Unknown", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "Unknown", + "body_parts_or_organs": "body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "moderate response to OTC steroid cream", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's symptoms, such as the duration and any associated body parts or organs affected?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide the patient's gender and age.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as directed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Please provide more information about the patient's history.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Please provide more information about the patient's gender, age, symptoms, and medicines.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair, worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "face, hair, beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair, beard, eyebrows, nares" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as needed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Sure, I can help you with that. Can you provide me with some more information?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Was the patient's gender mentioned in the notes?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "unknown", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "macular rash to face & hair", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "chronic", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I will need more information to extract the patient's history. Please provide the patient's gender, age, and any other symptoms or medical conditions they may have.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": null,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": null,\n \"body_parts_or_organs\": null\n },\n {\n \"description\": \"low fever\",\n \"duration\": null,\n \"body_parts_or_organs\": null\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's symptoms and any other relevant details?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": -1,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"Head\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"N/A\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "none" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "unknown", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide any additional information about the patient's symptoms or medical history?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"\",\n \"body_parts_or_organs\": \"\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash", + "duration": "ongoing", + "body_parts_or_organs": "face" + }, + { + "description": "worse in beard, eyebrows, and nares", + "duration": "ongoing", + "body_parts_or_organs": "face" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you provide any additional details about the patient's condition or symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "recently", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "recently", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "unknown", + "body_parts_or_organs": "unknown" + }, + { + "description": "wet cough", + "duration": "unknown", + "body_parts_or_organs": "unknown" + }, + { + "description": "fever", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you please provide more information about the patient's symptoms and any other relevant details?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Does the patient have any other symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "unknown", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "unknown", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information to extract the patient's history. Can you please provide the patient's gender and age?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"unknown\",\n \"body_parts_or_organs\": \"head\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"unknown\",\n \"body_parts_or_organs\": \"whole body\"\n }\n ]\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [ + "symptoms[0].duration" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "unknown", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "unknown", + "body_parts_or_organs": "lungs" + }, + { + "description": "fever", + "duration": "unknown", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I see that the patient is presenting with an acute headache. Can you provide me with any additional information about the patient's symptoms, such as the duration, description, and any specific body parts or organs affected?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you provide more details about the symptoms? How long has the patient had the headache and fever?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as directed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information to extract the patient's history. Can you provide any additional details about the patient's symptoms, gender, age, or any medications they are taking?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Did the patient have any other symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + }, + { + "description": "low fever", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "Chronic macular rash to face & hair", + "duration": "Unknown", + "body_parts_or_organs": "Face, hair" + }, + { + "description": "Worse in beard, eyebrows & nares", + "duration": "Unknown", + "body_parts_or_organs": "Beard, eyebrows, nares" + }, + { + "description": "Itchy, flaky, slightly scaly", + "duration": "Unknown", + "body_parts_or_organs": "Whole body" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "Unknown", + "frequency": "Unknown", + "effect": "Moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].description", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency", + "medicines[0].effect" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Please provide additional information about the patient's symptoms and any other relevant details.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Could you provide more information about the patient's gender and age?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "airways" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I understand that the patient is experiencing acute headache. Is there any additional information available about the patient's symptoms, gender, age, or any other relevant details?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you provide any more details about the patient's symptoms and medical history?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "present", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "moderate response to OTC steroid cream", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": 0,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"unknown\",\n \"body_parts_or_organs\": \"head\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"unknown\",\n \"body_parts_or_organs\": \"unknown\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash", + "duration": "unknown", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows, and nares", + "duration": "unknown", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "unknown", + "body_parts_or_organs": "unknown" + }, + { + "description": "moderate response to OTC steroid cream", + "duration": "unknown", + "body_parts_or_organs": "unknown" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].description", + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].description", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the patient's history?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": -1,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"unknown\",\n \"body_parts_or_organs\": \"head\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"unknown\",\n \"body_parts_or_organs\": \"whole body\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "unknown", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "unknown", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "chest" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": null, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you provide more information about the headache? Is there any specific location, duration, or any accompanying symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": "Can you provide more details about the patient's symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "unknown", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "unknown", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Can you please provide more details about the symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"duration not specified\",\n \"body_parts_or_organs\": \"head\"\n }\n ]\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "now", + "body_parts_or_organs": "head" + }, + { + "description": "low fever", + "duration": "now", + "body_parts_or_organs": "whole body" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "lungs" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "respiratory system" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "general" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "chronic", + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "worse", + "body_parts_or_organs": "beard, eyebrows & nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "chronic", + "body_parts_or_organs": "face & hair" + }, + { + "description": "moderate response to OTC steroid cream", + "duration": "chronic", + "body_parts_or_organs": "face & hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[1].duration", + "symptoms[1].body_parts_or_organs", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "Could you please provide more details about the patient's symptoms?", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"gender\": \"Unknown\",\n \"age\": undefined,\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"Unknown\",\n \"body_parts_or_organs\": \"head\"\n }\n ],\n \"medicines\": []\n }\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "ongoing", + "body_parts_or_organs": "head" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[1].duration", + "symptoms[2].duration" + ], + "incorrect": [ + "symptoms[0].body_parts_or_organs", + "symptoms[1].body_parts_or_organs", + "symptoms[2].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "chronic", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "chronic", + "body_parts_or_organs": "face, hair" + }, + { + "description": "moderate response to OTC steroid cream", + "duration": "chronic", + "body_parts_or_organs": "face" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "unknown", + "frequency": "unknown", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "single_user_message_simple_medical_information", + "expected": { + "role": "assistant", + "content": "What is the patient's age?" + }, + "actual": "I need more information to extract the patient's history. Please provide the following details:\n- Gender\n- Age\n- Description of symptoms\n- Duration of symptoms\n- Any specific body parts or organs affected\n- Any medicines taken, including the name, dosage, frequency, and effect.", + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_simple_medical_information", + "expected": { + "role": "assistant", + "content": "Need more info?" + }, + "actual": { + "role": "assistant", + "content": null, + "function_call": { + "name": "patient_history", + "arguments": "{\n \"history\": {\n \"symptoms\": [\n {\n \"description\": \"acute headache\",\n \"duration\": \"\",\n \"body_parts_or_organs\": \"\"\n },\n {\n \"description\": \"low fever\",\n \"duration\": \"\",\n \"body_parts_or_organs\": \"\"\n }\n ],\n \"gender\": \"\",\n \"age\": 0\n },\n \"medicines\": []\n}" + } + }, + "result": { + "missing_keys": [], + "extra_keys": [ + "function_call" + ], + "llm_hallucinated": [], + "incomplete": [], + "missing_values": [] + } + }, + { + "conversation_id": "multiple_user_messages_with_assistant_simple_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute headache", + "duration": "", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [], + "incorrect": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "1_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "wet cough", + "duration": null, + "body_parts_or_organs": null + }, + { + "description": "fever", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Female", + "age": 33, + "symptoms": [ + { + "description": "acute asthma", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "wet cough", + "duration": "ongoing", + "body_parts_or_organs": "" + }, + { + "description": "fever", + "duration": "ongoing", + "body_parts_or_organs": "" + } + ], + "medicines": [] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[1].duration", + "symptoms[2].duration" + ], + "incorrect": [ + "symptoms[0].body_parts_or_organs", + "symptoms[1].body_parts_or_organs", + "symptoms[2].body_parts_or_organs" + ], + "missing_values": [] + } + }, + { + "conversation_id": "2_complex_medical_information", + "expected": { + "function_call": { + "function_name": "save_patient_history", + "arguments": { + "history": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": null, + "body_parts_or_organs": "face & hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": null, + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": null, + "body_parts_or_organs": null + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": null, + "frequency": null, + "effect": "moderate response" + } + ] + } + } + }, + "role": "assistant" + }, + "actual": { + "gender": "Male", + "age": 49, + "symptoms": [ + { + "description": "chronic macular rash to face & hair", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + }, + { + "description": "worse in beard, eyebrows & nares", + "duration": "ongoing", + "body_parts_or_organs": "beard, eyebrows, nares" + }, + { + "description": "itchy, flaky, slightly scaly", + "duration": "ongoing", + "body_parts_or_organs": "face, hair" + } + ], + "medicines": [ + { + "name": "OTC steroid cream", + "dosage": "as directed", + "frequency": "as needed", + "effect": "moderate response" + } + ] + }, + "result": { + "missing_keys": [], + "extra_keys": [], + "llm_hallucinated": [ + "symptoms[0].duration", + "symptoms[0].body_parts_or_organs", + "symptoms[1].duration", + "symptoms[2].duration", + "symptoms[2].body_parts_or_organs", + "medicines[0].dosage", + "medicines[0].frequency" + ], + "incorrect": [], + "missing_values": [] + } + } +] \ No newline at end of file diff --git a/tests/test_func_parser.py b/tests/test_func_parser.py index 3229628..347f435 100644 --- a/tests/test_func_parser.py +++ b/tests/test_func_parser.py @@ -5,9 +5,7 @@ paper_dir_filepath = "examples/paper_dir" -def get_articles( - query: str, library: str = paper_dir_filepath, top_k: Optional[int] = 5 -) -> List[Dict[str, str]]: +def get_articles(query: str, library: str = paper_dir_filepath, top_k: Optional[int] = 5) -> List[Dict[str, str]]: """ This function gets the top_k articles based on a user's query, sorted by relevance. It also downloads the files and stores them in arxiv_library.csv to be retrieved by the read_article_and_summarize.