Check and fix some parameter type errors in possible answers

[zhangch-ss]

hi, I found some inconsistencies in parameter types between possible_answers and function descriptions, as shown below

possible answer: {"id": "live_multiple_857-180-5",..."Buses_3_FindBus": ... "num_passengers": ["", "1"], ...

function description: {"id": "live_multiple_857-180-5", ..."Buses_3_FindBus", ..."num_passengers": {"type": "integer"...

these errors have affected the evaluation of the model

"error": ["Invalid value for parameter 'num_passengers': 1. Expected one of ['', '1']."], "error_type": "value_error:others", ...

the checker is as follows

import os
import json
def possible_answer_param_checker():
    # check that the parameter type of the function call is consistent with the possible answers
    data_dir = "gorilla/berkeley-function-call-leaderboard/data"
    possible_answer_dir = "gorilla/berkeley-function-call-leaderboard/data/possible_answer"
    possible_answer_path_list = os.listdir(possible_answer_dir)
    for possible_answer_path in possible_answer_path_list:
        if "multi_turn" in possible_answer_path or "java" in possible_answer_path or "sql" in possible_answer_path:
            continue
        with open(os.path.join(possible_answer_dir, possible_answer_path), "r") as pa_data:
            pa_data = pa_data.readlines()
        with open(os.path.join(data_dir, possible_answer_path), "r") as data:
            data = data.readlines()
        for line in data:
            for line_pa in pa_data:
                line_json = json.loads(line)
                line_pa_json = json.loads(line_pa)
                id = line_json.get("id")
                if id == line_pa_json["id"]:
                    ground_truth_list = line_pa_json.get("ground_truth")
                    function_list = line_json.get("function")
                    for ground_truth in ground_truth_list:
                        for function in function_list:
                            for gtk, gtv in ground_truth.items():
                                if gtk == function.get("name"):
                                    function_parameters = function.get("parameters").get("properties")
                                    for fp_name, fp_value in function_parameters.items():
                                        for gtp_name, gtp_value_list in gtv.items():
                                            if gtp_name==fp_name:
                                                for gtp_value_i in gtp_value_list:
                                                    if gtp_value_i:
                                                        if fp_value.get("type")=="any":
                                                            continue
                                                        if fp_value.get("type")=="tuple":
                                                            if not isinstance(gtp_value_i, list):
                                                                print(id, gtk, gtp_name,":",gtp_value_i, fp_value.get("type"),)
                                                            continue
                                                        if not isinstance(gtp_value_i, v_type_map[fp_value.get("type")]):
                                                            print(id, gtk, gtp_name,":",gtp_value_i, fp_value.get("type"))

also, some cases confuse me, like

function description: {"id": "live_multiple_338-133-2", "question": [[{"role": "user", "content": "Can you search for some rock music"}]], "function": [...{"name": "Music_3_LookupMusic",... "year": {"type": "integer", "description": "The release year of the song. Use 'dontcare' to include songs from any year.", "default": "dontcare"}}, "required": []}}]}

possible answer: {"id": "live_multiple_338-133-2", "ground_truth": [{"Music_3_LookupMusic": {"artist": ["", "dontcare"], "album": ["", "dontcare"], "genre": ["Rock"], "year": ["", "dontcare"]}}]}

so, how do we define the type of the parameter 'year'

thanks!