diff --git a/scripts/evaluation/convertToAnswerList.py b/scripts/evaluation/convertToAnswerList.py
new file mode 100644
index 0000000..e36d5ac
--- /dev/null
+++ b/scripts/evaluation/convertToAnswerList.py
@@ -0,0 +1,77 @@
+import json
+
+# Read the first JSON file
+with open('abcout.json') as file:
+    data = json.load(file)
+
+# Initialize empty lists
+ids = []
+questions = []
+responses = []
+
+# Iterate over the data
+for item in data:
+    # Extract ID
+    id = item['id']
+    ids.append(id)
+
+    # Extract conversations
+    conversations = item['conversations']
+
+    # Extract question and response from each conversation
+    for conversation in conversations:
+        if conversation['from'] == 'human':
+            question = conversation['value']
+        elif conversation['from'] == 'gpt':
+            response = conversation['value']
+
+    questions.append(question)
+    responses.append(response)
+
+# Read the second JSON file
+with open('questionsl.json') as file:
+    question_data = file.readlines()
+
+# Initialize empty list
+question_ids = []
+
+# Extract question IDs from the second JSON file
+for question in questions:
+    question_id = None
+    for line in question_data:
+        data = json.loads(line)
+        if data['text'] == question:
+            question_id = data['question_id']
+            question_ids.append(question_id)
+            break
+
+print(question_id)
+# Print the first 10 questions and question IDs
+print("First 10 Questions:")
+for question in questions[:10]:
+    print(question)
+
+print("\nFirst 10 Question IDs:")
+print(question_ids[:10])
+
+
+# Create a list to hold the formatted data
+formatted_data = []
+
+# Iterate over the arrays and format the data
+for i in range(len(ids)):
+    data = {
+        "answer_id": ids[i],
+        "model_id": "gpt-3.5-turbo:20230327",
+        "question_id": question_ids[i],
+        "text": responses[i],
+        "metadata": {}
+    }
+    formatted_data.append(data)
+
+# Convert the formatted data to a JSON string with line breaks
+json_data = "\n".join(json.dumps(entry) for entry in formatted_data)
+
+# Write the formatted data to a JSON file
+with open('formatted_data.json', 'w') as file:
+    file.write(json_data)
\ No newline at end of file
diff --git a/scripts/evaluation/eval_gpt_review.py b/scripts/evaluation/eval_gpt_review.py
new file mode 100644
index 0000000..4223fca
--- /dev/null
+++ b/scripts/evaluation/eval_gpt_review.py
@@ -0,0 +1,115 @@
+import argparse
+import json
+import os
+
+import openai
+
+import ray
+import time
+
+
+@ray.remote(num_cpus=4)
+def get_eval(content: str, max_tokens: int):
+    counter = 0
+    while counter < 5:
+        counter += 1
+        try:
+            response = openai.ChatCompletion.create(
+                model='gpt-3.5-turbo',
+                messages=[{
+                    'role': 'system',
+                    'content': 'You are a helpful and precise assistant for checking the quality of the answer.'
+                }, {
+                    'role': 'user',
+                    'content': content,
+                }],
+                temperature=0.2,  # TODO: figure out which temperature is best for evaluation
+                max_tokens=max_tokens,
+            )
+            break
+        except openai.error.RateLimitError:
+            pass
+        except Exception as e:
+            print(e)
+        time.sleep(1)
+
+    print('success!')
+    return response['choices'][0]['message']['content']
+
+
+def parse_score(review):
+    try:
+        score_pair = review.split('\n')[0]
+        score_pair = score_pair.replace(',', ' ')
+        sp = score_pair.split(' ')
+        if len(sp) == 2:
+            return [float(sp[0]), float(sp[1])]
+        else:
+            print('error', review)
+            return [-1, -1]
+    except Exception as e:
+        print(e)
+        print('error', review)
+        return [-1, -1]
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='ChatGPT-based QA evaluation.')
+    parser.add_argument('-q', '--question')
+    # parser.add_argument('-a', '--answer')
+    parser.add_argument('-a', '--answer-list', nargs='+', default=[])
+    parser.add_argument('-r', '--rule')
+    parser.add_argument('-o', '--output')
+
+    parser.add_argument('--max-tokens', type=int, default=1024, help='maximum number of tokens produced in the output')
+    args = parser.parse_args()
+
+    ray.init()
+
+    f_q = open(os.path.expanduser(args.question))
+    f_ans1 = open(os.path.expanduser(args.answer_list[0]))
+    f_ans2 = open(os.path.expanduser(args.answer_list[1]))
+    rule_dict = json.load(open(os.path.expanduser(args.rule), 'r'))
+
+    review_file = open(f'{args.output}', 'w')
+
+    js_list = []
+    handles = []
+    idx = 0
+    for ques_js, ans1_js, ans2_js in zip(f_q, f_ans1, f_ans2):
+        # if idx == 1:
+        #     break
+
+        ques = json.loads(ques_js)
+        ans1 = json.loads(ans1_js)
+        ans2 = json.loads(ans2_js)
+
+        category = json.loads(ques_js)['category']
+        if category in rule_dict:
+            rule = rule_dict[category]
+        else:
+            rule = rule_dict['default']
+        prompt = rule['prompt']
+        role = rule['role']
+        content = (f'[Question]\n{ques["text"]}\n\n'
+                   f'[{role} 1]\n{ans1["text"]}\n\n[End of {role} 1]\n\n'
+                   f'[{role} 2]\n{ans2["text"]}\n\n[End of {role} 2]\n\n'
+                   f'[System]\n{prompt}\n\n')
+        js_list.append({
+            'id': idx+1,
+            'question_id': ques['question_id'],
+            'answer1_id': ans1['answer_id'],
+            'answer2_id': ans2['answer_id'],
+            'category': category})
+        idx += 1
+        handles.append(get_eval.remote(content, args.max_tokens))
+        # To avoid the rate limit set by OpenAI
+        time.sleep(1)
+
+    reviews = ray.get(handles)
+    for idx, review in enumerate(reviews):
+        scores = parse_score(review)
+        js_list[idx]['content'] = review
+        js_list[idx]['tuple'] = scores 
+        review_file.write(json.dumps(js_list[idx]) + '\n')
+    review_file.close()
diff --git a/scripts/evaluation/formatted_data.json b/scripts/evaluation/formatted_data.json
new file mode 100644
index 0000000..ae943a7
--- /dev/null
+++ b/scripts/evaluation/formatted_data.json
@@ -0,0 +1,28 @@
+{"answer_id": "36237313", "model_id": "gpt-3.5-turbo:20230327", "question_id": 15, "text": "The image appears to show three red objects that could potentially be supernovas. However, it is more likely that these red objects are foreground stars from our own galaxy. Additionally, there is a blue banner at the top of the image that mentions blog posts which may provide further information about the peculiar clumpy galaxies depicted.", "metadata": {}}
+{"answer_id": "37897318", "model_id": "gpt-3.5-turbo:20230327", "question_id": 22, "text": "Based on the visual characteristics of the image, it appears to show a galaxy that could be classified as either elliptical or possibly a late disk galaxy.", "metadata": {}}
+{"answer_id": "55345398", "model_id": "gpt-3.5-turbo:20230327", "question_id": 3, "text": "The image shows a galaxy with a lenticular shape, indicating a disk-like structure with a central bulge. The galaxy appears to be disturbed, as evidenced by the presence of tidal debris forming an arm-like feature. This suggests that the galaxy has undergone some sort of gravitational interaction or merger with another object.", "metadata": {}}
+{"answer_id": "39374539", "model_id": "gpt-3.5-turbo:20230327", "question_id": 19, "text": "The image shows a galaxy that appears to be viewed from its edge, with a noticeable dust lane running through it. The galaxy also appears slightly disturbed at the tips, which show signs of star formation.", "metadata": {}}
+{"answer_id": "55350794", "model_id": "gpt-3.5-turbo:20230327", "question_id": 12, "text": "The image appears to show a galaxy that initially resembles an elliptical shape. However, upon closer examination of the residual image, additional details become apparent.", "metadata": {}}
+{"answer_id": "36255152", "model_id": "gpt-3.5-turbo:20230327", "question_id": 22, "text": "Based on the visual characteristics of the image, it appears to show a galaxy with an edge-on orientation. The presence of broad lines suggests the presence of a quasar, which is a type of active galactic nucleus. The galaxy appears to belong to the Sc type, indicating a spiral structure.", "metadata": {}}
+{"answer_id": "40361164", "model_id": "gpt-3.5-turbo:20230327", "question_id": 10, "text": "The image shows a galaxy with spiral arms that have a noticeable blue color. The galaxy appears to have a clumpy and barred spiral structure, with a short strong bar and tightly wrapped spiral arms that almost form a ring. There is a possible tidal trail above the galaxy, which may be caused by the presence of a companion galaxy below it. Additionally, there are two greenish ansae located at either end of the bar.", "metadata": {}}
+{"answer_id": "36250543", "model_id": "gpt-3.5-turbo:20230327", "question_id": 13, "text": "The image appears to show two separate objects. One object seems to be a starforming region, indicated by its bright and active appearance. The other object appears to be a galaxy, with a slightly larger redshift value. The galaxy seems to have a more diffuse and extended structure compared to the starforming region. Overall, the image showcases different astronomical phenomena, highlighting the diversity of objects in the universe.", "metadata": {}}
+{"answer_id": "25811434", "model_id": "gpt-3.5-turbo:20230327", "question_id": 5, "text": "The image shows a central core with an x-shape pattern. The orbits of the core appear to be structured in a way that resembles the letter \"X\". This pattern suggests a complex and dynamic system, possibly involving multiple interacting objects or forces. Further analysis would be needed to determine the nature and significance of this x-shape core and its orbits.", "metadata": {}}
+{"answer_id": "34240953", "model_id": "gpt-3.5-turbo:20230327", "question_id": 6, "text": "In the image, there are various aspects that can be observed. However, without specific details or context, it is difficult to provide a comprehensive explanation.", "metadata": {}}
+{"answer_id": "36222753", "model_id": "gpt-3.5-turbo:20230327", "question_id": 12, "text": "The image shows a galaxy with a spiral structure. There are several objects in contact with the galaxy's core, including three objects with rings. Additionally, there appears to be a gas ring surrounding the core of the galaxy.", "metadata": {}}
+{"answer_id": "32498813", "model_id": "gpt-3.5-turbo:20230327", "question_id": 4, "text": "The image shows a subject with a large core, possibly indicating a central concentration of stars or a dense region of gas and dust. There are suggestions of a barred structure, which could be a feature within the galaxy. Scattered galaxies are also visible in the image, indicating the presence of other galaxies in the vicinity. The subject appears undisturbed and is viewed face-on. Additionally, there is a separate object labeled as a quasar in the image.", "metadata": {}}
+{"answer_id": "21661664", "model_id": "gpt-3.5-turbo:20230327", "question_id": 17, "text": "The image appears to show a galaxy with irregular shape and active star formation, indicated by the presence of starburst regions.", "metadata": {}}
+{"answer_id": "20923069", "model_id": "gpt-3.5-turbo:20230327", "question_id": 7, "text": "The image shows a double galactic nucleus, with a green blob on the very right that appears to be an asteroid. There is also a noticeable double-core with a debris cloud.", "metadata": {}}
+{"answer_id": "51727000", "model_id": "gpt-3.5-turbo:20230327", "question_id": 8, "text": "The image appears to show a celestial object that is not clearly defined. It has a shape that could be described as a blob or a fuzzy disturbed spiral. The details of the object are not discernible from the image alone.", "metadata": {}}
+{"answer_id": "16215408", "model_id": "gpt-3.5-turbo:20230327", "question_id": 4, "text": "The image shows a galaxy with a slightly twisted disc. There is a red oval-shaped object above the galaxy, which appears to be smeared. Other stars in the image also show similar smearing, but not all of them. The circular appearance of the red oval star in a different image suggests that it may be an artifact of the CCD collection. There is a mention of incorrect coordinates for the image, but it is not clear which image is being referred to. Additionally, there seems to be a halo of stars above and below the central bulge of the galaxy, possibly resembling a polar ring.", "metadata": {}}
+{"answer_id": "36238970", "model_id": "gpt-3.5-turbo:20230327", "question_id": 12, "text": "The image appears to show multiple objects with distinct features. There are indications of star formation and starburst activity, as well as a clumpy structure.", "metadata": {}}
+{"answer_id": "34060815", "model_id": "gpt-3.5-turbo:20230327", "question_id": 4, "text": "The image appears to show a barred spiral galaxy. The presence of a bar structure in the center of the galaxy is evident. The spiral arms extend outward from the bar, creating a distinct pattern. The overall shape of the galaxy is symmetrical and exhibits a spiral structure.", "metadata": {}}
+{"answer_id": "21660643", "model_id": "gpt-3.5-turbo:20230327", "question_id": 5, "text": "Based on the image, it appears that there may be an issue with the size of the object being observed. The repeated use of the hashtag #wrong-size suggests that the users have noticed something unusual or unexpected in terms of the size of the object. However, without access to the actual image, it is difficult to provide a more detailed analysis.", "metadata": {}}
+{"answer_id": "21661614", "model_id": "gpt-3.5-turbo:20230327", "question_id": 3, "text": "Based on the image, there appears to be a suggestion of another galaxy located at the 11 o'clock position. However, without further details or visual information, it is difficult to provide a more elaborate explanation of the image.", "metadata": {}}
+{"answer_id": "28483442", "model_id": "gpt-3.5-turbo:20230327", "question_id": 10, "text": "In the image, there appears to be an irregular galaxy. The central region of the galaxy shows a star in contact with the core or possibly two cores. The image was taken over a period of 9 hours.", "metadata": {}}
+{"answer_id": "55351013", "model_id": "gpt-3.5-turbo:20230327", "question_id": 27, "text": "The image shows a celestial object with a distinct ring-like structure surrounding a central region. The outer ring appears to have a lens-like shape. The overall appearance of the object is reminiscent of a sunny-side-up egg. The image is visually intriguing and invites further exploration and analysis.", "metadata": {}}
+{"answer_id": "21668459", "model_id": "gpt-3.5-turbo:20230327", "question_id": 15, "text": "The image appears to show a galaxy with a ring-like structure, a lenticular shape, and a bar in the center. There is also a suggestion of a spiral structure starting to form. Additionally, the image indicates the presence of a Seyfert-2 galaxy and an active galactic nucleus (AGN).", "metadata": {}}
+{"answer_id": "40355779", "model_id": "gpt-3.5-turbo:20230327", "question_id": 17, "text": "The image appears to show a spiral galaxy that exhibits characteristics typically associated with a young age.", "metadata": {}}
+{"answer_id": "28490503", "model_id": "gpt-3.5-turbo:20230327", "question_id": 22, "text": "The image appears to show a portion of the night sky with multiple celestial objects. There are various bright spots and patterns, suggesting the presence of stars and possibly galaxies. The image also includes some fainter structures, which could be remnants of previous observations or artifacts. Overall, it seems to capture a small section of the vastness of the universe.", "metadata": {}}
+{"answer_id": "36213141", "model_id": "gpt-3.5-turbo:20230327", "question_id": 25, "text": "The photo shows blue clumps of star formation.", "metadata": {}}
+{"answer_id": "21665278", "model_id": "gpt-3.5-turbo:20230327", "question_id": 10, "text": "The image shows a spiral galaxy with a prominent bar feature in its center. There is also a bulge and a ring structure visible in the image.", "metadata": {}}
+{"answer_id": "20926746", "model_id": "gpt-3.5-turbo:20230327", "question_id": 12, "text": "The image shows a spiral galaxy with a faint arm that appears tightly wound. There is a prominent bar structure in the center of the galaxy. The galaxy exhibits signs of active star formation, as indicated by the presence of starburst regions. Additionally, there are nodes of resonance and areas of overlap within the galaxy.", "metadata": {}}
\ No newline at end of file
diff --git a/scripts/evaluation/getGPTtoLAVASTRO.py b/scripts/evaluation/getGPTtoLAVASTRO.py
new file mode 100644
index 0000000..976f4c2
--- /dev/null
+++ b/scripts/evaluation/getGPTtoLAVASTRO.py
@@ -0,0 +1,39 @@
+import json
+
+# Read in file (1)
+with open('llavastro.jsonl', 'r') as file1:
+    data1 = file1.readlines()
+
+# Read in file (2)
+with open('qa.json', 'r') as file2:
+    data2 = json.load(file2)
+
+# Create an empty list to store the matching entries
+output = []
+
+# Iterate over entries in file (1)
+for line in data1:
+    entry = json.loads(line)
+    question_id = entry['image_id']
+    
+    # Search for matching entry in file (2)
+    for item in data2:
+        if item['id'] == str(question_id):
+            entry2 = item.copy()
+            entry2['answer'] = entry
+            output.append(entry2)
+            break
+
+# Write the output to a JSON file
+with open('ABCOUT.json', 'w') as outfile:
+    json.dump(output, outfile)
+
+# Create a set of all image_ids from qa.json
+image_ids = {item['id'] for item in data2}
+
+# Filter out entries in data1 that don't have a corresponding image_id in data2
+filtered_data1 = [line for line in data1 if json.loads(line)['image_id'] in image_ids]
+
+# Write the filtered data to a new file
+with open('llavastro_cleared.jsonl', 'w') as outfile:
+    outfile.writelines(filtered_data1)
\ No newline at end of file
diff --git a/scripts/evaluation/llavastro_cleared.jsonl b/scripts/evaluation/llavastro_cleared.jsonl
new file mode 100644
index 0000000..08b4699
--- /dev/null
+++ b/scripts/evaluation/llavastro_cleared.jsonl
@@ -0,0 +1,28 @@
+{"image_id": "36237313", "question_id": 0, "prompt": "Describe the following image in detail.\n<image>", "text": "Assistant: The image features a remarkable observation of a distant galaxy, showcasing a large cluster of stars surrounded by a vast cloud of gas and dust. This beautiful and bright galaxy is captured in a black background, emphasizing its contrast and depth. The multitude of stars within the galaxy vary in size and brightness, creating an impressive display of celestial objects. TheHubble Space Telescope has captured this stunning scene, allowing us to appreciate the beauty and complexity of the universe.", "answer_id": "6VdcrAZ9nasTExhR6WSZL7", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "37897318", "question_id": 1, "prompt": "Provide a detailed description of the given image.\n<image>", "text": "Assistant: The image features a large, dark, circular galaxy against a dark background, likely a space scene. The galaxy takes up most of the frame and appears as a prominent, round, black object. There are several smaller, fainter stars scattered throughout the space, giving the impression of a vast, starry sky. These distant stars create a sense of depth and enhance the beauty of the galaxy in the dark sky.", "answer_id": "DVhXeqeoSZ9WwJQmuc3o4S", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "55345398", "question_id": 6, "prompt": "Clarify the contents of the displayed image with great detail.\n<image>", "text": "Assistant: The image features a crowded and busy space filled with many stars. Among these stars, there is a black hole located at the center of the view. The black hole seems to be sucking things in, creating a captivating scene.\n\nSeveral other stars are scattered throughout the frame, varying in size and brightness. They can be seen at different distances from the black hole, with some closer to the center and others further away. The vastness of the space and the dense arrangement of the stars create a sense of complexity and wonder.", "answer_id": "J4xirBuoZW2T3gRypvkRzp", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "39374539", "question_id": 7, "prompt": "Characterize the image using a well-detailed description.\n<image>", "text": "Assistant: The image features a large, elongated galaxy called the \"Pinwheel Galaxy\" (also known as Messier 101), which is situated in the constellation of Ursa Major. This galaxy has a prominent, long, and narrow appearance, making it an ideal target for astronomers and enthusiasts. It lies at a distance of about 21 million light-years from the Earth and is visible in the northern sky. The Pinwheel Galaxy is not only a beautiful sight in the sky but also a hub of star-formation activity.", "answer_id": "o2A5vNfRL6msze536yHkty", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "55350794", "question_id": 8, "prompt": "Break down the elements of the image in a detailed manner.\n<image>", "text": "Assistant: The image showcases a cluster of galaxies in the sky, as seen through a telescope. The view is a close-up and clear, providing detailed observations of the various stars and their movements. The cluster consists of at least three prominent galaxies, surrounded by several other smaller galaxies, creating a mesmerizing pattern in the sky. The position of these galaxies indicates that they are located far away from the Earth, providing a glimpse into the vastness of the universe.", "answer_id": "FLq5avvCqghTM6RjY56tWD", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "36255152", "question_id": 12, "prompt": "Analyze the image in a comprehensive and detailed manner.\n<image>", "text": "Assistant: The image showcases a deep-space view of the universe, where a large galaxy is visible in the forefront, occupying most of the frame. In the background, there is another distant galaxy, making it a two-galaxy scene. \n\nAsteroids can also be seen in the picture, with one located in the lower-left part of the image and another in the upper-right area. The vastness of space and the presence of these galaxies emphasize the scale and beauty of the cosmos.", "answer_id": "JCLFJvqcWfkiLKzeLHmdEz", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "40361164", "question_id": 16, "prompt": "Describe the following image concisely.\n<image>", "text": "Assistant: The image depicts a black sky filled with numerous bright stars scattered throughout the view. Among these stars, there is a cluster of stars located next to each other, forming a nebula. The stars vary in size and brightness, creating a visually engaging and dynamic scene. It appears to be an image taken by a telescope, showcasing the wonders of the galaxy.", "answer_id": "kERKYvoevgkYwBzSntHJSB", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "36250543", "question_id": 18, "prompt": "Offer a succinct explanation of the picture presented.\n<image>", "text": "Assistant: The image features a dark and dense sky with two bright stars shining close to each other. These two stars, possibly part of a galaxy, are the main focal points of the scene. In addition to the bright pair, there are several other stars scattered throughout the sky, making the scene even more captivating. The stars showcase the beauty and depth of the universe, creating an engaging and awe-inspiring view.", "answer_id": "MvXt6v2qBJTvxvF7Z3fstN", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "25811434", "question_id": 19, "prompt": "Summarize the visual content of the following image.\n<image>", "text": "Assistant: The image displays a telescopic view of a distant galaxy, captured from a deep space perspective. The galaxy appears to be large and has a prominent, small star-forming region near its center. This region contains numerous stars, some of which are visibly concentrated in the middle of the galaxy.\n\nIn addition to the stars, a faint, thin ring-like structure can be seen surrounding the central region. This structure may be a dust disk or a faint ring galaxy, adding to the complexity of the scene. The image also features a few other smaller stars scattered throughout the large galaxy.", "answer_id": "T7eXT9ZRApsWDbZQDMFRpd", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "34240953", "question_id": 20, "prompt": "Give a short and clear explanation of the subsequent image.\n<image>", "text": "Assistant: The image depicts a vast, crowded space filled with several galaxies, including our own Milky Way. The galaxies are distributed throughout the scene, with some located towards the center, while others are positioned closer to the edges. These galaxies vary in size and are placed in different orientations, creating a captivating display of star systems and celestial bodies. The image offers a glimpse into the vastness of the universe and the intricate web of galaxies within it.", "answer_id": "5GVDd9BL6VenPMJuqLvyPF", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "36222753", "question_id": 22, "prompt": "Present a compact description of the photo's key features.\n<image>", "text": "Assistant: The image features a bright, large, and luminous galaxy, possibly a spiral, captured in a deep space setting. The galaxy appears to be the main focus, occupying most of the frame. Surrounding the galaxy, there are several much smaller stars scattered across the sky. These stars are located at various distances from the main galaxy, creating a captivating contrast between the large galaxy and the surrounding smaller celestial bodies.", "answer_id": "Wia3BKTfQ9DhFRu777kfr7", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "32498813", "question_id": 23, "prompt": "Relay a brief, clear account of the picture shown.\n<image>", "text": "Assistant: The image features a very large, bright, and beautiful spiral galaxy with a yellowish hue, possibly resembling a lemon. The galaxy is the main focus of the picture, occupying most of the frame. It is situated against a dark, star-filled background, giving the impression of looking into the vastness of space.\n\nIn addition to the main spiral galaxy, there are several smaller galaxies scattered around the frame, further emphasizing the vastness of the space in which they all exist.", "answer_id": "D9nAFNHyYEacmmqJpGHwtm", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "21661664", "question_id": 25, "prompt": "Write a terse but informative summary of the following picture.\n<image>", "text": "Assistant: The image showcases a vast expanse of the universe with numerous stars scattered throughout the sky. Among these stars, there is a prominent and bright star with a surrounding nebula, making it a beautiful sight to behold. In addition to the bright star, there is a smaller and more distant galaxy visible in the picture. The large field of view captures the beauty of the cosmos, leaving the viewer in awe of the wonders of the universe.", "answer_id": "KpjQeEVGNhpNMsmfKEiBZz", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "20923069", "question_id": 28, "prompt": "Provide a detailed description of the given image.\n<image>", "text": "Assistant: The image displays a deep space scene with a large cluster of stars. The view is captured from a distance, showcasing the stars and their orbits. There are several bright and distinct stars in the scene, some of which have companions. The stars are positioned at various distances and angles, creating a beautiful and intricate pattern. The view truly emphasizes the vastness and beauty of the universe.", "answer_id": "D4FKiPDAAWgs6ALAGHzGzy", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "51727000", "question_id": 29, "prompt": "Give an elaborate explanation of the image you see.\n<image>", "text": "Assistant: The image features a black sky with many distant stars scattered throughout. Among these stars, there is a noticeable black hole in the middle, drawing attention due to its dark presence. The other stars are positioned at various distances from the black hole, creating a captivating scene of interstellar space.", "answer_id": "XVj4J4V9GTDfNu92MFyQXM", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "16215408", "question_id": 30, "prompt": "Share a comprehensive rundown of the presented image.\n<image>", "text": "Assistant: The image features a sky full of stars, with a bright star and a red object standing out in the darkness. The bright star, which could be a planet or another star, is located towards the center of the image, capturing attention. The red object is closer to the bottom right corner of the sky.\n\nIn the sky, there are also several other stars, including those that form a line ranging from the top left corner to the bottom right corner. These stars vary in brightness, creating a beautiful and lively starry sky.", "answer_id": "XYo5UtwqTjxbevLcPxtEQL", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "36238970", "question_id": 31, "prompt": "Offer a thorough analysis of the image.\n<image>", "text": "Assistant: This image features a sky full of stars, with two of them being particularly close to each other. These two stars are in the process of merging, creating a beautiful sight in the night sky. The other stars in the picture showcase various sizes and distances, adding depth to the overall scene. It is a captivating view of the cosmos, showcasing the beauty of the universe and the ongoing process of star formation.", "answer_id": "LXSEaTSrthvq6BBnmyNTgV", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "34060815", "question_id": 32, "prompt": "Explain the various aspects of the image before you.\n<image>", "text": "Assistant: The image showcases a deep space view of the sky, where several galaxies are visible. In the center of the scene, there is a large and prominent galaxy that dominates the view. Surrounding this central galaxy, multiple smaller galaxies can be seen at various distances, creating a captivating cosmic scene. The galaxies appear to be aligned in a way that suggests they form a larger structure, providing a sense of depth and scale to the image.", "answer_id": "PU4nRuyGaCnTK9cYiPTgxr", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "21660643", "question_id": 34, "prompt": "Characterize the image using a well-detailed description.\n<image>", "text": "Assistant: The image is a beautiful, dark night sky filled with many bright and colorful stars. Among these stars, there is a notable cluster of stars located in the middle of the sky, capturing attention due to their proximity and brightness. These stars are surrounded by other, smaller stars, creating a visually stunning scene. The sky is so full of stars that they seem to go on forever, making it a truly immersive and captivating view.", "answer_id": "DiqJwAYSeZmC6j44YYsokw", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "21661614", "question_id": 36, "prompt": "Walk through the important details of the image.\n<image>", "text": "Assistant: The image features a large nebula, glowing with a bright blue light, possibly a supernova that is just minutes old. The nebula appears to be expanding, with the bright light spreading throughout the sky. In the background, there are several stars, including one that has a blue light. The picture has a deep black background, emphasizing the vibrant colors and light in the nebula.", "answer_id": "fQZqS7qVvBxr9s36nAhofx", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "28483442", "question_id": 37, "prompt": "Portray the image with a rich, descriptive narrative.\n<image>", "text": "Assistant: The image features a beautiful scene of the night sky with a bright, large star cluster visible. The stars are scattered throughout the sky, creating a sense of depth and space. The brightest star in the group shines with a blue hue, capturing attention and adding a touch of color to the mostly dark sky. \n\nAs the camera looks into the sky, it becomes apparent that there are many more stars in the distant universe, waiting to be explored and admired. The vastness of the stars and their arrangement creates a picturesque and engaging space scene for viewers to enjoy.", "answer_id": "7ctUWoArsrNkVAHLeTocoq", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "55351013", "question_id": 40, "prompt": "Illustrate the image through a descriptive explanation.\n<image>", "text": "Assistant: The image features a vast, seemingly endless sky filled with an impressive array of brightly shining stars. Some of these stars appear to be in clusters or groups, creating a captivating display. The sky is so full of stars that they seem to go on forever, offering a sense of limitless space and celestial beauty. The stars vary in size and brightness, with some standing out more prominently against the dark backdrop while others are subtler in their glow. Overall, this stunning celestial scene evokes a sense of wonder and appreciation for the vastness of the universe.", "answer_id": "A9j7RdPTwd27ncqws4c9Bp", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "21668459", "question_id": 43, "prompt": "Describe the following image concisely.\n<image>", "text": "Assistant: The image features a spiral galaxy, captured in a black and white photograph displaying a dark sky. The galaxy consists of a circular central region with a swirling pattern of stars and gases, creating a stunning visual effect. The view of the galaxy is captured from the side, allowing us to see the full extent of the swirling pattern, which spans from the center to the outer edges of the image. The large number of stars in the galaxy adds depth and complexity to the scene.", "answer_id": "ZMJwXUbBaAx8EpgZi3BEe8", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "40355779", "question_id": 44, "prompt": "Provide a brief description of the given image.\n<image>", "text": "Assistant: The image features a deep sky view captured by a telescope, showcasing a distant galaxy with a bright star forming in the center. The galaxy consists of a large central disc surrounded by two smaller discs, giving it a spiral shape reminiscent of the Milky Way. \n\nIn addition to the main galaxy, there are several other stars scattered throughout the image, some of which appear to be forming into a second galaxy. The entire scene is set against a dark, star-filled sky, creating a sense of depth and wonder at the vastness of the universe.", "answer_id": "SzDD3dppxyiFopZunaYdaF", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "28490503", "question_id": 45, "prompt": "Offer a succinct explanation of the picture presented.\n<image>", "text": "Assistant: In the image, there is a sky full of stars, with one particularly bright star in the foreground, surrounded by several other stars. The bright star appears to be a supernova, capturing attention and dominating the scene. Among the various stars, there is also an galaxy visible, adding to the richness of the celestial display. The image has a slight blurry effect, emphasizing the depth and vastness of the space in which these stars are scattered.", "answer_id": "gVMTcoFSpCMdJJkvXHrdYa", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "36213141", "question_id": 46, "prompt": "Summarize the visual content of the following image.\n<image>", "text": "Assistant: The image features a full view of a spiral galaxy, captured in a close-up shot. The galaxy appears to be faint and is positioned against a dark night sky. The Hubble Space Telescope has likely taken this image of the distant star, showcasing the impressive capabilities of the device. The galaxy consists of stars and dust, creating a beautiful and intricate view that covers the entire frame.", "answer_id": "A8KL8Ksb6nUgMe3ymgNehr", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "21665278", "question_id": 47, "prompt": "Give a short and clear explanation of the subsequent image.\n<image>", "text": "Assistant: The image depicts a dark space with a large, prominent galaxy in the center, surrounded by smaller galaxies. The main galaxy appears as a circular, celestial object with a bright, spherical core, which could possibly be a black hole. \n\nSeveral smaller galaxies are scattered around the main galaxy, creating a captivating scene of interstellar space. These smaller galaxies appear as faint, irregularly shaped objects in comparison to the dominant galaxy in the scene.", "answer_id": "RhhR4yVo65u8qXTCwjpn2U", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
+{"image_id": "20926746", "question_id": 48, "prompt": "Share a concise interpretation of the image provided.\n<image>", "text": "Assistant: The image features a crowded field of stars, including a dim star cluster with a total of nine stars in the view. Among these, there is a large star occupying a significant portion of the field, appearing to be an active galactic core. The other smaller stars are dispersed throughout the scene, with some located close to the large star, and others further away. The field of stars is captured in a black sky, which sets the contrasting backdrop for the bright celestial objects.", "answer_id": "U3XfaFcKSFB7g3Wnyoq3MH", "model_id": "/gpfswork/rech/owt/commun/LLaVA-13B-v0", "metadata": {}}
diff --git a/scripts/evaluation/summarize_gpt_review.py b/scripts/evaluation/summarize_gpt_review.py
new file mode 100644
index 0000000..6aaf30d
--- /dev/null
+++ b/scripts/evaluation/summarize_gpt_review.py
@@ -0,0 +1,34 @@
+import json
+import os
+from collections import defaultdict
+
+import numpy as np
+
+import argparse
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='ChatGPT-based QA evaluation.')
+    parser.add_argument('-d', '--dir')
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    review_files = [x for x in os.listdir(args.dir) if x.endswith('.jsonl') and (x.startswith('gpt4_text') or x.startswith('reviews_'))]
+
+    for review_file in sorted(review_files):
+        print("running")
+        config = review_file.replace('gpt4_text_', '').replace('.jsonl', '')
+        scores = defaultdict(list)
+        print(f'GPT-4 vs. {config}')
+        with open(os.path.join(args.dir, review_file)) as f:
+            for review_str in f:
+                review = json.loads(review_str)
+                scores[review['category']].append(review['tuple'])
+                scores['all'].append(review['tuple'])
+        
+        for k, v in scores.items():
+            stats = np.asarray(v).mean(0).tolist()
+            stats = [round(x, 3) for x in stats]
+            print(k, stats, round(stats[1]/stats[0]*100, 1))
+        print('=================================')