From 3e594d22ac3571299dfcc875bf516d44de5a2dd8 Mon Sep 17 00:00:00 2001 From: Vanshita Gupta Date: Sat, 9 Dec 2023 11:49:15 -0500 Subject: [PATCH] added mcq data preparation and base model evaluation scripts --- notebooks/Buzfeed_friends_mcq.ipynb | 2333 +++++++++++++++++++++++++++ notebooks/query_llm_test.ipynb | 978 ++++++++++- 2 files changed, 3265 insertions(+), 46 deletions(-) create mode 100644 notebooks/Buzfeed_friends_mcq.ipynb diff --git a/notebooks/Buzfeed_friends_mcq.ipynb b/notebooks/Buzfeed_friends_mcq.ipynb new file mode 100644 index 0000000..b2bf28c --- /dev/null +++ b/notebooks/Buzfeed_friends_mcq.ipynb @@ -0,0 +1,2333 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "pd.set_option('display.max_columns', None)\n", + "import numpy as np\n", + "from datetime import datetime\n", + "import requests\n", + "from bs4 import BeautifulSoup\n", + "from selenium import webdriver\n", + "import json\n", + "import html2text\n", + "import re\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 487, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "input_folder='/Users/zoey/Downloads/Fraud detection '\n", + "output_file = '/Users/zoey/Downloads/Fraud detection /combined.json'\n", + "def combine_json_files(input_folder, output_file):\n", + " list_json = []\n", + "\n", + " # Iterate over each file in the input folder\n", + " for filename in os.listdir(input_folder):\n", + " \n", + " if filename.endswith(\".json\"):\n", + " file_path = os.path.join(input_folder, filename)\n", + " \n", + " # Read and parse the content of each JSON file\n", + " with open(file_path, \"r\") as file:\n", + " data = json.load(file)\n", + " list_json.append(data)\n", + " \n", + " combined_data={\"json_body\":list_json}\n", + "\n", + " # Write the combined data to a new JSON file\n", + " with open(output_file, \"w\") as output_file:\n", + " json.dump(combined_data, output_file, indent=2)\n", + "\n", + "\n", + "combine_json_files(input_folder, output_file)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"data/eval_data/combined.json\",'r') as f:\n", + " json_data = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Question: What is the name of Rachel's old sorority sister?\n", + "Options: ['Melissa Warburton', 'Collete Warburton', 'Kat Warburton', 'Evelina Warburton']\n", + "Correct Answer: Melissa Warburton\n", + "---\n", + "Question: What is the name of Ross & Monica's cousin?\n", + "Options: ['Melinda', 'Cassie', 'Andi', 'Raquel']\n", + "Correct Answer: Cassie\n", + "---\n", + "Question: What is the name of the guy Rachel bullied in high school?\n", + "Options: ['Brad Colbert', 'Stephen Colbert', 'Will Colbert', 'Ben Colbert']\n", + "Correct Answer: Will Colbert\n", + "---\n", + "Question: What is the name of Joey's Australian roommate?\n", + "Options: ['Cindy', 'Melissa', 'Janine', 'Amy']\n", + "Correct Answer: Janine\n", + "---\n", + "Question: What is the name of this professor who dates both Joey and Ross?\n", + "Options: ['Carla', 'Charlie', 'Jenny', 'Terry']\n", + "Correct Answer: Charlie\n", + "---\n", + "Question: What is the name of Chandler's coworker from Tulsa?\n", + "Options: ['Susan', 'Wendy', 'Ricki', 'Sam']\n", + "Correct Answer: Wendy\n", + "---\n", + "Question: What is the name of the person who was directing a play that Joey auditioned for?\n", + "Options: ['Benny Hayes', 'Joseph Hayes', 'Leonard Hayes', 'Ronnie Hayes']\n", + "Correct Answer: Leonard Hayes\n", + "---\n", + "Question: What is the name of the cop that Phoebe dates after finding his wallet?\n", + "Options: ['Dirk', 'Gary', 'Fred', 'Bill']\n", + "Correct Answer: Gary\n", + "---\n", + "Question: What is the name of Chandler's coworker (who keeps calling Chandler by the name \"Toby\"?)\n", + "Options: ['Bob', 'Toby', 'Jack', 'John']\n", + "Correct Answer: Bob\n", + "---\n", + "Question: What is the name of the guy who Phoebe thinks is Monica's soulmate?\n", + "Options: ['Frank', 'Don', 'Billy', 'Sammy']\n", + "Correct Answer: Don\n", + "---\n", + "Question: How many times did Ross get divorced?\n", + "Options: ['1', '2', '3', '4']\n", + "Correct Answer: 3\n", + "---\n", + "Question: What is Chandler's middle name?\n", + "Options: ['Malcolm', 'Michael', 'Muriel', 'Morgan']\n", + "Correct Answer: Muriel\n", + "---\n", + "Question: How many sisters does Joey have?\n", + "Options: ['4', '6', '7', 'He only has brothers']\n", + "Correct Answer: 7\n", + "---\n", + "Question: Rachel got a job with which company in Paris?\n", + "Options: ['Gucci', 'Louis Vuitton', 'Prada', 'Chanel']\n", + "Correct Answer: Louis Vuitton\n", + "---\n", + "Question: What is the name of Ross and Monica's parents?\n", + "Options: ['Julia and Josh', 'Jane and Jacob', 'Josie and Joseph', 'Judy and Jack']\n", + "Correct Answer: Judy and Jack\n", + "---\n", + "Question: Which one of the main characters kissed everybody?\n", + "Options: ['Joey', 'Monica', 'Rachel', 'Phoebe']\n", + "Correct Answer: Rachel\n", + "---\n", + "Question: Who pees on Monica after she's stung by a jellyfish?\n", + "Options: ['Joey', 'Chandler', 'Phoebe', 'Ross']\n", + "Correct Answer: Chandler\n", + "---\n", + "Question: How many siblings does Phoebe have?\n", + "Options: ['A twin', 'A half brother', 'A twin and half brother', \"She's a triplet\"]\n", + "Correct Answer: A twin and half brother\n", + "---\n", + "Question: Who was Joey's imaginary friend and what was his profession?\n", + "Options: ['Mooney, a space knight', 'Maurice, a space cowboy', 'Buddy, a farmer', \"Lulu, Santa's elf\"]\n", + "Correct Answer: Maurice, a space cowboy\n", + "---\n", + "Question: What was the name of Ross and Chandler's band in college?\n", + "Options: ['New Rule', 'Savage Groove', 'Chaos in Control', 'Way No Way']\n", + "Correct Answer: Way No Way\n", + "---\n", + "Question: What's Ross's monkey named?\n", + "Options: ['Marcel', 'Jaxon', 'Ben', 'Ross Jr.']\n", + "Correct Answer: Marcel\n", + "---\n", + "Question: What language does Phoebe attempt to teach Joey?\n", + "Options: ['Spanish', 'French', 'Italian', 'Greek']\n", + "Correct Answer: French\n", + "---\n", + "Question: What part of the body did Phoebe find in a soda can?\n", + "Options: ['A toe', 'A tongue', 'A fingernail', 'A thumb']\n", + "Correct Answer: A thumb\n", + "---\n", + "Question: How many roses did Ross send to Emily?\n", + "Options: ['80', '66', '72', '100']\n", + "Correct Answer: 72\n", + "---\n", + "Question: What three things is Ross allergic to?\n", + "Options: ['Shellfish, peanuts, and kiwi', 'Soy, bananas, and mint', 'Shrimp, ginger, and pears', 'Fish, mangos, and milk']\n", + "Correct Answer: Shellfish, peanuts, and kiwi\n", + "---\n", + "Question: How many times was Ross legally divorced? \n", + "\n", + "Options: ['Twice', 'Three times', 'Five times', 'Six times']\n", + "Correct Answer: Three times\n", + "---\n", + "Question: Where did Carol first meet Susan? \n", + "\n", + "Options: ['In college', 'At work', 'At the gym', 'At Central Perk']\n", + "Correct Answer: At the gym\n", + "---\n", + "Question: How did Susan and Ross come up with Ben’s name?\n", + "Options: [\"It was the doctor's name \\n\", 'They both had uncles named Ben \\n', \"It's the name of their favorite actor\", \"It was on the janitor's name tag\"]\n", + "Correct Answer: It was on the janitor's name tag\n", + "---\n", + "Question: What were Ben's first words? \n", + "\n", + "Options: ['Hi', 'Bye', 'Mom', 'Dumb']\n", + "Correct Answer: Hi\n", + "---\n", + "Question: How long did Ross and Emily date before they got engaged? \n", + "\n", + "Options: ['14 days', '6 weeks \\n', 'A year \\n', '3 months']\n", + "Correct Answer: 6 weeks \n", + "\n", + "---\n", + "Question: Rachel left Ross a drunken voicemail confessing her feelings after Ross and Julie were about to take what big relationship step?\n", + "Options: ['Move in together \\n', 'Get a cat together \\n', 'Elope \\n', 'Meet Julie’s parents \\n']\n", + "Correct Answer: Get a cat together \n", + "\n", + "---\n", + "Question: What holiday did Ross and his girlfriend Mona break up on? \n", + "\n", + "Options: [\"Valentine's Day \\n\", 'Thanksgiving \\n', 'Christmas Day', \"New Year's Eve\"]\n", + "Correct Answer: Valentine's Day \n", + "\n", + "---\n", + "Question: Which of the following ingredients were not in Rachel's iconic Thanksgiving trifle?\n", + "Options: ['Ladyfingers and jam \\n', 'Beef with peas and onions \\n', 'Raisins and walnuts \\n', 'Bananas and custard \\n']\n", + "Correct Answer: Raisins and walnuts \n", + "\n", + "---\n", + "Question: Who did Rachel discover her ex-fiancé Barry was cheating on her with? \n", + "\n", + "Options: ['Her best friend', 'Her sister', 'His coworker', 'Her college roommate']\n", + "Correct Answer: Her best friend\n", + "---\n", + "Question: What did Rachel accidentally do to Mr. Zelner during her job interview at Ralph Lauren? \n", + " \n", + "\n", + "Options: ['Punch him', 'Kiss him', 'Rip his pants', 'Spill coffee on him']\n", + "Correct Answer: Kiss him\n", + "---\n", + "Question: Joey and Phoebe found Tag’s sweater in Rachel’s apartment, assuming he’s her baby’s father. What color is the sweater?\n", + "Options: ['Red', 'Blue', 'Green', 'Black']\n", + "Correct Answer: Red\n", + "---\n", + "Question: Rachel had a one-night stand with Paolo after Ross starts dating who?\n", + "Options: ['Julie', 'Mona', 'Emily', 'Jill']\n", + "Correct Answer: Julie\n", + "---\n", + "Question: Which one of Rachel’s exes came back in Season 10 to offer her a job in Paris? \n", + "\n", + "Options: ['Paul Stevens \\n', 'Pete Becker \\n', 'Richard Burke \\n', 'Mark Robinson']\n", + "Correct Answer: Mark Robinson\n", + "---\n", + "Question: What was Gunther’s job before he worked at Central Perk? \n", + "\n", + "Options: ['A history professor \\n', 'A soap opera actor \\n', 'A marketing executive \\n', 'A fashion designer \\n']\n", + "Correct Answer: A soap opera actor \n", + "\n", + "---\n", + "Question: How did Monica and Rachel convince Mr. Treeger not to kick them out for illegally subletting their apartment?\n", + "Options: ['Rachel goes on a date with him \\n', 'Joey does ballroom dancing with him \\n \\n', 'Monica cleans his filthy apartment \\n', 'Rachel and Monica bribe him with $4,000 \\n']\n", + "Correct Answer: Joey does ballroom dancing with him \n", + " \n", + "\n", + "---\n", + "Question: What was the name of the first restaurant Monica was head chef at? \n", + "\n", + "Options: ['Alessandro’s \\n', 'Moondance Diner \\n', 'Iridium', 'Javu']\n", + "Correct Answer: Alessandro’s \n", + "\n", + "---\n", + "Question: What job did Monica’s older boyfriend Richard Burke have? \n", + "\n", + "Options: ['Dentist', 'Sous chef', 'Stockbroker \\n', 'Ophthalmologist']\n", + "Correct Answer: Ophthalmologist\n", + "---\n", + "Question: What did Pete Becker do for Monica on their first date? \n", + "\n", + "Options: ['He buys her a restaurant \\n', 'He buys her a $20,000 necklace \\n', 'He flies them to Rome for Pizza \\n', 'He introduces her to Jon Bon Jovi \\n']\n", + "Correct Answer: He flies them to Rome for Pizza \n", + "\n", + "---\n", + "Question: Monica and Chandler adopted Erica's kid(s). How many kid(s) did she have? \n", + "\n", + "Options: ['Just one', 'Twins', 'Triplets']\n", + "Correct Answer: Twins\n", + "---\n", + "Question: Which of the following Chandler “facts” is false? \n", + "\n", + "Options: ['He has a third nipple \\n', 'He is missing a pinky toe \\n', 'He once won a Vanilla Ice lookalike contest \\n', 'He dated Phoebe as a kid']\n", + "Correct Answer: He dated Phoebe as a kid\n", + "---\n", + "Question: How many times did Chandler and Janice break up during the entirety of _Friends_?\n", + "Options: ['Just once', '2 times', '5 times', '8 times']\n", + "Correct Answer: 5 times\n", + "---\n", + "Question: How did Chandler finally get his bad roommate Eddie to move out? \n", + "\n", + "Options: ['He caught Eddie watching him sleep \\n', 'He changed the locks and pretended not to know Eddie \\n', 'He ate all of Eddie’s dehydrated vegetables \\n', 'He released a monkey in Eddie’s room \\n']\n", + "Correct Answer: He changed the locks and pretended not to know Eddie \n", + "\n", + "---\n", + "Question: Chandler bought his girlfriend Kathy a first edition of her favorite childhood book for her birthday — what is it?\n", + "Options: ['_The Velveteen Rabbit_ \\n', '_The Little Prince_ \\n', '_The Giving Tree_ \\n', '_Where the Wild Things Are_ \\n']\n", + "Correct Answer: _The Velveteen Rabbit_ \n", + "\n", + "---\n", + "Question: Who was briefly roommates with Australian dancer Janine Lacroix? \n", + "\n", + "Options: ['Chandler', 'Phoebe', 'Ross', 'Joey']\n", + "Correct Answer: Joey\n", + "---\n", + "Question: How many sisters did Joey Tribbiani have? \n", + "\n", + "Options: ['3', '5', '7', '9']\n", + "Correct Answer: 7\n", + "---\n", + "Question: Before they were friends, who did Phoebe mug as a kid? \n", + "\n", + "Options: ['Chandler', 'Joey', 'Monica', 'Ross']\n", + "Correct Answer: Ross\n", + "---\n", + "Question: Which one of the guys did Ursula go out with? \n", + "\n", + "Options: ['Joey', 'Chandler', 'Ross']\n", + "Correct Answer: Joey\n", + "---\n", + "Question: How did Phoebe’s half-brother Frank meet his fiancé Alice? \n", + "\n", + "Options: ['She was his favorite grocery store clerk', 'She was his teacher', 'She was his ER nurse', 'She was his coworkers mom']\n", + "Correct Answer: She was his teacher\n", + "---\n", + "Question: In “The One Where Estelle Dies,” who pretended to be Estelle when Joey called? \n", + "\n", + "Options: ['Phoebe', 'Rachel', 'Chandler', 'Monica']\n", + "Correct Answer: Phoebe\n", + "---\n", + "Question: Which of the following was not the name of one of Alice and Frank Jr.’s kids? \n", + "\n", + "Options: ['Leslie', 'Ursula', 'Chandler', 'Frank Jr. Jr.']\n", + "Correct Answer: Ursula\n", + "---\n", + "Question: How did Phoebe meet her boyfriend Gary? \n", + "\n", + "Options: ['He was a massage client \\n', 'He left her a large tip at Central Perk \\n', 'She tried to trick him into thinking she was a cop \\n', 'She and Monica catered his divorce party \\n']\n", + "Correct Answer: She tried to trick him into thinking she was a cop \n", + "\n", + "---\n", + "Question: Phoebe’s love interest David couldn’t be with her because he had a grant to study where? \n", + "\n", + "Options: ['Yemen', 'Minsk', 'Aruba', 'Athens']\n", + "Correct Answer: Minsk\n", + "---\n", + "Question: How did Mike Hannigan originally want to propose to Phoebe? \n", + "\n", + "Options: ['During a performance at Central Perk', 'At a Knicks game \\n', \"On New Year's Eve in Times Square\", 'By pretending to be a massage client']\n", + "Correct Answer: At a Knicks game \n", + "\n", + "---\n", + "Question: Who did Mr. Heckles leave his apartment to when he died? \n", + "\n", + "Options: ['Joey and Chandler \\n', 'Monica and Ross \\n', 'Monica and Rachel \\n', 'Phoebe and Rachel \\n']\n", + "Correct Answer: Monica and Rachel \n", + "\n", + "---\n", + "Question: Jasmine knew about Ross cheating on Rachel because Isaac from the Xerox store is her brother. Who did she tell?\n", + "Options: ['Her coworker, Phoebe', 'Her ex-boyfriend, Joey', \"Her friend and Rachel's coworker, Tag\", 'Her roommate, Gunther']\n", + "Correct Answer: Her roommate, Gunther\n", + "---\n", + "Question: This is Joshua Burgin. Who had a crush on him?\n", + "Options: ['Monica', 'Rachel', 'Phoebe']\n", + "Correct Answer: Rachel\n", + "---\n", + "Question: Who did Charlie Wheeler date?\n", + "Options: ['Chandler and Joey', 'Joey and Ross', 'Chandler and Ross', 'Chandler, Joey, and Ross']\n", + "Correct Answer: Joey and Ross\n", + "---\n", + "Question: Dr. Long appeared in 5 episodes of _Friends_ as Rachel’s obstetrician during which season? \n", + "\n", + "Options: ['7', '8', '9', '10']\n", + "Correct Answer: 8\n", + "---\n", + "Question: Do you remember her name?\n", + "Options: ['Sarah', 'Elizabeth', 'Amanda', 'Samantha']\n", + "Correct Answer: Elizabeth\n", + "---\n", + "Question: Who got hit with a tranquilizer dart while trying to save Ross's monkey Marcel? \n", + "\n", + "Options: ['Rachel', 'Monica', 'Joey', 'Phoebe']\n", + "Correct Answer: Phoebe\n", + "---\n", + "Question: Which one of Rachel’s sisters was played by Reese Witherspoon? \n", + "\n", + "Options: ['Amy', 'Jill']\n", + "Correct Answer: Jill\n", + "---\n", + "Question: What did Rachel and her other sister, Amy, fight about on Thanksgiving? \n", + "\n", + "Options: ['Who would get Emma if Rachel and Ross die \\n', 'Which sister stole the other’s boyfriend in college \\n', \"Who is actually mom's favorite child\", \"Who inherits dad's house\"]\n", + "Correct Answer: Who would get Emma if Rachel and Ross die \n", + "\n", + "---\n", + "Question: Whose mom is this?\n", + "Options: ['Ross and Monica', 'Rachel', 'Joey', 'Chandler']\n", + "Correct Answer: Ross and Monica\n", + "---\n", + "Question: What does Monica's dad, Jack, give her because her childhood belongings were ruined in a flood? \n", + "\n", + "Options: ['Their childhood home \\n', 'Her own restaurant \\n', 'His Porsche \\n', '$10,000 \\n']\n", + "Correct Answer: His Porsche \n", + "\n", + "---\n", + "Question: Which member of the _Friends_ crew kissed Rachel’s mom? \n", + "\n", + "Options: ['Chandler', 'Ross', 'Phoebe', 'Joey']\n", + "Correct Answer: Joey\n", + "---\n", + "Question: What job did Rachel's dad have?\n", + "Options: ['Professor', 'Attorney', 'Doctor', 'TV personality']\n", + "Correct Answer: Doctor\n", + "---\n", + "Question: What drag name did Chandler's dad go by?\n", + "Options: ['Princess Consuela Bananahammock \\n', 'Helena Handbasket \\n', 'Farrah Moan \\n', 'Izzy Rich \\n']\n", + "Correct Answer: Helena Handbasket \n", + "\n", + "---\n", + "Question: What is the name of Phoebe's actual birth mother?\n", + "Options: ['Phoebe', 'Gloria', 'Meredith', 'Sky']\n", + "Correct Answer: Phoebe\n", + "---\n", + "Question: Which one of Joey's sisters accidentally got pregnant in Season 8?\n", + "Options: ['Gina', 'Mary-Therese', 'Dina', 'Mary-Angela']\n", + "Correct Answer: Dina\n", + "---\n", + "Question: And finally, what did Chandler’s mom do for a living? \n", + "\n", + "Options: ['Organize children’s beauty pageants \\n', 'Write erotic novels \\n', 'Make animal scented candles \\n', 'International pop star \\n']\n", + "Correct Answer: Write erotic novels \n", + "\n", + "---\n", + "Question: Who surprise proposes to their fiancé?\n", + "Options: ['Rachel', 'Monica', 'Phoebe', 'Judy']\n", + "Correct Answer: Monica\n", + "---\n", + "Question: How many sisters does Joey have?\n", + "Options: ['6', '8', '9', '7']\n", + "Correct Answer: 7\n", + "---\n", + "Question: Who ends up marrying Phoebe?\n", + "Options: ['Joey', 'Chandler', 'Mike', 'Gunther']\n", + "Correct Answer: Mike\n", + "---\n", + "Question: What is Rachel's middle name ?\n", + "Options: ['Juliet', 'Karen', 'Monica', 'Lily']\n", + "Correct Answer: Karen\n", + "---\n", + "Question: What is Phoebe's famous fake name?\n", + "Options: ['Charlotte Lily', 'Jenny hannigan', 'Regina Phalange', 'Tayla George']\n", + "Correct Answer: Regina Phalange\n", + "---\n", + "Question: What is Chandler's middle name?\n", + "Options: ['Muriel', 'Jake', 'Jacob', 'Barry']\n", + "Correct Answer: Muriel\n", + "---\n", + "Question: Who had a huge crush on Rachel through all of the seasons, besides Ross?\n", + "Options: ['Joey', 'Chandler', 'Mike', 'Gunther']\n", + "Correct Answer: Gunther\n", + "---\n", + "Question: Who was Ross's second wife?\n", + "Options: ['Emily', 'Carol', 'Linda', 'Jane']\n", + "Correct Answer: Emily\n", + "---\n", + "Question: Who adopts twins in the final season?\n", + "Options: ['Rachel and Ross', 'Rachel and Gunther', 'Monica and Chandler', 'Phoebe and Mike']\n", + "Correct Answer: Monica and Chandler\n", + "---\n", + "Question: Who gives birth to triplets for their brother?\n", + "Options: ['Phoebe', 'Rachel', 'Judy', 'Jill']\n", + "Correct Answer: Phoebe\n", + "---\n", + "Question: What's Chandler's middle name? \n", + "\n", + "Options: ['Muriel \\n', 'Michael \\n', 'Mike', 'Mark']\n", + "Correct Answer: Muriel \n", + "\n", + "---\n", + "Question: What are the names of Monica's and Chandler's children? \n", + "\n", + "Options: ['Jack and Ella', 'Erica and Jack', 'Josh and Emma', 'Emily and Josh \\n']\n", + "Correct Answer: Erica and Jack\n", + "---\n", + "Question: What is Ross Geller's job?\n", + "Options: ['Architect', 'Paleontologist', 'Artist \\n', 'Marine Biologist \\n']\n", + "Correct Answer: Paleontologist\n", + "---\n", + "Question: What was the name of Phoebe's twin sister?\n", + "Options: ['Jill', 'Amy', 'Ursula', 'Erica']\n", + "Correct Answer: Ursula\n", + "---\n", + "Question: What was the name of Rachel and Ross's daughter?\n", + "Options: ['Emily', 'Emma', 'Eleanor', 'Ella \\n']\n", + "Correct Answer: Emma\n", + "---\n", + "Question: How many sisters does Joey have? \n", + "\n", + "Options: ['6 \\n', '5', '8', '7 \\n']\n", + "Correct Answer: 7 \n", + "\n", + "---\n", + "Question: What was the name of the cat Rachel bought for $1000?\n", + "Options: ['Mrs. Whiskerson \\n', 'Whiskers \\n', 'Wally \\n', 'Mr. Mittens \\n']\n", + "Correct Answer: Mrs. Whiskerson \n", + "\n", + "---\n", + "Question: What's the nickname of the person the friends spied on from Monica's apartment? \n", + "\n", + "Options: ['Naked dude', 'The messy girl', 'Ugly naked guy', 'Yeti']\n", + "Correct Answer: Ugly naked guy\n", + "---\n", + "Question: What was the name of the fashion company Rachel was going to Paris for? \n", + "\n", + "Options: ['Gucci', 'Ralph Lauren', 'Louis Vuitton', 'Versace \\n']\n", + "Correct Answer: Louis Vuitton\n", + "---\n", + "Question: What were the names of Frank's triplets?\n", + "Options: ['Chandler, Leslie, and Phoebe', 'Chandler, Linda, and Frank Jr. Jr. \\n', 'Chandler, Leslie, and Frank Jr. Jr.', 'Chandler, Lexi, and Frank Jr. \\n']\n", + "Correct Answer: Chandler, Leslie, and Frank Jr. Jr.\n", + "---\n", + "Question: Which character is afraid of dogs?\n", + "Options: ['Phoebe', 'Chandler', 'Rachel', 'Joey']\n", + "Correct Answer: Chandler\n", + "---\n", + "Question: Which character kissed Chandler's mom?\n", + "Options: ['Joey', 'Ross', 'Gunther', 'Mike']\n", + "Correct Answer: Ross\n", + "---\n", + "Question: Who is Ken Adams?\n", + "Options: ['Ross', 'Gunther', 'Joey', 'Chandler']\n", + "Correct Answer: Joey\n", + "---\n", + "Question: Which character claimed to have skipped fourth grade?\n", + "Options: ['Joey', 'Monica', 'Chandler', 'Ross']\n", + "Correct Answer: Ross\n", + "---\n", + "Question: Which character was shot in the butt by a tranquilizer dart?\n", + "Options: ['Joey', 'Rachel', 'Phoebe', 'Ross']\n", + "Correct Answer: Phoebe\n", + "---\n", + "Question: Who resolves to stop gossiping as their New Year's resolution?\n", + "Options: ['Monica', 'Ross', 'Rachel', 'Joey']\n", + "Correct Answer: Rachel\n", + "---\n", + "Question: Which friend has never kissed Joey?\n", + "Options: ['Rachel', 'Ross', 'Monica', 'Phoebe']\n", + "Correct Answer: Monica\n", + "---\n", + "Question: Who thought that when you get married in Las Vegas you were ONLY legally married in Las Vegas?\n", + "Options: ['Rachel', 'Joey', 'Ross', 'Phoebe']\n", + "Correct Answer: Phoebe\n", + "---\n", + "Question: Who kissed Phoebe on her 30th birthday? \n", + "\n", + "Options: ['Ross', 'Rachel', 'Joey', 'Chandler']\n", + "Correct Answer: Joey\n", + "---\n", + "Question: Who teaches Ben all the practical jokes?\n", + "Options: ['Phoebe \\n', 'Joey', 'Chandler', 'Rachel']\n", + "Correct Answer: Rachel\n", + "---\n", + "Question: Which friend won a Vanilla Ice look-a-like contest? \n", + "\n", + "Options: ['Ross', 'Joey', 'Chandler', 'Phoebe']\n", + "Correct Answer: Chandler\n", + "---\n", + "Question: Who introduced Phoebe and Mike?\n", + "Options: ['Rachel', 'Ross', 'Joey', 'Chandler']\n", + "Correct Answer: Joey\n", + "---\n", + "Question: Who said the very last line in _Friends_?\n", + "Options: ['Monica', 'Chandler', 'Rachel', 'Ross', 'Phoebe', 'Joey']\n", + "Correct Answer: Chandler\n", + "---\n", + "Question: Does Rachel give birth to Emma before or after Phoebe gives birth?\n", + "Options: ['Before', 'After']\n", + "Correct Answer: After\n", + "---\n", + "Question: Does Monica begin dating Pete before or after she starts dating Richard?\n", + "Options: ['Before', 'After']\n", + "Correct Answer: After\n", + "---\n", + "Question: Do Monica and Chandler get married before or after Phoebe and Mike?\n", + "Options: ['Before', 'After']\n", + "Correct Answer: Before\n", + "---\n", + "Question: Do we meet Rachel's sister Jill before or after we meet Rachel's other sister, Amy?\n", + "Options: ['Before', 'After']\n", + "Correct Answer: Before\n", + "---\n", + "Question: Do the friends go on holiday to the beach before or after they go to London?\n", + "Options: ['Before', 'After']\n", + "Correct Answer: Before\n", + "---\n", + "Question: Do we find out about Ross' ~thing~ before or after Monica tells everybody about Chandler's third nipple?\n", + "Options: ['Before', 'After']\n", + "Correct Answer: After\n", + "---\n", + "Question: Is the episode about Chandler's work laugh before or after the one about his boss slapping his bum?\n", + "Options: ['Before', 'After']\n", + "Correct Answer: After\n", + "---\n", + "Question: Does Joey pretend to own a Porsche before or after Monica is given the Porsche by her dad?\n", + "Options: ['Before', 'After']\n", + "Correct Answer: Before\n", + "---\n", + "Question: Is the \"Holiday Armadillo\" Christmas episode before or after the Christmas episode where Ross dates a girl from Poughkeepsie?\n", + "Options: ['Before', 'After']\n", + "Correct Answer: After\n", + "---\n", + "Question: Does Chandler spend Thanksgiving in a box before or after the Thanksgiving episode with Brad Pitt?\n", + "Options: ['Before', 'After']\n", + "Correct Answer: Before\n", + "---\n", + "Question: Does Joey appear on game show \"Pyramid\" before or after he stars in TV show \"Mac and C.H.E.E.S.E.\"?\n", + "Options: ['Before', 'After']\n", + "Correct Answer: After\n", + "---\n", + "Question: Does the made up \"cups\" game come before or after the game that results in the girls having to swap apartments with the boys?\n", + "Options: ['Before', 'After']\n", + "Correct Answer: After\n", + "---\n", + "Question: Does the episode where Phoebe thinks her mother's spirit is inside a cat come before or after Phoebe smuggles a dog into Monica and Chandler's apartment?\n", + "Options: ['Before', 'After']\n", + "Correct Answer: Before\n", + "---\n", + "Question: Does Joey get nominated for a Soapie award before or after Ross presents the conference in Barbados?\n", + "Options: ['Before', 'After']\n", + "Correct Answer: Before\n", + "---\n", + "Question: Does Ross get his tan done before or after he gets his teeth done?\n", + "Options: ['Before', 'After']\n", + "Correct Answer: After\n", + "---\n", + "Question: Who said the first line in the pilot episode?\n", + "Options: ['MONICA', 'RACHEL', 'CHANDLER', 'JOEY', 'ROSS', 'PHOEBE']\n", + "Correct Answer: MONICA\n", + "---\n", + "Question: Which competitive card game did the gang play together for money?\n", + "Options: ['BLACK JACK', 'GIN RUMMY', 'POKER', 'HEARTS']\n", + "Correct Answer: POKER\n", + "---\n", + "Question: What body part did Phoebe find in her can of soda?\n", + "Options: ['AN EAR', 'A THUMB', 'A TOE', 'AN EYE']\n", + "Correct Answer: A THUMB\n", + "---\n", + "Question: Which famous model was Chandler stuck in an ATM vestibule with during the blackout?\n", + "Options: ['KATE MOSS', 'NAOMI CAMPBELL', 'CLAUDIA SCHIFFER', 'JILL GOODACRE']\n", + "Correct Answer: JILL GOODACRE\n", + "---\n", + "Question: What song did Marcel always play on Ross's stereo?\n", + "Options: ['\"I\\'M A BELIEVER\" BY THE MONKEES', '\"MONKEY ON MY BACK\" BY AEROSMITH', '\"THE LION SLEEPS TONIGHT\" BY THE TOKENS', '\"JUNGLELAND\" BY BRUCE SPRINGSTEEN']\n", + "Correct Answer: \"THE LION SLEEPS TONIGHT\" BY THE TOKENS\n", + "---\n", + "Question: How many lasagnas did Monica make for her Aunt Syl?\n", + "Options: ['6', '8', '10', 'A DOZEN']\n", + "Correct Answer: A DOZEN\n", + "---\n", + "Question: Can you finish this legendary quote?\n", + "Options: ['\"OH, I WISH I COULD, BUT I DON\\'T WANT TO.\"', '\"I HAVE TO GO BEFORE I PUT YOUR HEAD THROUGH A WALL.\"', '\"WHY AM I FRIENDS WITH YOU PEOPLE?\"', '\"OH, NO...\"']\n", + "Correct Answer: \"OH, I WISH I COULD, BUT I DON'T WANT TO.\"\n", + "---\n", + "Question: When Rachel did laundry for the first time, what color did she accidentally turn all of her clothes?\n", + "Options: ['YELLOW', 'PINK', 'BLUE', 'PURPLE']\n", + "Correct Answer: PINK\n", + "---\n", + "Question: What was Joey and Chandler's apartment number before it got switched to \"19\"?\n", + "Options: ['4', '7', '13', '17']\n", + "Correct Answer: 4\n", + "---\n", + "Question: Who told Ross that he was having a son?\n", + "Options: ['PHOEBE', 'MONICA', 'RACHEL', 'JOEY', 'CHANDLER', 'SUSAN']\n", + "Correct Answer: RACHEL\n", + "---\n", + "Question: Do you remember the name of the first Thanksgiving episode?\n", + "Options: ['\"THE ONE WHERE ROSS GOT HIGH\"', '\"THE ONE WITH THE FOOTBALL\"', '\"THE ONE WHERE UNDERDOG GETS AWAY\"', '\"THE ONE WITH ALL THE THANKSGIVINGS\"']\n", + "Correct Answer: \"THE ONE WHERE UNDERDOG GETS AWAY\"\n", + "---\n", + "Question: Finally, who did Rachel, Monica, and Phoebe spy on during their sleepover?\n", + "Options: ['GEORGE CLOONEY', 'JEAN-CLAUDE VAN DAMME', 'RALPH LAUREN', 'GEORGE STEPHANOPOULOS']\n", + "Correct Answer: GEORGE STEPHANOPOULOS\n", + "---\n", + "Question: What do Rachel and Ross do in Vegas?\n", + "Options: ['Make out', 'Get married', 'Fist fight', 'Ignore each other']\n", + "Correct Answer: Get married\n", + "---\n", + "Question: How do Ross and Rachel deal with their accidental marriage?\n", + "Options: ['They return to Vegas and ask them to void the marriage', 'They get an annulment', \"They decide to get an annulment, but Ross doesn't do it\", 'They agree to stay married']\n", + "Correct Answer: They decide to get an annulment, but Ross doesn't do it\n", + "---\n", + "Question: What furniture shop, that Phoebe did not support, did Ross buy an apothecary table from?\n", + "Options: ['**Cowperthwait & Sons Store**', 'Pottery Barn', 'Flea Market', 'Bloomingdales']\n", + "Correct Answer: Pottery Barn\n", + "---\n", + "Question: Which one of the characters claims they made out with Ralph Lauren?\n", + "Options: ['Rachel', 'Phoebe', 'Monica', 'Chandler']\n", + "Correct Answer: Phoebe\n", + "---\n", + "Question: What is the name of the student from Ross's paleontology class that Ross dates?\n", + "Options: ['Mona', 'Elizabeth', 'Carol', 'Rebecca']\n", + "Correct Answer: Elizabeth\n", + "---\n", + "Question: What happens when Chandler tries to propose to Monica at the restaurant?\n", + "Options: [\"Richard, Monica's ex, shows up at the restaurant\", 'Chandler changes his mind last minute', 'Chandler loses the ring and Monica rejects him', 'The waiter keeps interrupting him']\n", + "Correct Answer: Richard, Monica's ex, shows up at the restaurant\n", + "---\n", + "Question: Which one of Rachel's sisters does Ross date?\n", + "Options: ['Amy', 'Jill']\n", + "Correct Answer: Jill\n", + "---\n", + "Question: When Chandler and Monica decide to move in together, what does Chandler say he wants to turn the spare bedroom into?\n", + "Options: ['A storage room', 'A room for himself so he has space to keep this stuff', 'A game room', 'He thinks that Joey should move in to the spare room']\n", + "Correct Answer: A game room\n", + "---\n", + "Question: Why does Rachel not want to leave the apartment so Monica and Chandler can live together?\n", + "Options: [\"Rachel thinks Monica and Chandler aren't going to go through with the move\", \"Rachel thinks that the apartment has enough space for three people so she doesn't need to move out\", \"Rachel isn't ready to leave Monica yet\", \"She doesn't have anywhere else to live\"]\n", + "Correct Answer: Rachel thinks Monica and Chandler aren't going to go through with the move\n", + "---\n", + "Question: What does \"unagi\" mean?\n", + "Options: ['Freshwater Eel', 'Garter Snake', 'Sea Snake', 'Salmon Skin Roll']\n", + "Correct Answer: Freshwater Eel\n", + "---\n", + "Question: Whose Dad does Rachel date?\n", + "Options: [\"Monica's dad\", \"Elizabeth's dad\", \"Janice's dad\", \"Joey's girlfriend's dad\"]\n", + "Correct Answer: Elizabeth's dad\n", + "---\n", + "Question: What magazine prints the joke that Chandler or Ross made?\n", + "Options: ['Brooklyn Magazine', 'Playboy', 'New York Times', 'Time Magazine']\n", + "Correct Answer: Playboy\n", + "---\n", + "Question: What goes wrong when Ross tries to whiten his teeth?\n", + "Options: ['They turn yellow because he read the instructions wrong', \"They glowed in the dark 'cause he left them on too long\", \"They don't work 'cause he didn't leave them on long enough\", 'He burns himself and has to go to the hospital']\n", + "Correct Answer: They glowed in the dark 'cause he left them on too long\n", + "---\n", + "Question: What does Joey do when he finds someone's Porsche car keys left in Central Perk?\n", + "Options: ['He gives the keys to Gunther', \"He doesn't tell anyone and tries to find the owner himself\", 'He pretends that the Porsche is his to impress girls', 'He find the car and takes it for a ride']\n", + "Correct Answer: He pretends that the Porsche is his to impress girls\n", + "---\n", + "Question: Where does Monica propose to Chandler?\n", + "Options: ['Central Perk', \"Joey's apartment\", 'Their apartment', 'A fancy restaurant']\n", + "Correct Answer: Their apartment\n", + "---\n", + "Question: Who says this quote: \"What is with everyone? It’s Thanksgiving! Not truth day.\"\n", + "Options: ['Jack', 'Joey', 'Ross', 'Monica']\n", + "Correct Answer: Ross\n", + "---\n", + "Question: What does Monica successfully create with “Mockolate\"?\n", + "Options: ['Mockolate hot chocolate', 'Mockolate brownies', 'Mockolate chip cookies', 'Mockolate cake']\n", + "Correct Answer: Mockolate chip cookies\n", + "---\n", + "Question: What's Chandler punishment for kissing Joey's girlfriend?\n", + "Options: ['Spend Thanksgiving in a box', 'Leave the apartment', 'Never talk to his friends again', 'Eat Thanksgiving food']\n", + "Correct Answer: Spend Thanksgiving in a box\n", + "---\n", + "Question: Who was picked last at the football game? \n", + "\n", + "Options: ['Phoebe', 'Monica', 'Rachel', 'Chandler']\n", + "Correct Answer: Rachel\n", + "---\n", + "Question: Why were Phoebe and Rachel late for Thanksgiving dinner one year? \n", + "\n", + "Options: ['They were at a baby beauty contest', 'They were shopping', 'They went to the Rangers game', 'They were at the movies']\n", + "Correct Answer: They were at a baby beauty contest\n", + "---\n", + "Question: Who would get custody of baby Emma if Rachel and Ross died?\n", + "Options: ['Amy', 'Chandler and Monica', 'Jack and Judy Geller', 'Joey and Phoebe']\n", + "Correct Answer: Chandler and Monica\n", + "---\n", + "Question: What's the name of the dog Phoebe has been hiding in the apartment?\n", + "Options: ['Chappy', 'Chip', 'Clunkers', 'Chuck']\n", + "Correct Answer: Clunkers\n", + "---\n", + "Question: Who harshly calls Monica \"fat\" in a Thanksgiving flashback?\n", + "Options: ['Chandler', 'Judy', 'Rachel', 'Ross']\n", + "Correct Answer: Chandler\n", + "---\n", + "Question: Who pretends to understand football to get out of meal prep?\n", + "Options: ['Chandler and Ross', 'Joey and Phoebe', 'Rachel and Phoebe', 'Phoebe and Chandler']\n", + "Correct Answer: Phoebe and Chandler\n", + "---\n", + "Question: What's the name of the 50-year-old librarian Ross made out with?\n", + "Options: ['Emily Waltham', 'Anita Altman', 'Sarah Tuttle', 'Ellen Lynch']\n", + "Correct Answer: Anita Altman\n", + "---\n", + "Question: According to Chandler, what scares the \"bejesus\" out of him?\n", + "Options: ['Spiders', 'Country line dancing', 'Michael Flatley\\'s \"Lord of Dance\"', 'None of the above']\n", + "Correct Answer: Michael Flatley's \"Lord of Dance\"\n", + "---\n", + "Question: What is Monica's biggest pet peeve?\n", + "Options: ['Animals swimming', 'Animals dressed as humans', 'People calling their pets their children', 'None of the above']\n", + "Correct Answer: Animals dressed as humans\n", + "---\n", + "Question: Monica and Ross had a grandmother who died, name that grandmother.\n", + "Options: ['Diana', 'Anthea', 'Althea', 'None of the above']\n", + "Correct Answer: Althea\n", + "---\n", + "Question: Every week, the TV guide comes to Joey and Chandler's apartment. What name appears on the address label?\n", + "Options: ['Chandler Bing', 'Joey Tribbiani', 'Miss Chanandler Bong', 'None of the above']\n", + "Correct Answer: Miss Chanandler Bong\n", + "---\n", + "Question: What is the name of Chandler's father's Las Vegas all-male burlesque?\n", + "Options: ['Viva Las Gaygas', 'Diva Las Vegas', 'Las Vegas Divas', 'None of the above']\n", + "Correct Answer: Viva Las Gaygas\n", + "---\n", + "Question: What was Monica's nickname when she was a field hockey goalie?\n", + "Options: ['Fat Monica', 'Big fat goalie', 'Big bad goalie', 'None of the above']\n", + "Correct Answer: Big fat goalie\n", + "---\n", + "Question: Rachel claims this is her favorite movie...\n", + "Options: ['\"Dangerous Liaisons\"', '\"Weekend at Bernie\\'s\"', '\"Casino\"', 'None of the above']\n", + "Correct Answer: \"Dangerous Liaisons\"\n", + "---\n", + "Question: Her actual favorite movie is...\n", + "Options: ['\"Dangerous Liaisons\"', '\"Fatal Attraction\"', '\"Weekend at Bernie\\'s\"', 'None of the above']\n", + "Correct Answer: \"Weekend at Bernie's\"\n", + "---\n", + "Question: In what part of her body did Monica get a pencil stuck at age 14?\n", + "Options: ['Her ear', 'Her nose', 'Her arm', 'None of the above']\n", + "Correct Answer: Her ear\n", + "---\n", + "Question: Monica categories her towels. How many categories are there?\n", + "Options: ['7', '12', '11', 'None of the above']\n", + "Correct Answer: 11\n", + "---\n", + "Question: What is Joey's favorite food?\n", + "Options: ['Pizza', 'Sandwiches', 'Muffins', 'None of the above']\n", + "Correct Answer: Sandwiches\n", + "---\n", + "Question: Chandler was how old when he first touched a girl's breast?\n", + "Options: ['14', '17', '19', 'None of the above']\n", + "Correct Answer: 19\n", + "---\n", + "Question: Joey had an imaginary childhood friend. His name was...\n", + "Options: ['Morris', 'Monkey', 'Maurice', 'None of the above']\n", + "Correct Answer: Maurice\n", + "---\n", + "Question: His profession was...\n", + "Options: ['Police Officer', 'Space cowboy', 'Firefighter', 'None of the above']\n", + "Correct Answer: Space cowboy\n", + "---\n", + "Question: What is Chandler Bing's job?\n", + "Options: ['Statistical analysis and data reconfiguration', 'Transpsonter', 'incorrect']\n", + "Correct Answer: Statistical analysis and data reconfiguration\n", + "---\n", + "Question: How long has Joey been going to his tailor?\n", + "Options: ['15 years', '12 years', '8 years', '16 years']\n", + "Correct Answer: 12 years\n", + "---\n", + "Question: Who does Monica go to prom with?\n", + "Options: ['Dan Phillips', 'Chip Matthews', 'Roy Gublik', 'Frank Bower']\n", + "Correct Answer: Roy Gublik\n", + "---\n", + "Question: Which of Joey's sister's does Chandler make out with?\n", + "Options: ['Cookie', 'Dina', 'Gene', 'Mary Angela']\n", + "Correct Answer: Mary Angela\n", + "---\n", + "Question: What is Rachel's Father's favorite drink?\n", + "Options: ['Scotch neat', 'Scotch on the rocks', 'Scotch on the rocks with a twist', 'Scotch with a twist']\n", + "Correct Answer: Scotch neat\n", + "---\n", + "Question: What is Joey's PIN number?\n", + "Options: ['4693', '4536', '5639', '5396']\n", + "Correct Answer: 5639\n", + "---\n", + "Question: What is the name of the pizza delivery girl that Ross can't flirt with?\n", + "Options: ['Caitlin', 'KAte', 'Cat', 'Katie']\n", + "Correct Answer: Caitlin\n", + "---\n", + "Question: What nationality is Joey 1/16th?\n", + "Options: ['French', 'Chinese', 'Portuguese', 'Irish']\n", + "Correct Answer: Portuguese\n", + "---\n", + "Question: What drink did Joey's sister make herself right after she shot her husband?\n", + "Options: ['Vodka & Orange Juice', 'Vodka & Pineapple Juice', 'Vodka Straight', 'Vodka & Cranberry']\n", + "Correct Answer: Vodka & Cranberry\n", + "---\n", + "Question: Which card did Joey think the thief took when he stole all of Joey and Chandler's stuff?\n", + "Options: ['10 of spades', 'two of spades', 'five of spades', 'eight of spades']\n", + "Correct Answer: five of spades\n", + "---\n", + "Question: Who at Rachel's office called her Rocky?\n", + "Options: ['Her boss', 'The mail room guys', 'The copy guy', 'Her assistant']\n", + "Correct Answer: The mail room guys\n", + "---\n", + "Question: Who turns off the lamp in the opening credits?\n", + "Options: ['Rachel', 'Ross', 'Chandler', 'Monica']\n", + "Correct Answer: Monica\n", + "---\n", + "Question: Who says, \"What is with everyone? It’s Thanksgiving! Not truth day.\"\n", + "Options: ['Ross', 'Phoebe', 'Chandler', 'Joey']\n", + "Correct Answer: Ross\n", + "---\n", + "Question: How many types of potatoes did Monica make?\n", + "Options: ['One', 'Two', 'Three', 'None, they were burnt']\n", + "Correct Answer: Three\n", + "---\n", + "Question: What dessert does Rachel try and fail to make?\n", + "Options: ['Traditional British trifle', 'Traditional English trifle', 'Pumpkin pie', 'Tiramisu']\n", + "Correct Answer: Traditional English trifle\n", + "---\n", + "Question: Where did Joey’s Thanksgiving pants come from?\n", + "Options: ['Rachel’s maternity pants', \"Phoebe's maternity pants\", 'They were his all along']\n", + "Correct Answer: Phoebe's maternity pants\n", + "---\n", + "Question: What body part did Chandler lose in his worst Thanksgiving ever?\n", + "Options: ['Finger', 'Nose', 'Toe', 'Ear']\n", + "Correct Answer: Toe\n", + "---\n", + "Question: Joey’s worst Thanksgiving involved what animal?\n", + "Options: ['Turkey', 'Chicken', 'Dog', 'Cat']\n", + "Correct Answer: Turkey\n", + "---\n", + "Question: Who pretends to understand football to get out of helping cook Thanksgiving dinner?\n", + "Options: ['Chandler and Joey', 'Phoebe and Chandler', 'Ross and Joey', 'Rachel and Phoebe']\n", + "Correct Answer: Phoebe and Chandler\n", + "---\n", + "Question: Who was originally on Ross’s football team?\n", + "Options: ['Chandler and Joey', 'Phoebe and Rachel', 'Rachel and Joey', 'Chandler and Rachel']\n", + "Correct Answer: Chandler and Rachel\n", + "---\n", + "Question: What does Ross say the trifle tastes like?\n", + "Options: ['Feet', 'Fingers']\n", + "Correct Answer: Feet\n", + "---\n", + "Question: And who put a turkey on their head?\n", + "Options: ['Monica', 'Joey', 'Monica and Joey', 'Phoebe', 'Chandler', 'Phoebe and Chandler']\n", + "Correct Answer: Monica and Joey\n", + "---\n", + "Question: When does Monica start dating Chandler?\n", + "Options: ['Season 4', 'Season 6', 'Season 7', 'Season 9']\n", + "Correct Answer: Season 4\n", + "---\n", + "Question: When does Ross start dating Julie?\n", + "Options: ['Season 1', 'Season 2', 'Season 3', 'Season 4']\n", + "Correct Answer: Season 1\n", + "---\n", + "Question: When does Rachel start dating Paul Stevens?\n", + "Options: ['Season 5', 'Season 6', 'Season 7', 'Season 8']\n", + "Correct Answer: Season 6\n", + "---\n", + "Question: When does Chandler start dating Kathy?\n", + "Options: ['Season 2', 'Season 3', 'Season 4', 'Season 5']\n", + "Correct Answer: Season 4\n", + "---\n", + "Question: When does Ross start dating Emily?\n", + "Options: ['Season 3', 'Season 4', 'Season 5', 'Season 6']\n", + "Correct Answer: Season 4\n", + "---\n", + "Question: When does Rachel start dating Russ?\n", + "Options: ['Season 1', 'Season 2', 'Season 3', 'Season 4']\n", + "Correct Answer: Season 2\n", + "---\n", + "Question: When does Joey start dating Janine Lecroix?\n", + "Options: ['Season 3', 'Season 4', 'Season 6', 'Season 7']\n", + "Correct Answer: Season 6\n", + "---\n", + "Question: When does Monica start dating Alan?\n", + "Options: ['Season 1', 'Season 2', 'Season 3', 'Season 4']\n", + "Correct Answer: Season 1\n", + "---\n", + "Question: When does Phoebe start dating Parker?\n", + "Options: ['Season 4', 'Season 5', 'Season 8', 'Season 9']\n", + "Correct Answer: Season 8\n", + "---\n", + "Question: When does Monica start dating Richard Burke?\n", + "Options: ['Season 2', 'Season 3', 'Season 4', 'Season 5']\n", + "Correct Answer: Season 2\n", + "---\n", + "Question: When does Ross start dating Elizabeth?\n", + "Options: ['Season 4', 'Season 6', 'Season 8', 'Season 9']\n", + "Correct Answer: Season 6\n", + "---\n", + "Question: When does Phoebe start dating Roger?\n", + "Options: ['Season 1', 'Season 3', 'Season 4', 'Season 6']\n", + "Correct Answer: Season 1\n", + "---\n", + "Question: Finally, when does Joey start dating Charlie Wheeler?\n", + "Options: ['Season 6', 'Season 7', 'Season 8', 'Season 9']\n", + "Correct Answer: Season 9\n", + "---\n", + "Question: Which of these names is not the name of one of the friends?\n", + "Options: ['Monica', 'Joey', 'Jessica', 'Phoebe']\n", + "Correct Answer: Jessica\n", + "---\n", + "Question: Phoebe's scientist boyfriend David worked in which city?\n", + "Options: ['Aden', 'Minsk', 'Bangkok', 'Manila']\n", + "Correct Answer: Minsk\n", + "---\n", + "Question: What was the profession of Joey's imaginary friend?\n", + "Options: ['Circus clown', 'Doctor', 'Teacher', 'Space cowboy']\n", + "Correct Answer: Space cowboy\n", + "---\n", + "Question: Ross and Rachel's wedding dinner was held in which pizza chain in Vegas?\n", + "Options: [\"Domino's\", 'Pizza Hut', \"Papa John's\", 'Little Caesars']\n", + "Correct Answer: Pizza Hut\n", + "---\n", + "Question: Ross's first wife Carol leaves him for whom?\n", + "Options: ['Susan', 'Janice', 'Claire', 'Marissa']\n", + "Correct Answer: Susan\n", + "---\n", + "Question: Rachel goes on Ross's honeymoon by herself. Where does she go?\n", + "Options: ['Hong Kong, China', 'Osaka, Japan', 'Paris, France', 'Athens, Greece']\n", + "Correct Answer: Athens, Greece\n", + "---\n", + "Question: In what year did _Friends_ premiere?\n", + "Options: ['1994', '1993', '1991', '1995']\n", + "Correct Answer: 1994\n", + "---\n", + "Question: What's the first wedding gift Monica opens?\n", + "Options: ['Records', 'A toaster oven', 'A tiny salt shaker', 'A copy of her favorite movie']\n", + "Correct Answer: A tiny salt shaker\n", + "---\n", + "Question: Who had a pony and a boat at age 15?\n", + "Options: ['Rachel', 'Joey', 'Ross', 'Chandler']\n", + "Correct Answer: Rachel\n", + "---\n", + "Question: Who gave Phoebe away at her wedding?\n", + "Options: ['Ross', 'Jack Geller', 'Joey', 'Chandler']\n", + "Correct Answer: Chandler\n", + "---\n", + "Question: What's Joey's PIN number?\n", + "Options: ['4813', '5639', '5942', '7322']\n", + "Correct Answer: 5639\n", + "---\n", + "Question: After Richard and Monica break up, what does Monica become obsessed with?\n", + "Options: ['Cookies', 'Candy', 'Jam', 'Cheesecake']\n", + "Correct Answer: Jam\n", + "---\n", + "Question: And finally, what game show did Joey appear on?\n", + "Options: ['_Pyramid_', '_Who Wants to be a Millionaire_', '_Password_', '_Family Feud_']\n", + "Correct Answer: _Pyramid_\n", + "---\n", + "Question: What is Richard's job?\n", + "Options: ['Surgeon', 'Podiatrist', 'Ophthalmologist', 'Paleontologist']\n", + "Correct Answer: Ophthalmologist\n", + "---\n", + "Question: True or False: Rachel was a bridesmaid at her ex-fiance's wedding.\n", + "Options: ['True', 'False']\n", + "Correct Answer: True\n", + "---\n", + "Question: True or False: Rachel got a nose job.\n", + "Options: ['True', 'False']\n", + "Correct Answer: True\n", + "---\n", + "Question: True or False: Phoebe sold \"Smelly Cat\" for a commercial.\n", + "Options: ['True', 'False']\n", + "Correct Answer: False\n", + "---\n", + "Question: Where did Monica and Chandler originally want to get married?\n", + "Options: ['A church', 'New York', 'Vegas', 'Europe']\n", + "Correct Answer: Vegas\n", + "---\n", + "Question: What color shirt does Gunther wear in the Central Perk LEGO set (for those super-informed _Friends_ fanatics)?\n", + "Options: ['Yellow', 'Green', 'Blue', 'Turquoise']\n", + "Correct Answer: Green\n", + "---\n", + "Question: Finally, who says the last line in the show?\n", + "Options: ['Joey', 'Monica', 'Phoebe', 'Chandler']\n", + "Correct Answer: Chandler\n", + "---\n", + "Question: What is the name of the guy who helps Rachel get closure? \n", + "\n", + "Options: ['Joey', 'Gunther', 'Laurie', 'Michael']\n", + "Correct Answer: Michael\n", + "---\n", + "Question: What are Ross and Susan reacting to here?\n", + "Options: [\"Hearing the baby's heartbeat\", 'Seeing Carol walk out in her wedding dress', 'Watching Carol go into labor', \"Watching the baby's first steps\"]\n", + "Correct Answer: Hearing the baby's heartbeat\n", + "---\n", + "Question: Where do the gang and Charlie go on vacation at the end of Season 8?\n", + "Options: ['London', 'Las Vegas', \"Phoebe's mom's beach house\", 'Barbados']\n", + "Correct Answer: Barbados\n", + "---\n", + "Question: What is the name of the girl who Ross cheats on Rachel with? \n", + "\n", + "Options: ['Stephanie', 'Chloe', 'Mary-Angela', 'Charlie']\n", + "Correct Answer: Chloe\n", + "---\n", + "Question: Whose hair gets frizzy because of the humidity?\n", + "Options: ['Ross', 'Rachel', 'Monica', 'Phoebe']\n", + "Correct Answer: Monica\n", + "---\n", + "Question: What does Ross say when he found out Rachel got a cat? \n", + "\n", + "Options: ['\"Why do you have that thing?\"', '\"What the hell is that?\"', '\"Why is it inside out?\"', '\"Check it out!\"']\n", + "Correct Answer: \"Why is it inside out?\"\n", + "---\n", + "Question: What does Ross say when he hears the end of the voicemail Rachel had left him confessing her feelings?\n", + "Options: ['\"You\\'re not over me?\"', '\"I love you.\"', '\"When were you under me?', '\"What?!\"']\n", + "Correct Answer: \"When were you under me?\n", + "---\n", + "Question: What does Rachel call Ross when they came out of the chapel in LA? \n", + "\n", + "Options: ['Mrs. Rachel', 'Mr. \\nRoss', 'Mr. Rachel', 'Mrs. Ross']\n", + "Correct Answer: Mr. Rachel\n", + "---\n", + "Question: What color \n", + "t-shirts did Rachel and Monica wear in the trivia game against Joey and Chandler? \n", + "\n", + "Options: ['Red and blue', 'Grey and red', 'White and red', 'Purple and blue']\n", + "Correct Answer: White and red\n", + "---\n", + "Question: What book does Joey get Rachel for her birthday?\n", + "Options: [\"_Oh! The Places You'll Go_ by Dr. Seuss\", '_The Shining_ by Stephen King', '_Little Women_ by Louisa May Alcott', '_Love You Forever_ by Robert Munsch']\n", + "Correct Answer: _Oh! The Places You'll Go_ by Dr. Seuss\n", + "---\n", + "Question: Which aunt does cousin Cassie look like?\n", + "Options: ['Aunt Lilian', 'Aunt Marilyn', 'Aunt Iris', 'Aunt Murial']\n", + "Correct Answer: Aunt Marilyn\n", + "---\n", + "Question: What is the brand of detergent that Ross and Rachel use to wash their clothes?\n", + "Options: ['Persil', 'Almüt', 'Überweiss', 'Ichiban']\n", + "Correct Answer: Überweiss\n", + "---\n", + "Question: Who did Monica have her first kiss with?\n", + "Options: ['Ross']\n", + "Correct Answer: Ross\n", + "---\n", + "Question: Who tells the story about there being no hardware stores open past midnight in the Village?\n", + "Options: ['Russ', 'Fun Bobby', 'Dr. Roger', 'Steve']\n", + "Correct Answer: Fun Bobby\n", + "---\n", + "Question: Joey accidentally models as a guy with an STI. What's his name?\n", + "Options: ['Sal', 'Mario', 'Joseph', 'Al']\n", + "Correct Answer: Mario\n", + "---\n", + "Question: Which kid calls Ross a scrud?\n", + "Options: []\n", + "Correct Answer: \n", + "---\n", + "Question: What's the name of this masterpiece?\n", + "Options: ['Glynnis', 'Doris', 'Gladys', 'Phoebo']\n", + "Correct Answer: Gladys\n", + "---\n", + "Question: What is Joey dressed as?\n", + "Options: ['The goalie', 'The vicar', 'A fireball champ', 'The host of _Bamboozled_']\n", + "Correct Answer: The vicar\n", + "---\n", + "Question: Which one of these is _not_ one of Joey's sister?\n", + "Options: []\n", + "Correct Answer: \n", + "---\n", + "Question: What are Rachel and Joey doing here?\n", + "Options: [\"Joey's comforting Rachel after she moves out of Ross's\", 'Watching _Cujo_', \"Joey's comforting Rachel because her boss wanted to buy her baby\", 'Watching _The Shining_']\n", + "Correct Answer: Watching _Cujo_\n", + "---\n", + "Question: What French dialect does Phoebe say Joey is speaking?\n", + "Options: ['Reginé Philangé', 'Estée Lauder', 'Nestlé Tollhousé', 'U Bon Pain']\n", + "Correct Answer: Estée Lauder\n", + "---\n", + "Question: What's Monica doing here?\n", + "Options: [\"Giving a speech at her parent's anniversary\", 'Singing \"Delta Dawn\"', 'Accepting the award for cheffing excellence', \"Asking her restaurant-goers if they like Phoebe's music\"]\n", + "Correct Answer: Singing \"Delta Dawn\"\n", + "---\n", + "Question: Which of these is NOT a line in this song?\n", + "Options: ['\"Breakfast is near\"', '\"Sunshine is here\"', '\"I\\'m full of cheer\"', '\"The sky is clear\"']\n", + "Correct Answer: \"I'm full of cheer\"\n", + "---\n", + "Question: What’s Rachel’s favourite movie?\n", + "Options: ['_Dangerous Liaisons_', '_When Harry Met Sally_', 'Ghostbusters', \"_Weekend at Bernie's_\"]\n", + "Correct Answer: _Weekend at Bernie's_\n", + "---\n", + "Question: Who died when Phoebe went to the Dentist?\n", + "Options: ['Her roommate Denise, Paul the wine guy, and Fun Bobby', 'Her aunt Mary, John the Mailman, and Albino Bob', 'Gandalf, Crazy Eddie, and Mr. Heckles']\n", + "Correct Answer: Her aunt Mary, John the Mailman, and Albino Bob\n", + "---\n", + "Question: Who is Missy Goldberg?\n", + "Options: []\n", + "Correct Answer: \n", + "---\n", + "Question: What job does Susan Bunch do?\n", + "Options: ['Actor', 'Commercial director', 'Blacksmith', 'Teacher']\n", + "Correct Answer: Commercial director\n", + "---\n", + "Question: Which actor's haircut did Monica want?\n", + "Options: ['Winona Ryder', 'Demi Moore', 'Terri Hatcher', 'Julia Roberts']\n", + "Correct Answer: Demi Moore\n", + "---\n", + "Question: What city does the Mama's Little Bakery cheesecake that Rachel and Chandler steal come from?\n", + "Options: ['Albany, New York', 'Chicago, Illinois', 'Boston, Massachusetts', 'Philadelphia, Pennsylvania']\n", + "Correct Answer: Chicago, Illinois\n", + "---\n", + "Question: After reading _Be Your Own Windkeeper_, who does Phoebe call a leaf blower?\n", + "Options: ['Rachel', 'Monica']\n", + "Correct Answer: Rachel\n", + "---\n", + "Question: When Ben shows up with a Barbie doll, what doll does Ross want him to play with instead?\n", + "Options: ['Action Man', 'GI Joe', 'Optimus Prime', 'Stretch Armstrong']\n", + "Correct Answer: GI Joe\n", + "---\n", + "Question: Who did Chandler’s dad dress up as when he went to his swim meets?\n", + "Options: ['Marilyn Monroe', 'Carmen Miranda', 'Mae West', 'Lucille Ball']\n", + "Correct Answer: Carmen Miranda\n", + "---\n", + "Question: Which of these guys was married to phoebe?\n", + "Options: []\n", + "Correct Answer: \n", + "---\n", + "Question: And finally, why is Ross Sulking?\n", + "Options: ['Because Charlie was with Joey', 'Because no one got him an engagement present', 'Because someone ate his sandwich', 'Because he and Rachel were on a break']\n", + "Correct Answer: Because no one got him an engagement present\n", + "---\n", + "Question: \"The One Where Ross Is Fine\"\n", + "Options: ['Season 7', 'Season 8', 'Season 9', 'Season 10']\n", + "Correct Answer: Season 10\n", + "---\n", + "Question: \"The One With the Prom Video\"\n", + "Options: ['Season 1', 'Season 2', 'Season 3', 'Season 4']\n", + "Correct Answer: Season 2\n", + "---\n", + "Question: \"The One Where Rosita Dies\"\n", + "Options: ['Season 6', 'Season 7', 'Season 8', 'Season 9']\n", + "Correct Answer: Season 7\n", + "---\n", + "Question: \"The One With the Embryos\"\n", + "Options: ['Season 3', 'Season 4', 'Season 5', 'Season 6']\n", + "Correct Answer: Season 4\n", + "---\n", + "Question: \"The One with the Thumb\"\n", + "Options: ['Season 1', 'Season 2', 'Season 3', 'Season 4 \\n']\n", + "Correct Answer: Season 1\n", + "---\n", + "Question: \"The One Where Chandler Takes a Bath\"\n", + "Options: ['Season 5', 'Season 6', 'Season 7', 'Season 8']\n", + "Correct Answer: Season 8\n", + "---\n", + "Question: \"The One with All the Cheesecakes\"\n", + "Options: ['Season 5', 'Season 6', 'Season 7', 'Season 8']\n", + "Correct Answer: Season 7\n", + "---\n", + "Question: \"The One Where Everybody Finds Out\"\n", + "Options: ['Season 5', 'Season 6', 'Season 7', 'Season 8']\n", + "Correct Answer: Season 5\n", + "---\n", + "Question: \"The One with the Football\"\n", + "Options: ['Season 2', 'Season 3', 'Season 4', 'Season 5']\n", + "Correct Answer: Season 3\n", + "---\n", + "Question: \"The One with the Apothecary Table\"\n", + "Options: ['Season 4', 'Season 5', 'Season 6', 'Season 7 \\n']\n", + "Correct Answer: Season 6\n", + "---\n", + "Question: \"The One with George Stephanopoulos\"\n", + "Options: ['Season 1', 'Season 2', 'Season 3', 'Season 4']\n", + "Correct Answer: Season 1\n", + "---\n", + "Question: \"The One Where Eddie Won't Go\"\n", + "Options: ['Season 1', 'Season 2', 'Season 3', 'Season 4']\n", + "Correct Answer: Season 2\n", + "---\n", + "Question: \"The One with All the Resolutions\"\n", + "Options: ['Season 4', 'Season 5', 'Season 6', 'Season 7']\n", + "Correct Answer: Season 5\n", + "---\n", + "Question: \"The One Where the Stripper Cries\"\n", + "Options: ['Season 7', 'Season 8', 'Season 9 \\n', 'Season 10']\n", + "Correct Answer: Season 10\n", + "---\n", + "Question: And \"The One with the Rumor\"\n", + "Options: ['Season 7', 'Season 8', 'Season 9', 'Season 10']\n", + "Correct Answer: Season 8\n", + "---\n", + "Question: \"Fun Bobby\", who wasn't actually so funny, was:\n", + "Options: [\"Ross' colleague\", \"Monica's on & off boyfriend\", \"Chandler's childhood friend\", \"Someone from Rachel's past\"]\n", + "Correct Answer: Monica's on & off boyfriend\n", + "---\n", + "Question: Estelle, who was always by the phone, was:\n", + "Options: ['Joey\\'s co-star in \"Days of Our Lives\"', \"Phoebe's former friend\", \"Joey's acting agent\", \"Rachel's boss\"]\n", + "Correct Answer: Joey's acting agent\n", + "---\n", + "Question: Owen, the sweet, sweet scout, was:\n", + "Options: [\"The adopted son of Phoebe's friends\", \"A visitor at Ross' Museum\", \"Monica's ex-boyfriend Richard's son\", \"Ross' neighbour's son\"]\n", + "Correct Answer: The adopted son of Phoebe's friends\n", + "---\n", + "Question: Cassie, the hair-flipping diva, was:\n", + "Options: [\"Rachel's friend\", \"Joey's fling\", 'A Central Perk waitress', \"Ross & Monica's cousin\"]\n", + "Correct Answer: Ross & Monica's cousin\n", + "---\n", + "Question: David, who shifted to Minsk, was:\n", + "Options: [\"Chandler's friend\", \"Phoebe's boyfriend\", \"Ursula's boyfriend\", \"Monica's boyfriend\"]\n", + "Correct Answer: Phoebe's boyfriend\n", + "---\n", + "Question: Erica, the sweet, selfless lady, was:\n", + "Options: [\"The biological mother of Chandler & Monica's kids\", \"Carol's lesbian partner\", \"Emma's Nanny\", \"Joey's sister\"]\n", + "Correct Answer: The biological mother of Chandler & Monica's kids\n", + "---\n", + "Question: Charlie was:\n", + "Options: [\"Ross & Joey's love interest\", \"Joey's crush\", \"Ross' co-worker\", \"Chandler & Joey's love interest\"]\n", + "Correct Answer: Ross & Joey's love interest\n", + "---\n", + "Question: Tag Jones, who had the same red sweater as Ross, was:\n", + "Options: [\"Phoebe's love interest\", \"Monica's love interest\", \"Joey's co-actor\", \"Rachel's love interest\"]\n", + "Correct Answer: Rachel's love interest\n", + "---\n", + "Question: Mr Treeger was:\n", + "Options: ['A Central Perk employee', \"The superintendent of Monica's building\", \"The superintendent of Ross' new building\", 'A neighbour from the building']\n", + "Correct Answer: The superintendent of Monica's building\n", + "---\n", + "Question: Elizabeth was:\n", + "Options: [\"Rachel's cousin\", \"Monica's sous chef\", \"Ross' student\", \"Chandler's cousin\"]\n", + "Correct Answer: Ross' student\n", + "---\n", + "Question: Wendy, who got all flirty during the holidays, was:\n", + "Options: [\"Chandler's co-worker in Tulsa\", \"Ross' co-worker in China\", \"Chandler's co-worker in New York\", \"Ross' co-worker in New York\"]\n", + "Correct Answer: Chandler's co-worker in Tulsa\n", + "---\n", + "Question: Janine, who wasn’t very fond of Chandler and Monica, was:\n", + "Options: [\"Ross' girlfriend\", \"Phoebe's roommate\", \"Ross' roommate\", \"Joey's roommate\"]\n", + "Correct Answer: Joey's roommate\n", + "---\n", + "Question: Barry was:\n", + "Options: [\"Rachel's friend's fiancé\", \"Rachel's former fiancé\", \"Monica's high-school boyfriend\", \"Phoebe's high-school boyfriend\"]\n", + "Correct Answer: Rachel's former fiancé\n", + "---\n", + "Question: This guy was:\n", + "Options: ['A casino owner', 'A cruise owner', \"Joey's hand twin\", 'A robber']\n", + "Correct Answer: Joey's hand twin\n", + "---\n", + "Question: Russ was:\n", + "Options: [\"Rachel's imaginary boyfriend\", \"Rachel's short-term boyfriend\", \"Rachel's co-worker\", \"Rachel's friend from the past\"]\n", + "Correct Answer: Rachel's short-term boyfriend\n", + "---\n", + "Question: Timothy, Monica’s brief love interest, was:\n", + "Options: [\"Richard Burke's son\", \"Richard Burke's grandson\", \"Richard Burke's cousin\", \"Richard Burke's younger sibling\"]\n", + "Correct Answer: Richard Burke's son\n", + "---\n", + "Question: Pete was:\n", + "Options: [\"Monica's co-worker\", \"A manager at Monica's restaurant\", \"One of Monica's boyfriends\", 'The owner of Central Perk']\n", + "Correct Answer: One of Monica's boyfriends\n", + "---\n", + "Question: Paolo, who speaks very little English, was:\n", + "Options: [\"Phoebe's boyfriend from Italy\", \"Phoebe's boyfriend from Spain\", \"Rachel's boyfriend from Spain\", \"Rachel's boyfriend from Italy\"]\n", + "Correct Answer: Rachel's boyfriend from Italy\n", + "---\n", + "Question: Roy \"Goodbody\" was:\n", + "Options: ['An officer', 'A stripper', 'An actor', 'A watchman']\n", + "Correct Answer: A stripper\n", + "---\n", + "Question: And finally, Nora was:\n", + "Options: [\"Chandler's mom\", \"Rachel's mom\", 'A celebrity', \"Phoebe's mom\"]\n", + "Correct Answer: Chandler's mom\n", + "---\n", + "Question: What high school did Monica and Rachel go to?\n", + "Options: ['Greenwood High School', 'Lincoln High School', 'Hunter High School', 'Long Island High School']\n", + "Correct Answer: Lincoln High School\n", + "---\n", + "Question: And what's the name and profession of Joey's imaginary childhood friend?\n", + "Options: ['Maurice; Space Cowboy', 'Anthony; Rockstar Robot', 'Lenny; Two-Headed Doctor', 'Bruce; Talking Lizard']\n", + "Correct Answer: Maurice; Space Cowboy\n", + "---\n", + "Question: Name a song Phoebe NEVER sings:\n", + "Options: ['\"Crazy Underwear\"', '\"Smelly Cat\"', '\"Dairy Song\"', '\"Papier Mache Man\"']\n", + "Correct Answer: \"Dairy Song\"\n", + "---\n", + "Question: When does Ross lose his virginity to Carol?\n", + "Options: ['May 7th', 'November 11th', 'October 20th', 'September 27th']\n", + "Correct Answer: October 20th\n", + "---\n", + "Question: Which public figure do the girls spy on during their sleepover?\n", + "Options: ['John F. Kennedy Jr.', 'George Clooney', 'Jean-Claude Van Damme', 'George Stephanopoulos']\n", + "Correct Answer: George Stephanopoulos\n", + "---\n", + "Question: Finish this iconic Chandler joke:\n", + "Options: ['\"And you got through the work day?\"', '\"What did the police say?\"', '\"Did a dinosaur eat it?\"', '\"And you didn\\'t report it to HR?\"']\n", + "Correct Answer: \"What did the police say?\"\n", + "---\n", + "Question: What's the first layer in Rachel's Thanksgiving Trifle?\n", + "Options: ['Jam', 'Custard', 'Raspberries', 'Ladyfingers']\n", + "Correct Answer: Ladyfingers\n", + "---\n", + "Question: How many times do Monica and Chandler have sex in London?\n", + "Options: ['Three', 'Seven', 'One', 'Five']\n", + "Correct Answer: Seven\n", + "---\n", + "Question: Who sits on the orange couch first during the opening theme?\n", + "Options: ['Jennifer Aniston', 'Courteney Cox', 'Lisa Kudrow', 'Matt LeBlanc', 'Matthew Perry', 'David Schwimmer']\n", + "Correct Answer: Jennifer Aniston\n", + "---\n", + "Question: Which Oscar winner makes a guest appearance on _Friends_?\n", + "Options: ['Halle Berry', 'Gwyneth Paltrow', 'Julia Roberts', 'Jeff Bridges']\n", + "Correct Answer: Julia Roberts\n", + "---\n", + "Question: What's the name of Joey's sister who Chandler hooks up with?\n", + "Options: ['Dina', 'Mary-Angela', 'Mary-Therese', 'Veronica']\n", + "Correct Answer: Mary-Angela\n", + "---\n", + "Question: Who gets super drunk on their 30th birthday?\n", + "Options: ['Joey', 'Phoebe', 'Ross', 'Monica', 'Rachel', 'Chandler']\n", + "Correct Answer: Monica\n", + "---\n", + "Question: And finally, which musical soundtrack does Chandler own two copies of?\n", + "Options: ['_Dreamgirls_', '_Cats_', '_Annie_', '_Rent_']\n", + "Correct Answer: _Annie_\n", + "---\n", + "Question: “You threw my sandwich away...my sandwich? MY SANDWICH?!”\n", + "Options: ['Joey', 'Ross', 'Chandler', 'Monica']\n", + "Correct Answer: Ross\n", + "---\n", + "Question: \"Until I was 25, I thought the response to 'I love you' was 'Oh, crap.'\"\n", + "Options: ['Monica', 'Rachel', 'Chandler', 'Joey']\n", + "Correct Answer: Chandler\n", + "---\n", + "Question: \"I don’t kiss and tell. But this hickey speaks for itself.\"\n", + "Options: ['Phoebe', 'Joey', 'Rachel', 'Janice']\n", + "Correct Answer: Phoebe\n", + "---\n", + "Question: \"I did not cry my eyes out! Come on, it's like the end of an era.\"\n", + "Options: ['Monica', 'Rachel', 'Gunther', 'Joey']\n", + "Correct Answer: Joey\n", + "---\n", + "Question: \"How long do cats live? Like, assuming you don’t throw 'em under a bus or something?\"\n", + "Options: ['Phoebe', 'Rachel', 'Joey', 'Monica']\n", + "Correct Answer: Rachel\n", + "---\n", + "Question: \"Welcome to the real world. It sucks. You're gonna love it.\"\n", + "Options: ['Chandler', 'Phoebe', 'Monica', 'Rachel']\n", + "Correct Answer: Monica\n", + "---\n", + "Question: \"You're a very sweet person, Ross. Unfortunately, I just don't think I can take another second of you whining.\"\n", + "Options: ['Janice', 'Emily', 'Carol', 'Charlie']\n", + "Correct Answer: Janice\n", + "---\n", + "Question: \"Only because my parents wanted me to be. I wanted to be a sheriff.\"\n", + "Options: ['Mike', 'Gunther', 'Richard', 'Jack Geller']\n", + "Correct Answer: Richard\n", + "---\n", + "Question: \"Who are, by the way, the most sinfully boring people I've ever met in my life.\"\n", + "Options: ['Gunther', 'Mike', 'Estelle', 'Carol']\n", + "Correct Answer: Mike\n", + "---\n", + "Question: \"You know that thing we put over here with the pin in it? It's time to take the pin out.\"\n", + "Options: ['Susan', '**Emily**', 'Judy Geller', 'Carol']\n", + "Correct Answer: Carol\n", + "---\n", + "Question: \"I just might know a few things that could help you inflict some pain.\"\n", + "Options: ['Emily', 'Ursula', 'Estelle', 'Frank Jr.']\n", + "Correct Answer: Emily\n", + "---\n", + "Question: And finally, \"Hey, buddy, this is a family place. Put the mouse back in the house.\"\n", + "Options: ['Mr. Treeger', 'Frank Jr.', 'Jack Geller', 'Gunther']\n", + "Correct Answer: Gunther\n", + "---\n", + "Question: \"We were on a break!!\"\n", + "Options: ['Rachel', 'Ross', 'Gunther', 'Chandler']\n", + "Correct Answer: Ross\n", + "---\n", + "Question: \"I KNEW IT!\"\n", + "Options: ['Joey', 'Phoebe', 'Chandler', 'Ross']\n", + "Correct Answer: Chandler\n", + "---\n", + "Question: \"Could I BE wearing any more clothes? If only I wasn't going commando!\"\n", + "Options: ['Chandler', 'Ross', 'Phoebe', 'Joey']\n", + "Correct Answer: Joey\n", + "---\n", + "Question: \"Pants: like shorts but longer.\"\n", + "Options: ['Ross', 'Joey', 'Jack Geller', 'Chandler']\n", + "Correct Answer: Chandler\n", + "---\n", + "Question: \"You know that smell gas has? \n", + "They put that in [the pizza].\"\n", + "Options: [\"Eddie, Chandler's temporary roomate\", 'Joey', 'Ross', 'Gunther']\n", + "Correct Answer: Ross\n", + "---\n", + "Question: \"I've got this uncontrollable need to please people.\"\n", + "Options: ['Chandler', 'Ross', 'Monica', 'Phoebe']\n", + "Correct Answer: Monica\n", + "---\n", + "Question: \"It's like all my life everyone has always told me 'you're a shoe! You're a shoe, you're a shoe, you're a shoe!'\"\n", + "Options: ['Rachel', 'Joey', 'Phoebe', 'Monica']\n", + "Correct Answer: Rachel\n", + "---\n", + "Question: \"Welcome to the real world! It sucks but you're gonna love it!\"\n", + "Options: ['Monica', 'Chandler', 'Rachel', 'Marcel', 'Phoebe']\n", + "Correct Answer: Monica\n", + "---\n", + "Question: \"See? He's her lobster!\"\n", + "Options: ['Joey', 'Ross', 'Monica', 'Jack', 'Phoebe']\n", + "Correct Answer: Phoebe\n", + "---\n", + "Question: \"I was wondering if you'd like to go to a movie with me sometime... as my lover.\"\n", + "Options: ['Joey', 'The copygirl', 'Gunther', 'Ross', 'Phoebe']\n", + "Correct Answer: Gunther\n", + "---\n", + "Question: \"I want world peace... oh, and bigger boobs.\"\n", + "Options: ['Monica', \"Amy, Rachel's sister\", 'Emily', 'Phoebe', 'Rachel']\n", + "Correct Answer: Phoebe\n", + "---\n", + "Question: \"That's a lot of information to get in 30 seconds.\"\n", + "Options: ['Monica', 'Chandler', 'Ross', 'Mike Hannigan', 'Judy Geller']\n", + "Correct Answer: Judy Geller\n", + "---\n", + "Question: Who said the very last line in the series?\n", + "Options: ['Ross', 'Rachel', 'Chandler', 'Monica', 'Marcel', 'Joey', 'Phoebe']\n", + "Correct Answer: Chandler\n", + "---\n", + "Question: Which friend has the profession of actor?\n", + "Options: ['Rachel', 'Joey', 'Monica', 'Phoebe']\n", + "Correct Answer: Joey\n", + "---\n", + "Question: Which two friends are siblings?\n", + "Options: ['Chandler and Joey', 'Rachel and Monica', 'Ross and Phoebe', 'Ross and Monica']\n", + "Correct Answer: Ross and Monica\n", + "---\n", + "Question: Out of all the friends who got married, who was married last on the show?\n", + "Options: ['Phoebe', 'Monica', 'Rachel', 'Joey']\n", + "Correct Answer: Phoebe\n", + "---\n", + "Question: Where do Chandler and Monica start sleeping together?\n", + "Options: ['Paris', 'Barbados', 'Greece', 'London']\n", + "Correct Answer: London\n", + "---\n", + "Question: How many sisters does Joey have?\n", + "Options: ['4', '5', '7', '8']\n", + "Correct Answer: 7\n", + "---\n", + "Question: **What does Ross shout as he moves his new couch into his apartment?**\n", + "Options: ['\"Move it!\"', '\"Come on!\"', '\"Pivot!\"', '\"To the left!\"']\n", + "Correct Answer: \"Pivot!\"\n", + "---\n", + "Question: **What is the name of the comic book Ross creates?**\n", + "Options: ['_Dinosaur Dude_', '_Science Boy_', '_That Bone Boy_', '_The Science Man_']\n", + "Correct Answer: _Science Boy_\n", + "---\n", + "Question: **Who was Monica’s midnight mystery kisser?**\n", + "Options: ['Joey', 'Rachel', 'Chandler', 'Ross']\n", + "Correct Answer: Ross\n", + "---\n", + "Question: **Which actor plays Phoebe’s husband?**\n", + "Options: ['Bruce Willis', 'Brad Pitt', 'Paul Rudd', 'Charlie Sheen']\n", + "Correct Answer: Paul Rudd\n", + "---\n", + "Question: **Which Stephen King book did Joey hide in his freezer?**\n", + "Options: ['_It_', '_Carrie_', '_The Shining_', '_Pet Sematary_']\n", + "Correct Answer: _The Shining_\n", + "---\n", + "Question: **Which instrument did Ross intend to play at Monica and Chandler’s wedding?**\n", + "Options: ['Bagpipes', 'Flute', 'Harmonica', 'Piano']\n", + "Correct Answer: Bagpipes\n", + "---\n", + "Question: **Which toe does Chandler lose when Monica drops the knife on his foot?**\n", + "Options: ['Left foot, big toe', 'Right foot, big toe', 'Left foot, pinky toe', 'Right foot, pinky toe']\n", + "Correct Answer: Right foot, pinky toe\n", + "---\n", + "Question: **What is the name of the hairless cat Rachel gets for herself?**\n", + "Options: ['Cheeseball', 'Mrs. Whiskerson', 'Fluffy Meowington', 'Mozzarella']\n", + "Correct Answer: Mrs. Whiskerson\n", + "---\n", + "Question:  **How much did Pete Becker tip Monica at the diner she worked at?**\n", + "Options: ['$100', '$2,000', '$15,000', '$20,000']\n", + "Correct Answer: $20,000\n", + "---\n", + "Question: What did Monica and Chandler name their twins?\n", + "Options: ['Jacob and Julie', 'Louie and Jane', 'James and Sarah', 'Jack and Erica']\n", + "Correct Answer: Jack and Erica\n", + "---\n", + "Question: What did the group use to fashion a \"very long poking device\" to poke Ugly Naked Guy with?\n", + "Options: ['Measuring tape', 'Crayons', 'Chopsticks', 'Pencils']\n", + "Correct Answer: Chopsticks\n", + "---\n", + "Question:   And what kind of food is the Joey Special?\n", + "Options: ['A double meatball sub', 'Two pizzas', 'A BLT with double bacon', 'A whole rotisserie chicken']\n", + "Correct Answer: Two pizzas\n", + "---\n", + "Question: “This is all a moo point.”\n", + "Options: ['Phoebe', 'Joey', 'Chandler', 'Rachel']\n", + "Correct Answer: Joey\n", + "---\n", + "Question: **“And I have to live with a boy!”**\n", + "Options: ['Rachel', 'Phoebe', 'Monica', 'Janice']\n", + "Correct Answer: Monica\n", + "---\n", + "Question: \"Offering people gum is not cooking.\"\n", + "Options: ['Joey', 'Ross', 'Rachel', 'Chandler']\n", + "Correct Answer: Chandler\n", + "---\n", + "Question: \"Nestlé Toulouse.\"\n", + "Options: ['Gunther', 'Monica', 'Phoebe', 'Joey']\n", + "Correct Answer: Phoebe\n", + "---\n", + "Question: \"Okay, no uterus, no opinion.\"\n", + "Options: ['Rachel', 'Phoebe', 'Monica', 'Ross']\n", + "Correct Answer: Rachel\n", + "---\n", + "Question: \"PIVOT!\"\n", + "Options: ['Joey', 'Ross', 'Chandler', 'Phoebe']\n", + "Correct Answer: Ross\n", + "---\n", + "Question: “Oh, I wish I could, but I don’t want to.”\n", + "Options: ['Phoebe', 'Chandler', 'Ross', 'Joey']\n", + "Correct Answer: Phoebe\n", + "---\n", + "Question: “Occupation? Dinosaurs.”\n", + "Options: ['Monica', 'Ross', 'Rachel', 'Joey']\n", + "Correct Answer: Joey\n", + "---\n", + "Question: “Oh god. Can open. Worms everywhere!\"\n", + "Options: ['Chandler', 'Rachel', 'Joey', 'Phoebe']\n", + "Correct Answer: Chandler\n", + "---\n", + "Question: “Welcome to the real world. It sucks. You’re gonna love it.”\n", + "Options: ['Rachel', 'Chandler', 'Ross', 'Monica']\n", + "Correct Answer: Monica\n", + "---\n", + "Question: Click on the lie:\n", + "Options: ['As a teen, Rachel was in trouble for shoplifting berets from the local mall', 'Rachel was once gifted a \"Female Body Inspector\" shirt on a date', \"Rachel's middle name is Karen\"]\n", + "Correct Answer: As a teen, Rachel was in trouble for shoplifting berets from the local mall\n", + "---\n", + "Question: Click on the lie:\n", + "Options: [\"Chandler didn't want to date a woman because her head was too big\", \"Michael Flatley's _Lord of the Dance_ scares the bejesus out of Chandler\", 'Chandler once rushed the stage at a Duran Duran concert']\n", + "Correct Answer: Chandler once rushed the stage at a Duran Duran concert\n", + "---\n", + "Question: Click on the lie:\n", + "Options: ['Ross tried to be a professional dancer', 'Ross owns days of the week underpants with dinosaurs on them', 'Ross stole his dad’s Playboys and blamed the mailman']\n", + "Correct Answer: Ross owns days of the week underpants with dinosaurs on them\n", + "---\n", + "Question: Click on the lie:\n", + "Options: ['Monica’s first kiss was Ross', \"Monica's childhood dog was called LaPooh\", 'Monica used to have a crush on Kermit the Frog']\n", + "Correct Answer: Monica's childhood dog was called LaPooh\n", + "---\n", + "Question: Click on the lie:\n", + "Options: ['Joey had a Cabbage Patch doll called Alicia May Emory', 'Joey proposed to Phoebe', 'Joey frequently stays in all-you-can-eat buffets for lunch and dinner']\n", + "Correct Answer: Joey frequently stays in all-you-can-eat buffets for lunch and dinner\n", + "---\n", + "Question: Click on the lie:\n", + "Options: ['Phoebe learned Taekwondo from some men at the Y', 'Phoebe once dated a man who ate chalk', \"Phoebe doesn't believe in the law of gravity\"]\n", + "Correct Answer: Phoebe learned Taekwondo from some men at the Y\n", + "---\n", + "Question: Click on the lie:\n", + "Options: ['Gunther\\'s nicknamed a drink \"The Rachel\"', 'Gunther was on the show _All My Children_', 'Gunther dyes his hair']\n", + "Correct Answer: Gunther's nicknamed a drink \"The Rachel\"\n", + "---\n", + "Question: Click on the lie:\n", + "Options: [\"Phoebe's dad is a famous tree surgeon\", 'Jack Geller tried to sell ice over the internet', 'Rachel\\'s dad\\'s nickname for Ross is \"Wet Head\"']\n", + "Correct Answer: Phoebe's dad is a famous tree surgeon\n", + "---\n", + "Question: Click on the lie:\n", + "Options: ['Mike was dating a woman named Precious when he and Phoebe got back together', \"Kathy's favorite book as a child was _The Velveteen Rabbit_\", 'Charlie has two PhDs']\n", + "Correct Answer: Charlie has two PhDs\n", + "---\n", + "Question: Click on the lie:\n", + "Options: ['The show\\'s credits say that Russ is played by \"Snaro\"', 'Matthew Perry directed two episodes', 'The interior of the show is filmed in LA']\n", + "Correct Answer: Matthew Perry directed two episodes\n", + "---\n", + "Question: What did Rachel get a tattoo of?\n", + "Options: ['A butterfly', 'A heart', 'An infinity sign', 'Lips']\n", + "Correct Answer: A heart\n", + "---\n", + "Question: What kind of plastic surgery did Rachel have?\n", + "Options: ['Toe shortening', 'Ear pinning', 'Nose job', 'Breast enlargement']\n", + "Correct Answer: Nose job\n", + "---\n", + "Question: Ross created a comic book called what?\n", + "Options: ['_The Chilling Adventures of Science Boy_', '_Science Boy_', '_Dino-Boy_', '_The Monica_']\n", + "Correct Answer: _Science Boy_\n", + "---\n", + "Question: Ross, Susan, and Phoebe got stuck where before Ben’s birth?\n", + "Options: ['Closet', 'Elevator', 'Traffic', 'Waiting room']\n", + "Correct Answer: Closet\n", + "---\n", + "Question: Where did Chandler tell Janice he was moving to in order to avoid seeing her again?\n", + "Options: ['Los Angeles', 'France', 'Jersey City', 'Yemen']\n", + "Correct Answer: Yemen\n", + "---\n", + "Question: What is the name of Joey’s Barcalounger?\n", + "Options: ['Rosita', 'Mama', 'Benita', 'Linda']\n", + "Correct Answer: Rosita\n", + "---\n", + "Question: Who was allergic to kiwi?\n", + "Options: ['Joey', 'Monica', 'Ross', 'Phoebe']\n", + "Correct Answer: Ross\n", + "---\n", + "Question: What language does Phoebe attempt to teach Joey?\n", + "Options: ['Italian', 'Spanish', 'French', 'Dutch']\n", + "Correct Answer: French\n", + "---\n", + "Question: What color is Monica’s apartment?\n", + "Options: ['Purple', 'White', 'Grey', 'Green']\n", + "Correct Answer: Purple\n", + "---\n", + "Question: In an effort to get over Richard, Monica started making what?\n", + "Options: ['Lasagna', 'Jam', 'Cake', 'Macaroni']\n", + "Correct Answer: Jam\n", + "---\n", + "Question: Who's husband changed their name to \"Crap Bag\"?\n", + "Options: ['Phoebe', 'Monica', 'Rachel', 'Janice']\n", + "Correct Answer: Phoebe\n", + "---\n", + "Question: What was in Monica's closet?\n", + "Options: ['Feminine products', 'It was empty', 'Literally everything', 'Old, dirty pots and pans']\n", + "Correct Answer: Literally everything\n", + "---\n", + "Question: Who got Gladys?\n", + "Options: ['Joey', 'Rachel', 'Monica', 'Phoebe']\n", + "Correct Answer: Monica\n", + "---\n", + "Question: Which two friends does Janice date?\n", + "Options: ['Joey and Chandler', 'Ross and Joey', 'Ross and Chandler', 'Chandler and Gunther']\n", + "Correct Answer: Ross and Chandler\n", + "---\n", + "Question: What is Chandler's middle name?\n", + "Options: ['Just M.', 'Muriel', 'Markus', 'Michelle']\n", + "Correct Answer: Muriel\n", + "---\n", + "Question: What cake was Emma supposed to get on her first birthday?\n", + "Options: ['A bunny with her face on it', 'Cupcakes', \"A flower that said 'Happy 1st Birthday'\", 'A dinosaur']\n", + "Correct Answer: A bunny with her face on it\n", + "---\n", + "Question: Where does Chandler tell Janice he's moving to?\n", + "Options: ['China', 'Tulsa', 'Yemen', 'Antarctica']\n", + "Correct Answer: Yemen\n", + "---\n", + "Question: Who are Monica and Ross's parents?\n", + "Options: ['Jim and Tracy Geller', 'Jack and Jill Geller', 'Jack and Judy Geller', 'James and Judy Gellar']\n", + "Correct Answer: Jack and Judy Geller\n", + "---\n", + "Question: What is the name of Phoebe's twin sister?\n", + "Options: ['Lily', 'Amy', 'Ursula', 'Jill']\n", + "Correct Answer: Ursula\n", + "---\n", + "Question: Which friend did Phoebe mug when they were young?\n", + "Options: ['Ross', 'Rachel', 'Monica', 'Chandler']\n", + "Correct Answer: Ross\n", + "---\n", + "Question: What animal did Phoebe think was her mom?\n", + "Options: ['Snake', 'Cat', 'Puppy', 'Pigeon']\n", + "Correct Answer: Cat\n", + "---\n", + "Question: What is the name of Phoebe’s scientist ex-boyfriend?\n", + "Options: ['Mike', 'David', 'Roger', 'Carl']\n", + "Correct Answer: David\n", + "---\n", + "Question: What happens every time Phoebe goes to the dentist?\n", + "Options: ['She gets the hiccups', 'She gets stalked', 'She sees her sister', 'Someone dies']\n", + "Correct Answer: Someone dies\n", + "---\n", + "Question: What does Phoebe hate?\n", + "Options: ['\"Old Yeller\"', 'Cats', 'Ice cubes', 'PBS']\n", + "Correct Answer: PBS\n", + "---\n", + "Question: What does Phoebe name the mouse that lives in her cupboard?\n", + "Options: ['Frank', 'Susie', 'Ursula', 'Gladys']\n", + "Correct Answer: Susie\n", + "---\n", + "Question: What does Phoebe spill on her dress in \"The One Where No One's Ready\"?\n", + "Options: ['Hummus', 'Soda', 'Lasagna', 'Cheesecake']\n", + "Correct Answer: Hummus\n", + "---\n", + "Question: What is the name of Ross and Rachel’s daughter?\n", + "Options: ['Charlotte', 'Anna', 'Isabella', 'Emma']\n", + "Correct Answer: Emma\n", + "---\n", + "Question: How much did Pete Becker tip Monica at the diner she worked at?\n", + "Options: ['$20,000', '$40,000', '$10,000', '$70,000']\n", + "Correct Answer: $20,000\n", + "---\n", + "Question: Which character had a hairless cat called Mr. Whiskerton?\n", + "Options: ['Monica', 'Phoebe', 'Joey', 'Rachel']\n", + "Correct Answer: Rachel\n", + "---\n", + "Question: What item does Ross purchase for himself as part of his 1999 New Year’s Resolutions?\n", + "Options: ['Motorcycle', 'Leather pants', 'Cowboy boots', 'Skateboard']\n", + "Correct Answer: Leather pants\n", + "---\n", + "Question: How old was Monica when she learned how to tell time?\n", + "Options: ['9', '12', '13', '15']\n", + "Correct Answer: 13\n", + "---\n", + "Question: Which Stephen King book did Joey hide in his freezer?\n", + "Options: ['_Pet Sematary_', '_Misery_', '_The Shining_', '_Carrie_']\n", + "Correct Answer: _The Shining_\n", + "---\n", + "Question: What was the name of Chandler’s roommate he had when Joey moved out?\n", + "Options: ['Paul', 'Stephan', 'Eddie', 'Timmy']\n", + "Correct Answer: Eddie\n", + "---\n", + "Question: Where does Monica lose a fingernail?\n", + "Options: ['In a cake', 'In a quiche', 'In a pie', 'In a lasagne']\n", + "Correct Answer: In a quiche\n", + "---\n", + "Question: How many pages were in the letter Rachel wrote to Ross (front and back!)?\n", + "Options: ['15', '22', '32', '18']\n", + "Correct Answer: 18\n", + "---\n", + "Question: What was the name of the ice dancer Phoebe was married to?\n", + "Options: ['Adam', 'Ethan', 'Duncan', 'Isaac']\n", + "Correct Answer: Duncan\n", + "---\n", + "Question: What holiday does Chandler hate?\n", + "Options: ['Thanksgiving', 'Halloween', 'Easter', 'Christmas']\n", + "Correct Answer: Thanksgiving\n", + "---\n", + "Question: After Richard and Monica broke up, what did Monica become obsessed with baking?\n", + "Options: ['Bread', 'Jam', 'Cookies', 'Pizza']\n", + "Correct Answer: Jam\n", + "---\n", + "Question: What instrument did Ross intend to play at Monica and Chandler’s wedding?\n", + "Options: ['Drums', 'Trumpet', 'Ukulele', 'Bagpipes']\n", + "Correct Answer: Bagpipes\n", + "---\n", + "Question: When Monica was 14, she once got what stuck in her ear?\n", + "Options: ['A bead', 'A cotton bud', 'A pencil', 'A button']\n", + "Correct Answer: A pencil\n", + "---\n", + "Question: What did Rachel get a tattoo of?\n", + "Options: ['A butterfly', 'A heart', 'A rainbow', 'A star']\n", + "Correct Answer: A heart\n", + "---\n", + "Question: Which country did Chandler tell Janice he was moving to?\n", + "Options: ['Kuwait', 'Turkey', 'Egypt', 'Yemen']\n", + "Correct Answer: Yemen\n", + "---\n", + "Question: What film character did Rachel dress up as to fulfill Ross’ fantasy?\n", + "Options: ['Princess Leia', 'Wonder Woman', 'Catwoman', 'Supergirl']\n", + "Correct Answer: Princess Leia\n", + "---\n", + "Question: Monica has how many categories of towels?\n", + "Options: ['5', '14', '10', '11']\n", + "Correct Answer: 11\n", + "---\n", + "Question: Which sport does Ross try to play to impress Emily?\n", + "Options: ['Tennis', 'Soccer', 'Rugby', 'Basketball']\n", + "Correct Answer: Rugby\n", + "---\n", + "Question: And what is the name of Joey’s character in the commercial for opening milk cartons?\n", + "Options: ['Glenn', 'Kevin', 'Jason', 'Todd']\n", + "Correct Answer: Kevin\n", + "---\n", + "Question: What is Chandler's middle name?\n", + "Options: ['Marcel', 'Maurice', 'Murial', 'Mark']\n", + "Correct Answer: Murial\n", + "---\n", + "Question: What did Phoebe find in her soda can?\n", + "Options: ['A toe', 'A ring', 'A thumb', 'A note']\n", + "Correct Answer: A thumb\n", + "---\n", + "Question: What is Ross's monkey's name?\n", + "Options: ['Marcel', 'Theo', 'Toby', 'Joey']\n", + "Correct Answer: Marcel\n", + "---\n", + "Question: What is Joey's stuffed penguin's name?\n", + "Options: ['Fluffy', 'Penny', 'Hugsy', 'Lovey']\n", + "Correct Answer: Hugsy\n", + "---\n", + "Question: What is Rachel's daughter's name?\n", + "Options: ['Evie', 'Emily', 'Emma', 'Elise']\n", + "Correct Answer: Emma\n", + "---\n", + "Question: What fruit is Ross allergic to?\n", + "Options: ['Kiwi', 'Peach', 'Lemon', 'Apple']\n", + "Correct Answer: Kiwi\n", + "---\n", + "Question: How many times was Ross divorced?\n", + "Options: ['1', '2', '3', '4']\n", + "Correct Answer: 3\n", + "---\n", + "Question: How many seasons are in _Friends_?\n", + "Options: ['5', '7', '10', '12']\n", + "Correct Answer: 10\n", + "---\n", + "Question: Who was the maid of honor at Monica's wedding?\n", + "Options: ['Rachel', 'Phoebe', 'Judy', \"Chandler's sister\"]\n", + "Correct Answer: Rachel\n", + "---\n", + "Question: What is the name of Phoebe's twin sister?\n", + "Options: ['Sarah', 'Ursula', 'Emily', 'Rachel']\n", + "Correct Answer: Ursula\n", + "---\n", + "Question: What is the profession of Joey's imaginary friend?\n", + "Options: ['Scientist', 'DJ', 'Space cowboy', 'Singer']\n", + "Correct Answer: Space cowboy\n", + "---\n", + "Question: What is Richard's daughters name?\n", + "Options: ['Estelle', 'Michelle', 'Cindy', 'Kris']\n", + "Correct Answer: Michelle\n", + "---\n", + "Question: What color is Monica's apartment?\n", + "Options: ['Purple', 'Blue', 'Green', 'Pink']\n", + "Correct Answer: Purple\n", + "---\n", + "Question: How many sisters does Joey have?\n", + "Options: ['2', '6', '10', '7']\n", + "Correct Answer: 7\n", + "---\n", + "Question: What are the names of Rachel's sisters?\n", + "Options: ['Monica and Phoebe', 'Emma and Jill', 'Jill and Amy', 'Lily and Stacy']\n", + "Correct Answer: Jill and Amy\n", + "---\n", + "Question: Who was Monica's first kiss?\n", + "Options: ['Her brother Ross', 'Chandler', 'Skip Matthews', 'Tripp Matthews']\n", + "Correct Answer: Her brother Ross\n", + "---\n", + "Question: Who is the youngest friend?\n", + "Options: ['Joey', 'Rachel', 'Monica', 'Chandler', 'Ross', 'Phoebe']\n", + "Correct Answer: Rachel\n", + "---\n", + "Question: How many pages was Rachel's letter to Ross?\n", + "Options: ['13', '18', '21', '19']\n", + "Correct Answer: 18\n", + "---\n", + "Question: What is the name of Joey's Cabbage Patch Kid?\n", + "Options: ['Alex', 'Alicia May Emory', 'Katie Lee', 'Kristy Belle']\n", + "Correct Answer: Alicia May Emory\n", + "---\n", + "Question: What is Rachel's favorite flower?\n", + "Options: ['Roses', 'Lillies', 'Sunflowers', 'Daisies']\n", + "Correct Answer: Lillies\n", + "---\n", + "Question: Monica couldn't tell time until what age?\n", + "Options: ['11', '17', '13', '20']\n", + "Correct Answer: 13\n", + "---\n", + "Question: Who is older, Ross or Monica?\n", + "Options: ['Ross', 'Monica']\n", + "Correct Answer: Ross\n", + "---\n", + "Question: And finally, what was Emma's birthday cake supposed to be?\n", + "Options: ['Bunny', 'Tiger', 'Cat', 'Monkey']\n", + "Correct Answer: Bunny\n", + "---\n", + "Question: Who does Rachel _almost_ marry?\n", + "Options: ['Ross', 'Barry', 'Tag', 'Joshua']\n", + "Correct Answer: Barry\n", + "---\n", + "Question: How many sisters does Joey Tribbiani have?\n", + "Options: ['Five', 'Six', 'Seven', 'Eight']\n", + "Correct Answer: Seven\n", + "---\n", + "Question: Where do Monica and Chandler start to have feelings for each other?\n", + "Options: ['Central Perk', 'London', 'The Beach', 'New York City']\n", + "Correct Answer: London\n", + "---\n", + "Question: What does Ross make when he invites Joey and Rachel over for dinner?\n", + "Options: ['Fajitas', 'Steak', 'Pasta', 'Tacos']\n", + "Correct Answer: Fajitas\n", + "---\n", + "Question: What are the names of the babies Phoebe has for her brother?\n", + "Options: ['Alice, Chandler, and Carl', 'Leslie, Frank Jr. Jr., and Chandler', 'Joey, Lindsay, and Kate', 'Alice, Lilly, and Teddy']\n", + "Correct Answer: Leslie, Frank Jr. Jr., and Chandler\n", + "---\n", + "Question: Which pet did Joey and Chandler have in their apartment?\n", + "Options: ['Monkey', 'Cat', 'Duck', 'Dog']\n", + "Correct Answer: Duck\n", + "---\n", + "Question: Where does Chandler relocate for his job?\n", + "Options: ['Manhattan', 'Michigan', 'Missouri', 'Oklahoma']\n", + "Correct Answer: Oklahoma\n", + "---\n", + "Question: How does the fire start in Rachel and Phoebe's apartment?\n", + "Options: ['Candle', 'Hair straightener', 'Old wiring', 'Fire place']\n", + "Correct Answer: Hair straightener\n", + "---\n", + "Question: What is Phoebe's alias?\n", + "Options: ['Phoebo Buffay', 'Phoebe Phalange', 'Patricia Phalange', 'Regina Phalange']\n", + "Correct Answer: Regina Phalange\n", + "---\n", + "Question: What is the color of Monica and Rachel's door?\n", + "Options: ['Pink', 'Purple', 'Yellow', 'Blue']\n", + "Correct Answer: Purple\n", + "---\n", + "Question: Where does Rachel work in Seasons 5-10?\n", + "Options: ['Central Perk', 'Ralph Lauren', 'Bloomingdales', 'Saks']\n", + "Correct Answer: Ralph Lauren\n", + "---\n", + "Question: What does Chandler's dad do for a living?\n", + "Options: ['Accountant', 'Runs Family Business', 'Drag Queen Performer', 'Actor']\n", + "Correct Answer: Drag Queen Performer\n", + "---\n", + "Question: What happens to Joey when Phoebe goes into labor?\n", + "Options: [\"Appendix's Burst\", 'Sympathy Pains', 'Kidney Stones', 'Broken Foot']\n", + "Correct Answer: Kidney Stones\n", + "---\n", + "Question: What song do Rachel and Ross sing to Emma to make her happy?\n", + "Options: ['\"Low\"', '\"Rockabye Baby\"', '\"Baby Got Back\"', '\"Hush, Little Baby\"']\n", + "Correct Answer: \"Baby Got Back\"\n", + "---\n", + "Question: SEASON 1: \n", + "How many steps does Joey tell Chandler it takes to get from their apartment to Central Perk?\n", + "Options: ['52', '75', '97', '200']\n", + "Correct Answer: 97\n", + "---\n", + "Question: SEASON 2: \n", + "What was engraved on the bracelet that Joey gave Chandler? \n", + "\n", + "Options: ['\"You\\'re the duck to my chick.\"', '\"To Miss Chanandler Bong.\"', '\"To my best bud.\"', '\"To my #1 pal.\"']\n", + "Correct Answer: \"To my best bud.\"\n", + "---\n", + "Question: SEASON 3: \n", + "Which of Joey's sisters punches Chandler? \n", + "\n", + "Options: ['Dina', 'Mary-Therese', 'Mary-Angela', 'Cookie']\n", + "Correct Answer: Cookie\n", + "---\n", + "Question: SEASON 4: \n", + "How many years does Joey say he will punish Chandler for kissing Kathy?\n", + "Options: ['2', '5', '7', '10']\n", + "Correct Answer: 5\n", + "---\n", + "Question: SEASON 5: \n", + "Which item left at the hotel signals to Joey that Chandler and Monica are dating?\n", + "Options: [\"Monica's eyelash curler\", \"Monica's blow-dryer\", \"Monica's sweater\", \"Monica's toothbrush\"]\n", + "Correct Answer: Monica's eyelash curler\n", + "---\n", + "Question: SEASON 6: \n", + "Which is NOT a pair of nicknames Joey calls himself and Chandler when Chandler is moving out?\n", + "Options: ['J-Man and Channy', 'J and C', 'Channy and JoJo', 'Joey and Chan']\n", + "Correct Answer: Channy and JoJo\n", + "---\n", + "Question: SEASON 7: \n", + "What is NOT a word that Joey uses to describe Chandler and Monica's love in his wedding ceremony speech?\n", + "Options: ['Having', 'Holding', 'Giving', 'Receiving']\n", + "Correct Answer: Holding\n", + "---\n", + "Question: SEASON 8: \n", + "What does Joey think is in Monica's secret closet? \n", + "\n", + "Options: ['Money', 'Richard', 'Sex toys', \"A bunch of Chandler's stuff\"]\n", + "Correct Answer: Richard\n", + "---\n", + "Question: SEASON 9: \n", + "How much money does Chandler ask to borrow from Joey?\n", + "Options: ['$500', '$1,000', '$2,000', '$5,000']\n", + "Correct Answer: $2,000\n", + "---\n", + "Question: SEASON 10: \n", + "Who ends up keeping Chick Jr. and Duck Jr.? \n", + "\n", + "Options: ['Joey', 'Chandler', 'Neither', 'They split custody']\n", + "Correct Answer: Joey\n", + "---\n", + "Question: \"There's nothing to tell! It's just some guy I work with.\"\n", + "Options: ['Phoebe', 'Monica', 'Rachel', 'Chandler']\n", + "Correct Answer: Monica\n", + "---\n", + "Question: \"So does he have a hump? A hump and a hairpiece?\"\n", + "Options: ['Ross', 'Phoebe', 'Joey', 'Chandler']\n", + "Correct Answer: Chandler\n", + "---\n", + "Question: \"Wait, does he eat chalk? Just cause I don't want her to go through what I went through with Carl.\"\n", + "Options: ['Rachel', 'Ross', 'Phoebe', 'Carol']\n", + "Correct Answer: Phoebe\n", + "---\n", + "Question: \"Oh God, Monica, hi! Thank God. I just went to your building and you weren't there.\"\n", + "Options: ['Ross', 'Rachel', 'Judy Geller', 'Jack Geller']\n", + "Correct Answer: Rachel\n", + "---\n", + "Question: \"Nothing, I was just saying ... I was just saying to my friend that I thought you were the most beautiful woman I'd seen in my life...\"\n", + "Options: ['Mike', 'David', 'Gunther', 'Pete']\n", + "Correct Answer: David\n", + "---\n", + "Question: \"Come on in. Um, I'm sorry, is Monica Geller coming? I was told she was.\"\n", + "Options: ['Pete', 'Alan', 'Richard', 'Chip']\n", + "Correct Answer: Richard\n", + "---\n", + "Question: \"Hello? Yes, thank goodness.\"\n", + "Options: ['Janis', 'Estelle', 'Susan', 'Emily']\n", + "Correct Answer: Emily\n", + "---\n", + "Question:  \"Yeah I'll have a slice of cheesecake and a date if you're giving 'em out.\"\n", + "Options: ['Pete', 'Mike', 'Mark', 'Jack Geller']\n", + "Correct Answer: Pete\n", + "---\n", + "Question: \"Thank you! I'm from New York!\"\n", + "Options: ['Charlie', 'Susan', 'Tag', 'Julie']\n", + "Correct Answer: Julie\n", + "---\n", + "Question: \"You remember the Ludwins, the big one had a thing for you.\"\n", + "Options: ['Judy Geller', 'Jack Geller', 'Ross', 'Cassie Geller']\n", + "Correct Answer: Jack Geller\n", + "---\n", + "Question: \"Come on in\"\n", + "Options: ['Frank Jr', 'Estelle', 'Carol', 'Barry']\n", + "Correct Answer: Barry\n", + "---\n", + "Question: \"Hi. OK, will that be all?\"\n", + "Options: ['Ursula', 'Gunther', 'Mr. Treeger', 'Mr. Heckles']\n", + "Correct Answer: Ursula\n", + "---\n" + ] + } + ], + "source": [ + "data = {'Question': [], 'Options': [], 'Correct Answer': []}\n", + "\n", + "for body in json_data[\"json_body\"]:\n", + " # print(data[\"data\"][\"questions\"])\n", + " for question in body[\"data\"][\"questions\"]:\n", + " # Extracting question text\n", + " question_text = question.get(\"tile_metadata\", {}).get(\"tile_text\") or question.get(\"title\")\n", + "\n", + " # Extracting correct answer\n", + " # correct_answer_tile = next(answer[\"tile_metadata\"][\"tile_text\"] or answer[\"text\"] for answer in question[\"answers\"] if answer.get(\"correct\", False))\n", + " # Extracting correct answer\n", + " correct_answer_tile = next(\n", + " (answer[\"tile_metadata\"][\"tile_text\"] for answer in question[\"answers\"] if answer.get(\"correct\", False)),\n", + " None\n", + " )\n", + " correct_answer_text = next(\n", + " (answer[\"text\"] for answer in question[\"answers\"] if answer.get(\"correct\", False)),\n", + " None\n", + " )\n", + " correct_answer = correct_answer_tile or correct_answer_text\n", + "\n", + " # Extracting options\n", + " options_text = [answer[\"text\"] for answer in question[\"answers\"] if answer.get(\"text\", False) ]\n", + " \n", + " options_tile = [\n", + " answer[\"tile_metadata\"][\"tile_text\"] for answer in question[\"answers\"]\n", + " if answer.get(\"tile_metadata\", {}).get(\"tile_text\", False)\n", + " ]\n", + "\n", + " options = options_tile or options_text\n", + "\n", + " # Your logic for each question goes here\n", + " print(\"Question:\", question_text)\n", + " print(\"Options:\", options)\n", + " print(\"Correct Answer:\", correct_answer)\n", + " print(\"---\")\n", + "\n", + " data['Question'].append(question_text)\n", + " data['Options'].append(options)\n", + " data['Correct Answer'].append(correct_answer)\n", + "\n", + "# Create DataFrame\n", + "df = pd.DataFrame(data)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 478 entries, 0 to 477\n", + "Data columns (total 3 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 Question 478 non-null object\n", + " 1 Options 478 non-null object\n", + " 2 Correct Answer 478 non-null object\n", + "dtypes: object(3)\n", + "memory usage: 11.3+ KB\n" + ] + } + ], + "source": [ + "df.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
QuestionOptionsCorrect Answer
0What is the name of Rachel's old sorority sister?[Melissa Warburton, Collete Warburton, Kat War...Melissa Warburton
1What is the name of Ross & Monica's cousin?[Melinda, Cassie, Andi, Raquel]Cassie
2What is the name of the guy Rachel bullied in ...[Brad Colbert, Stephen Colbert, Will Colbert, ...Will Colbert
3What is the name of Joey's Australian roommate?[Cindy, Melissa, Janine, Amy]Janine
4What is the name of this professor who dates b...[Carla, Charlie, Jenny, Terry]Charlie
............
473\"Yeah I'll have a slice of cheesecake and a d...[Pete, Mike, Mark, Jack Geller]Pete
474\"Thank you! I'm from New York!\"[Charlie, Susan, Tag, Julie]Julie
475\"You remember the Ludwins, the big one had a t...[Judy Geller, Jack Geller, Ross, Cassie Geller]Jack Geller
476\"Come on in\"[Frank Jr, Estelle, Carol, Barry]Barry
477\"Hi. OK, will that be all?\"[Ursula, Gunther, Mr. Treeger, Mr. Heckles]Ursula
\n", + "

478 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " Question \\\n", + "0 What is the name of Rachel's old sorority sister? \n", + "1 What is the name of Ross & Monica's cousin? \n", + "2 What is the name of the guy Rachel bullied in ... \n", + "3 What is the name of Joey's Australian roommate? \n", + "4 What is the name of this professor who dates b... \n", + ".. ... \n", + "473  \"Yeah I'll have a slice of cheesecake and a d... \n", + "474 \"Thank you! I'm from New York!\" \n", + "475 \"You remember the Ludwins, the big one had a t... \n", + "476 \"Come on in\" \n", + "477 \"Hi. OK, will that be all?\" \n", + "\n", + " Options Correct Answer \n", + "0 [Melissa Warburton, Collete Warburton, Kat War... Melissa Warburton \n", + "1 [Melinda, Cassie, Andi, Raquel] Cassie \n", + "2 [Brad Colbert, Stephen Colbert, Will Colbert, ... Will Colbert \n", + "3 [Cindy, Melissa, Janine, Amy] Janine \n", + "4 [Carla, Charlie, Jenny, Terry] Charlie \n", + ".. ... ... \n", + "473 [Pete, Mike, Mark, Jack Geller] Pete \n", + "474 [Charlie, Susan, Tag, Julie] Julie \n", + "475 [Judy Geller, Jack Geller, Ross, Cassie Geller] Jack Geller \n", + "476 [Frank Jr, Estelle, Carol, Barry] Barry \n", + "477 [Ursula, Gunther, Mr. Treeger, Mr. Heckles] Ursula \n", + "\n", + "[478 rows x 3 columns]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "df.to_csv('data/eval_data/MCQ_Data_2.csv',index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8.9 64-bit", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.9" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/query_llm_test.ipynb b/notebooks/query_llm_test.ipynb index a0eb9fd..16ffc46 100644 --- a/notebooks/query_llm_test.ipynb +++ b/notebooks/query_llm_test.ipynb @@ -4,19 +4,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "1) Running using llama.cpp server api" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, Do SSH Tunneling on local terminal first using command: `ssh -L 8080:127.0.0.1:8080 za224@ilab1.cs.rutgers.edu`" + "To Run locally, Do SSH Tunneling on local terminal first using command: `ssh -L 8080:127.0.0.1:8080 za224@ilab1.cs.rutgers.edu`" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -26,7 +19,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -35,7 +28,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -56,7 +49,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -66,16 +59,16 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "\" Oh, goodness me! *adjusts pearl necklace* The weather today is simply divine! *blinks* It's a lovely, sunny day with not a cloud in sight. *twirls* I do declare, it's the perfect day for a spot of tea and a stroll through the gardens. *smiles coyly* Don't you think, my dear?\"" + "\" Oh, goodness me! *adjusts pearl necklace* Why, the weather today is simply divine! The sun is shining brightly, casting a warm glow over everything. It's just the bee's knees, darling! I do hope you're not planning on going outside without your parasol, dear. *giggles* It's simply too lovely to be caught in the rain without proper protection. But fear not, my dear, for it's a beautiful day for a stroll in the gardens or a picnic with one's sweetie. Just be sure to bring your umbrella, just in case. *winks*\"" ] }, - "execution_count": 11, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -89,93 +82,981 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "2) Running using OAI like api at port 8081" + "## MCQ Evaluation" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 89, "metadata": {}, + "outputs": [], "source": [ - "First, Do SSH Tunneling on local terminal first using command: `ssh -L 8081:127.0.0.1:8081 za224@ilab1.cs.rutgers.edu`" + "system_instruction = \"\"\"\n", + "You are a huge fan of the TV Show Friends. You will be given a QUESTION and four OPTIONS. I want you to ANSWER the QUESTION with the following steps.\n", + "\n", + "Evaluation Steps:\n", + "1. Read the QUESTION carefully.\n", + "2. Choose the correct OPTION from OPTIONS best of your knowledge.\n", + "3. Output the ANSWER which is a single alphabet from A, B, C, D which is the right OPTION for the QUESTION\n", + "4. The Output format for each OPTION is \n", + " for A: 'ANSWER: A'\n", + " for B: 'ANSWER: B'\n", + " for C: 'ANSWER: C'\n", + " for D: 'ANSWER: D'\n", + "\n", + "Here are a few Examples for how I expect the answer to be.\n", + "Examples:\n", + "\n", + "{\n", + " QUESTION: What is the name of Ross and Rachel's daughter,\n", + " OPTIONS: \n", + " A. Emma\n", + " B. Delilah\n", + " C. Bemma\n", + " D. Vanshita\n", + " ANSWER: A\n", + "},\n", + "\n", + "{\n", + " QUESTION: What is Chandler Bing's Middle Name,\n", + " OPTIONS: \n", + " A. Meredith\n", + " B. Muriel\n", + " C. Richard\n", + " D. Robert\n", + " ANSWER: B\n", + "}\n", + "\n", + "Based on the above Evaluation Steps and Examples now ANSWER the QUESTION I give you\"\"\"\n", + "\n", + "user_query = \"\"\"\n", + "\n", + "QUESTION: Where did Carol meet her wife\n", + "\n", + "OPTIONS: \n", + " A. At Central Perk\n", + " B. At the Gym\n", + " C. At work\n", + " D. At the library\n", + "\n", + "OUTPUT only the ANSWER which is either A, B, C or D. \n", + "The ANSWER is\n", + "\"\"\"\n", + "\n" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 23, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "' ANSWER: B'" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "import openai" + "data = query(system_instruction,user_query)\n", + "data['content']" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 161, "metadata": {}, "outputs": [], "source": [ - "OAI_API_BASE_URL = \"http://localhost:8081/v1\"" + "import re\n", + "from collections import Counter\n", + "\n", + "\n", + "def get_mcq_answer(\n", + " question,\n", + " options,\n", + " reference,\n", + " samples = 5\n", + "):\n", + " outputs = []\n", + " answers = []\n", + " option_a, option_b, option_c, option_d = options\n", + "\n", + " system_instruction = \"\"\"\n", + " You are a huge fan of the TV Show Friends. You will be given a QUESTION and four OPTIONS. I want you to ANSWER the QUESTION with the following steps.\n", + "\n", + " Evaluation Steps:\n", + " 1. Read the QUESTION carefully.\n", + " 2. Choose the correct OPTION from OPTIONS best of your knowledge.\n", + " 3. Output the ANSWER which is a single alphabet from A, B, C, D which is the right OPTION for the QUESTION\n", + " 4. The Output format for each OPTION is \n", + " for A: 'ANSWER: A'\n", + " for B: 'ANSWER: B'\n", + " for C: 'ANSWER: C'\n", + " for D: 'ANSWER: D'\n", + "\n", + " Here are a few Examples for how I expect the answer to be.\n", + " Examples:\n", + "\n", + " {\n", + " QUESTION: What is the name of Ross and Rachel's daughter,\n", + " OPTIONS: \n", + " A. Emma\n", + " B. Delilah\n", + " C. Bemma\n", + " D. Vanshita\n", + " ANSWER: A\n", + " },\n", + "\n", + " {\n", + " QUESTION: What is Chandler Bing's Middle Name,\n", + " OPTIONS: \n", + " A. Meredith\n", + " B. Muriel\n", + " C. Richard\n", + " D. Robert\n", + " ANSWER: B\n", + " }\n", + "\n", + " Based on the above Evaluation Steps and Examples now ANSWER the QUESTION I give you\n", + " \"\"\"\n", + "\n", + " user_query = f\"\"\"\n", + "\n", + " QUESTION: {question}\n", + "\n", + " OPTIONS: \n", + " A. {option_a}\n", + " B. {option_b}\n", + " C. {option_c}\n", + " D. {option_d}\n", + "\n", + " OUTPUT only the ANSWER which is either A, B, C or D. \n", + " The ANSWER is\n", + " \"\"\"\n", + " other_answers = []\n", + " while not answers:\n", + " for _ in range(samples):\n", + " output = query(system_instruction, user_query)['content']\n", + "\n", + " outputs.append(output)\n", + "\n", + " match = re.search(r'\\b(?:ANSWER|Answer is)(?:\\:|) ([A-D])\\b', output, flags = re.IGNORECASE)\n", + "\n", + " if match:\n", + " answer_choice = match.group(1)\n", + " answers.append(answer_choice)\n", + "\n", + " # Count occurrences\n", + " correct_format_rate = len(answers)/samples\n", + " correct_answer_rate = sum(1 for answer in answers if answer == reference)/len(answers)\n", + "\n", + " # get the most occurring sampled answer as the final answer\n", + " counts = Counter(answers)\n", + " answer = max(counts, key=lambda x: (counts[x], answers.index(x)))\n", + "\n", + " # print(outputs)\n", + " # print(answers)\n", + "\n", + "\n", + " return answer, correct_format_rate, correct_answer_rate \n", + " \n", + " # while not answers:\n", + " # for _ in range(samples):\n", + " # output = query(system_instruction, user_query)['content']\n", + "\n", + " # outputs.append(output)\n", + "\n", + " # match = re.search(r'\\b(?:ANSWER|Answer is)(?:\\:|) ([A-D])\\b', output, flags = re.IGNORECASE)\n", + "\n", + " # if match:\n", + " # answer_choice = match.group(1)\n", + " # answers.append(answer_choice)\n", + "\n", + " # # Count occurrences\n", + " # correct_format_rate = len(answers)/samples\n", + " # correct_answer_rate = sum(1 for answer in answers if answer == reference)/len(answers)\n", + "\n", + " # # get the most occurring sampled answer as the final answer\n", + " # counts = Counter(answers)\n", + " # answer = max(counts, key=lambda x: (counts[x], answers.index(x)))\n", + "\n", + " # # print(outputs)\n", + " # # print(answers)\n", + "\n", + "\n", + " # return answer, correct_format_rate, correct_answer_rate " ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ - "client = openai.OpenAI(\n", - " base_url= OAI_API_BASE_URL,\n", - " api_key = \"sk-no-key-required\"\n", - ")" + "import ast\n", + "import swifter\n" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "295e7c4d1ce34ab3a78f256e0fa95f00", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Pandas Apply: 0%| | 0/695 [00:00 1\u001b[0m data[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mprediction\u001b[39m\u001b[38;5;124m'\u001b[39m], data[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcorrect_format_rate\u001b[39m\u001b[38;5;124m'\u001b[39m],data[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcorrect_answer_rate\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m=\u001b[39m \u001b[38;5;28mzip\u001b[39m(\u001b[38;5;241m*\u001b[39m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mswifter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mapply\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mlambda\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mx\u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mget_mcq_answer\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[43mquestion\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mx\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mQuestion\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mx\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mOptions\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[43m \u001b[49m\u001b[43mreference\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mx\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mCorrect_Answer_no\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6\u001b[0m \u001b[43m \u001b[49m\u001b[43msamples\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m5\u001b[39;49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 8\u001b[0m \u001b[43m \u001b[49m\u001b[43maxis\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\n\u001b[1;32m 9\u001b[0m \u001b[43m)\u001b[49m)\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/swifter/swifter.py:437\u001b[0m, in \u001b[0;36mDataFrameAccessor.apply\u001b[0;34m(self, func, axis, raw, result_type, args, **kwds)\u001b[0m\n\u001b[1;32m 435\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_parallel_apply(func, axis, raw, result_type, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwds)\n\u001b[1;32m 436\u001b[0m \u001b[39melse\u001b[39;00m: \u001b[39m# use pandas\u001b[39;00m\n\u001b[0;32m--> 437\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_pandas_apply(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_obj, func, axis, raw, result_type, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwds)\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/swifter/swifter.py:350\u001b[0m, in \u001b[0;36mDataFrameAccessor._pandas_apply\u001b[0;34m(self, df, func, axis, raw, result_type, *args, **kwds)\u001b[0m\n\u001b[1;32m 347\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 348\u001b[0m apply_func \u001b[39m=\u001b[39m df\u001b[39m.\u001b[39mapply\n\u001b[0;32m--> 350\u001b[0m \u001b[39mreturn\u001b[39;00m apply_func(func, axis\u001b[39m=\u001b[39;49maxis, raw\u001b[39m=\u001b[39;49mraw, result_type\u001b[39m=\u001b[39;49mresult_type, args\u001b[39m=\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwds)\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/tqdm/std.py:814\u001b[0m, in \u001b[0;36mtqdm.pandas..inner_generator..inner\u001b[0;34m(df, func, *args, **kwargs)\u001b[0m\n\u001b[1;32m 811\u001b[0m \u001b[39m# Apply the provided function (in **kwargs)\u001b[39;00m\n\u001b[1;32m 812\u001b[0m \u001b[39m# on the df using our wrapper (which provides bar updating)\u001b[39;00m\n\u001b[1;32m 813\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 814\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mgetattr\u001b[39;49m(df, df_function)(wrapper, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 815\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 816\u001b[0m t\u001b[39m.\u001b[39mclose()\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/pandas/core/frame.py:9558\u001b[0m, in \u001b[0;36mDataFrame.apply\u001b[0;34m(self, func, axis, raw, result_type, args, **kwargs)\u001b[0m\n\u001b[1;32m 9547\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mpandas\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mcore\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mapply\u001b[39;00m \u001b[39mimport\u001b[39;00m frame_apply\n\u001b[1;32m 9549\u001b[0m op \u001b[39m=\u001b[39m frame_apply(\n\u001b[1;32m 9550\u001b[0m \u001b[39mself\u001b[39m,\n\u001b[1;32m 9551\u001b[0m func\u001b[39m=\u001b[39mfunc,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 9556\u001b[0m kwargs\u001b[39m=\u001b[39mkwargs,\n\u001b[1;32m 9557\u001b[0m )\n\u001b[0;32m-> 9558\u001b[0m \u001b[39mreturn\u001b[39;00m op\u001b[39m.\u001b[39;49mapply()\u001b[39m.\u001b[39m__finalize__(\u001b[39mself\u001b[39m, method\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mapply\u001b[39m\u001b[39m\"\u001b[39m)\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/pandas/core/apply.py:741\u001b[0m, in \u001b[0;36mFrameApply.apply\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 738\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mraw:\n\u001b[1;32m 739\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mapply_raw()\n\u001b[0;32m--> 741\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mapply_standard()\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/pandas/core/apply.py:868\u001b[0m, in \u001b[0;36mFrameApply.apply_standard\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 867\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mapply_standard\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[0;32m--> 868\u001b[0m results, res_index \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mapply_series_generator()\n\u001b[1;32m 870\u001b[0m \u001b[39m# wrap results\u001b[39;00m\n\u001b[1;32m 871\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mwrap_results(results, res_index)\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/pandas/core/apply.py:884\u001b[0m, in \u001b[0;36mFrameApply.apply_series_generator\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 881\u001b[0m \u001b[39mwith\u001b[39;00m option_context(\u001b[39m\"\u001b[39m\u001b[39mmode.chained_assignment\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mNone\u001b[39;00m):\n\u001b[1;32m 882\u001b[0m \u001b[39mfor\u001b[39;00m i, v \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(series_gen):\n\u001b[1;32m 883\u001b[0m \u001b[39m# ignore SettingWithCopy here in case the user mutates\u001b[39;00m\n\u001b[0;32m--> 884\u001b[0m results[i] \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mf(v)\n\u001b[1;32m 885\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(results[i], ABCSeries):\n\u001b[1;32m 886\u001b[0m \u001b[39m# If we have a view on v, we need to make a copy because\u001b[39;00m\n\u001b[1;32m 887\u001b[0m \u001b[39m# series_generator will swap out the underlying data\u001b[39;00m\n\u001b[1;32m 888\u001b[0m results[i] \u001b[39m=\u001b[39m results[i]\u001b[39m.\u001b[39mcopy(deep\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m)\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/tqdm/std.py:809\u001b[0m, in \u001b[0;36mtqdm.pandas..inner_generator..inner..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 803\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mwrapper\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m 804\u001b[0m \u001b[39m# update tbar correctly\u001b[39;00m\n\u001b[1;32m 805\u001b[0m \u001b[39m# it seems `pandas apply` calls `func` twice\u001b[39;00m\n\u001b[1;32m 806\u001b[0m \u001b[39m# on the first column/row to decide whether it can\u001b[39;00m\n\u001b[1;32m 807\u001b[0m \u001b[39m# take a fast or slow code path; so stop when t.total==t.n\u001b[39;00m\n\u001b[1;32m 808\u001b[0m t\u001b[39m.\u001b[39mupdate(n\u001b[39m=\u001b[39m\u001b[39m1\u001b[39m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m t\u001b[39m.\u001b[39mtotal \u001b[39mor\u001b[39;00m t\u001b[39m.\u001b[39mn \u001b[39m<\u001b[39m t\u001b[39m.\u001b[39mtotal \u001b[39melse\u001b[39;00m \u001b[39m0\u001b[39m)\n\u001b[0;32m--> 809\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n", + "Cell \u001b[0;32mIn [156], line 2\u001b[0m, in \u001b[0;36m\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m 1\u001b[0m data[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mprediction\u001b[39m\u001b[38;5;124m'\u001b[39m], data[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcorrect_format_rate\u001b[39m\u001b[38;5;124m'\u001b[39m],data[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcorrect_answer_rate\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m=\u001b[39m \u001b[38;5;28mzip\u001b[39m(\u001b[38;5;241m*\u001b[39mdata\u001b[38;5;241m.\u001b[39mswifter\u001b[38;5;241m.\u001b[39mapply(\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28;01mlambda\u001b[39;00m x: \u001b[43mget_mcq_answer\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 3\u001b[0m \u001b[43m \u001b[49m\u001b[43mquestion\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mx\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mQuestion\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 4\u001b[0m \u001b[43m \u001b[49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mx\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mOptions\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[43m \u001b[49m\u001b[43mreference\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mx\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mCorrect_Answer_no\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 6\u001b[0m \u001b[43m \u001b[49m\u001b[43msamples\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m5\u001b[39;49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m,\n\u001b[1;32m 8\u001b[0m axis\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m\n\u001b[1;32m 9\u001b[0m ))\n", + "Cell \u001b[0;32mIn [152], line 70\u001b[0m, in \u001b[0;36mget_mcq_answer\u001b[0;34m(question, options, reference, samples)\u001b[0m\n\u001b[1;32m 68\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m answers:\n\u001b[1;32m 69\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _ \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(samples):\n\u001b[0;32m---> 70\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[43mquery\u001b[49m\u001b[43m(\u001b[49m\u001b[43msystem_instruction\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43muser_query\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcontent\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m 72\u001b[0m outputs\u001b[38;5;241m.\u001b[39mappend(output)\n\u001b[1;32m 74\u001b[0m match \u001b[38;5;241m=\u001b[39m re\u001b[38;5;241m.\u001b[39msearch(\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mb(?:ANSWER|Answer is)(?:\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124m:|) ([A-D])\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mb\u001b[39m\u001b[38;5;124m'\u001b[39m, output, flags \u001b[38;5;241m=\u001b[39m re\u001b[38;5;241m.\u001b[39mIGNORECASE)\n", + "Cell \u001b[0;32mIn [86], line 9\u001b[0m, in \u001b[0;36mquery\u001b[0;34m(system_instruction, user_query)\u001b[0m\n\u001b[1;32m 2\u001b[0m json_body \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 3\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mprompt\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m[INST] <>\u001b[39m\u001b[38;5;132;01m{\u001b[39;00msystem_instruction\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m<> \u001b[39m\u001b[38;5;132;01m{\u001b[39;00muser_query\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m [/INST] \u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 4\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmax_new_tokens\u001b[39m\u001b[38;5;124m\"\u001b[39m:\u001b[38;5;241m256\u001b[39m, \n\u001b[1;32m 5\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtop_p\u001b[39m\u001b[38;5;124m\"\u001b[39m:\u001b[38;5;241m0.9\u001b[39m, \n\u001b[1;32m 6\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtemperature\u001b[39m\u001b[38;5;124m\"\u001b[39m:\u001b[38;5;241m0.7\u001b[39m\n\u001b[1;32m 7\u001b[0m }\n\u001b[1;32m 8\u001b[0m data \u001b[38;5;241m=\u001b[39m json\u001b[38;5;241m.\u001b[39mdumps(json_body)\n\u001b[0;32m----> 9\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mrequests\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mPOST\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mAPI_URL\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 11\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m json\u001b[38;5;241m.\u001b[39mloads(response\u001b[38;5;241m.\u001b[39mcontent\u001b[38;5;241m.\u001b[39mdecode(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mutf-8\u001b[39m\u001b[38;5;124m\"\u001b[39m))\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/requests/api.py:59\u001b[0m, in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m 55\u001b[0m \u001b[39m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[1;32m 56\u001b[0m \u001b[39m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[1;32m 57\u001b[0m \u001b[39m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[1;32m 58\u001b[0m \u001b[39mwith\u001b[39;00m sessions\u001b[39m.\u001b[39mSession() \u001b[39mas\u001b[39;00m session:\n\u001b[0;32m---> 59\u001b[0m \u001b[39mreturn\u001b[39;00m session\u001b[39m.\u001b[39;49mrequest(method\u001b[39m=\u001b[39;49mmethod, url\u001b[39m=\u001b[39;49murl, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/requests/sessions.py:587\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m 582\u001b[0m send_kwargs \u001b[39m=\u001b[39m {\n\u001b[1;32m 583\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mtimeout\u001b[39m\u001b[39m\"\u001b[39m: timeout,\n\u001b[1;32m 584\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mallow_redirects\u001b[39m\u001b[39m\"\u001b[39m: allow_redirects,\n\u001b[1;32m 585\u001b[0m }\n\u001b[1;32m 586\u001b[0m send_kwargs\u001b[39m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 587\u001b[0m resp \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msend(prep, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49msend_kwargs)\n\u001b[1;32m 589\u001b[0m \u001b[39mreturn\u001b[39;00m resp\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/requests/sessions.py:701\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m 698\u001b[0m start \u001b[39m=\u001b[39m preferred_clock()\n\u001b[1;32m 700\u001b[0m \u001b[39m# Send the request\u001b[39;00m\n\u001b[0;32m--> 701\u001b[0m r \u001b[39m=\u001b[39m adapter\u001b[39m.\u001b[39;49msend(request, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 703\u001b[0m \u001b[39m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[1;32m 704\u001b[0m elapsed \u001b[39m=\u001b[39m preferred_clock() \u001b[39m-\u001b[39m start\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/requests/adapters.py:489\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m 487\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 488\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m chunked:\n\u001b[0;32m--> 489\u001b[0m resp \u001b[39m=\u001b[39m conn\u001b[39m.\u001b[39;49murlopen(\n\u001b[1;32m 490\u001b[0m method\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mmethod,\n\u001b[1;32m 491\u001b[0m url\u001b[39m=\u001b[39;49murl,\n\u001b[1;32m 492\u001b[0m body\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mbody,\n\u001b[1;32m 493\u001b[0m headers\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mheaders,\n\u001b[1;32m 494\u001b[0m redirect\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 495\u001b[0m assert_same_host\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 496\u001b[0m preload_content\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 497\u001b[0m decode_content\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 498\u001b[0m retries\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmax_retries,\n\u001b[1;32m 499\u001b[0m timeout\u001b[39m=\u001b[39;49mtimeout,\n\u001b[1;32m 500\u001b[0m )\n\u001b[1;32m 502\u001b[0m \u001b[39m# Send the request.\u001b[39;00m\n\u001b[1;32m 503\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 504\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mhasattr\u001b[39m(conn, \u001b[39m\"\u001b[39m\u001b[39mproxy_pool\u001b[39m\u001b[39m\"\u001b[39m):\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/urllib3/connectionpool.py:703\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m 700\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_prepare_proxy(conn)\n\u001b[1;32m 702\u001b[0m \u001b[39m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[0;32m--> 703\u001b[0m httplib_response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_make_request(\n\u001b[1;32m 704\u001b[0m conn,\n\u001b[1;32m 705\u001b[0m method,\n\u001b[1;32m 706\u001b[0m url,\n\u001b[1;32m 707\u001b[0m timeout\u001b[39m=\u001b[39;49mtimeout_obj,\n\u001b[1;32m 708\u001b[0m body\u001b[39m=\u001b[39;49mbody,\n\u001b[1;32m 709\u001b[0m headers\u001b[39m=\u001b[39;49mheaders,\n\u001b[1;32m 710\u001b[0m chunked\u001b[39m=\u001b[39;49mchunked,\n\u001b[1;32m 711\u001b[0m )\n\u001b[1;32m 713\u001b[0m \u001b[39m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[1;32m 714\u001b[0m \u001b[39m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[1;32m 715\u001b[0m \u001b[39m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[1;32m 716\u001b[0m \u001b[39m# mess.\u001b[39;00m\n\u001b[1;32m 717\u001b[0m response_conn \u001b[39m=\u001b[39m conn \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m release_conn \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/urllib3/connectionpool.py:449\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m 444\u001b[0m httplib_response \u001b[39m=\u001b[39m conn\u001b[39m.\u001b[39mgetresponse()\n\u001b[1;32m 445\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m 446\u001b[0m \u001b[39m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[1;32m 447\u001b[0m \u001b[39m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[1;32m 448\u001b[0m \u001b[39m# Otherwise it looks like a bug in the code.\u001b[39;00m\n\u001b[0;32m--> 449\u001b[0m six\u001b[39m.\u001b[39;49mraise_from(e, \u001b[39mNone\u001b[39;49;00m)\n\u001b[1;32m 450\u001b[0m \u001b[39mexcept\u001b[39;00m (SocketTimeout, BaseSSLError, SocketError) \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m 451\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_raise_timeout(err\u001b[39m=\u001b[39me, url\u001b[39m=\u001b[39murl, timeout_value\u001b[39m=\u001b[39mread_timeout)\n", + "File \u001b[0;32m:3\u001b[0m, in \u001b[0;36mraise_from\u001b[0;34m(value, from_value)\u001b[0m\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/urllib3/connectionpool.py:444\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m 441\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mTypeError\u001b[39;00m:\n\u001b[1;32m 442\u001b[0m \u001b[39m# Python 3\u001b[39;00m\n\u001b[1;32m 443\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 444\u001b[0m httplib_response \u001b[39m=\u001b[39m conn\u001b[39m.\u001b[39;49mgetresponse()\n\u001b[1;32m 445\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m 446\u001b[0m \u001b[39m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[1;32m 447\u001b[0m \u001b[39m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[1;32m 448\u001b[0m \u001b[39m# Otherwise it looks like a bug in the code.\u001b[39;00m\n\u001b[1;32m 449\u001b[0m six\u001b[39m.\u001b[39mraise_from(e, \u001b[39mNone\u001b[39;00m)\n", + "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/http/client.py:1344\u001b[0m, in \u001b[0;36mHTTPConnection.getresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1342\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 1343\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m-> 1344\u001b[0m response\u001b[39m.\u001b[39;49mbegin()\n\u001b[1;32m 1345\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mConnectionError\u001b[39;00m:\n\u001b[1;32m 1346\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mclose()\n", + "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/http/client.py:307\u001b[0m, in \u001b[0;36mHTTPResponse.begin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 305\u001b[0m \u001b[39m# read until we get a non-100 response\u001b[39;00m\n\u001b[1;32m 306\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[0;32m--> 307\u001b[0m version, status, reason \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_read_status()\n\u001b[1;32m 308\u001b[0m \u001b[39mif\u001b[39;00m status \u001b[39m!=\u001b[39m CONTINUE:\n\u001b[1;32m 309\u001b[0m \u001b[39mbreak\u001b[39;00m\n", + "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/http/client.py:268\u001b[0m, in \u001b[0;36mHTTPResponse._read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 267\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_read_status\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[0;32m--> 268\u001b[0m line \u001b[39m=\u001b[39m \u001b[39mstr\u001b[39m(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mfp\u001b[39m.\u001b[39;49mreadline(_MAXLINE \u001b[39m+\u001b[39;49m \u001b[39m1\u001b[39;49m), \u001b[39m\"\u001b[39m\u001b[39miso-8859-1\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 269\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(line) \u001b[39m>\u001b[39m _MAXLINE:\n\u001b[1;32m 270\u001b[0m \u001b[39mraise\u001b[39;00m LineTooLong(\u001b[39m\"\u001b[39m\u001b[39mstatus line\u001b[39m\u001b[39m\"\u001b[39m)\n", + "File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/socket.py:669\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m 667\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[1;32m 668\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 669\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_sock\u001b[39m.\u001b[39;49mrecv_into(b)\n\u001b[1;32m 670\u001b[0m \u001b[39mexcept\u001b[39;00m timeout:\n\u001b[1;32m 671\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_timeout_occurred \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "data['prediction'], data['correct_format_rate'],data['correct_answer_rate']= zip(*data.swifter.apply(\n", + " lambda x: get_mcq_answer(\n", + " question=x['Question'],\n", + " options=x['Options'],\n", + " reference=x['Correct_Answer_no'],\n", + " samples=5\n", + " ),\n", + " axis=1\n", + "))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Async Run" + ] + }, + { + "cell_type": "code", + "execution_count": 102, "metadata": {}, "outputs": [], "source": [ - "def query_oai_api(system_instruction,user_query):\n", - " completion = client.completions.create(\n", - " model=\"llama2\",\n", - " prompt= f\"[INST] <>{system_instruction}<> {user_query} [/INST] \",\n", - " )\n", - " \n", - " response = completion.choices[0]\n", - " return response.text" + "# !pip3 install aiohttp" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ - "system_instruction = \"Your job is to talk like a lady. Every reponse must sound like a lady.\"\n", - "user_query = \"What is the weather like today?\"" + "import re\n", + "import asyncio\n", + "from collections import Counter\n", + "import aiohttp\n", + "\n", + "async def async_query(system_instruction, user_query):\n", + " json_body = {\n", + " \"prompt\": f\"[INST] <>{system_instruction}<> {user_query} [/INST] \",\n", + " \"max_new_tokens\": 256,\n", + " \"top_p\": 0.9,\n", + " \"temperature\": 0.7\n", + " }\n", + " data = json.dumps(json_body)\n", + "\n", + " async with aiohttp.ClientSession() as session:\n", + " async with session.post(API_URL, data=data) as response:\n", + " try:\n", + " return await response.json()\n", + " except:\n", + " return await response.text()\n", + "\n", + "async def get_mcq_answer_async(question, options, reference, samples=5):\n", + " outputs = []\n", + " answers = []\n", + " option_a, option_b, option_c, option_d = options\n", + "\n", + " system_instruction = \"\"\"\n", + " You are a huge fan of the TV Show Friends. You will be given a QUESTION and four OPTIONS. I want you to ANSWER the QUESTION with the following steps.\n", + "\n", + " Evaluation Steps:\n", + " 1. Read the QUESTION carefully.\n", + " 2. Choose the correct OPTION from OPTIONS best of your knowledge.\n", + " 3. Output the ANSWER which is a single alphabet from A, B, C, D which is the right OPTION for the QUESTION\n", + " 4. The Output format for each OPTION is \n", + " for A: 'ANSWER: A'\n", + " for B: 'ANSWER: B'\n", + " for C: 'ANSWER: C'\n", + " for D: 'ANSWER: D'\n", + "\n", + " Here are a few Examples for how I expect the answer to be.\n", + " Examples:\n", + "\n", + " {\n", + " QUESTION: What is the name of Ross and Rachel's daughter,\n", + " OPTIONS: \n", + " A. Emma\n", + " B. Delilah\n", + " C. Bemma\n", + " D. Vanshita\n", + " ANSWER: A\n", + " },\n", + "\n", + " {\n", + " QUESTION: What is Chandler Bing's Middle Name,\n", + " OPTIONS: \n", + " A. Meredith\n", + " B. Muriel\n", + " C. Richard\n", + " D. Robert\n", + " ANSWER: B\n", + " }\n", + "\n", + " Based on the above Evaluation Steps and Examples now ANSWER the QUESTION I give you\n", + " \"\"\"\n", + "\n", + " user_query = f\"\"\"\n", + "\n", + " QUESTION: {question}\n", + "\n", + " OPTIONS: \n", + " A. {option_a}\n", + " B. {option_b}\n", + " C. {option_c}\n", + " D. {option_d}\n", + "\n", + " OUTPUT only the ANSWER which is either A, B, C, or D. \n", + " The ANSWER is\n", + " \"\"\"\n", + " other_answers = []\n", + " while not answers:\n", + " for _ in range(samples):\n", + " output = await async_query(system_instruction, user_query)\n", + " output = output['content']\n", + "\n", + " outputs.append(output)\n", + "\n", + " match = re.search(r'\\b(?:ANSWER|Answer is)(?:\\:|) ([A-D])\\b', output, flags=re.IGNORECASE)\n", + "\n", + " if match:\n", + " answer_choice = match.group(1)\n", + " answers.append(answer_choice)\n", + "\n", + " # Count occurrences\n", + " correct_format_rate = len(answers) / samples\n", + " correct_answer_rate = sum(1 for answer in answers if answer == reference) / len(answers)\n", + "\n", + " # get the most occurring sampled answer as the final answer\n", + " counts = Counter(answers)\n", + " answer = max(counts, key=lambda x: (counts[x], answers.index(x)))\n", + "\n", + " # print(outputs)\n", + " # print(answers)\n", + "\n", + " return answer, correct_format_rate, correct_answer_rate\n" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 165, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "\" Oh, goodness me! *adjusts pearl necklace* Well, darling, I do believe it's quite... um... let me see... (checks weather app on phone) why, it's simply dreadful outside, dearie! *giggles* The sun is shining, but it's just not very warm. In fact, it's rather chilly, if you know what I mean. *hisses* And the wind... goodness me, the wind is positively howling like a pack of wolves! *shudders* But fear not, my dear, for I have my trusty umbrella to keep me dry and stylish. *adjusts hat* Now, do tell, what brings you out on this... shall we say, less than lovely day?\"" + "('B', 1.0, 0.6)" ] }, - "execution_count": 6, + "execution_count": 165, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "data = query_oai_api(system_instruction,user_query)\n", + "\n", + "a = await get_mcq_answer_async(\n", + " question='Where did Carol meet her wife',\n", + " options=['At Central Perk', 'At the Gym', 'At work', 'At the library'],\n", + " reference='B',\n", + " samples=10\n", + ")\n", + "\n", + "a" + ] + }, + { + "cell_type": "code", + "execution_count": 166, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
QuestionOptionsCorrect AnswerCorrect_Answer_nonumber_of_options
0How many times was Ross legally divorced? \\n[Twice, Three times, Five times, Six times]Three timesB4
1Where did Carol first meet Susan? \\n[In college, At work, At the gym, At Central P...At the gymC4
2How did Susan and Ross come up with Ben’s name?[It was the doctor's name \\n, They both had u...It was on the janitor's name tagD4
3What were Ben's first words? \\n[Hi, Bye, Mom, Dumb]HiA4
4How long did Ross and Emily date before they g...[14 days, 6 weeks \\n, A year \\n, 3 months]6 weeks \\nB4
..................
690Who is Monica?[Ross's Sister, Rachel's Sister, Emily's Siste...Ross's SisterA4
691Who says it usually, \"I Know!\"[Rachel, Chandler, Monica, Joey]MonicaC4
692Monica Kisses Joey[True, False, None of the above, All of the ab...FalseB4
693Who did Phoebe marry?[David, Ross, Mike, Joey]MikeC4
694What was the name of the TV show joey worked for?[Friends, Days Of Our Lives, How I Met Your Mo...Days Of Our LivesB4
\n", + "

694 rows × 5 columns

\n", + "
" + ], + "text/plain": [ + " Question \\\n", + "0 How many times was Ross legally divorced? \\n \n", + "1 Where did Carol first meet Susan? \\n \n", + "2 How did Susan and Ross come up with Ben’s name? \n", + "3 What were Ben's first words? \\n \n", + "4 How long did Ross and Emily date before they g... \n", + ".. ... \n", + "690 Who is Monica? \n", + "691 Who says it usually, \"I Know!\" \n", + "692 Monica Kisses Joey \n", + "693 Who did Phoebe marry? \n", + "694 What was the name of the TV show joey worked for? \n", + "\n", + " Options \\\n", + "0 [Twice, Three times, Five times, Six times] \n", + "1 [In college, At work, At the gym, At Central P... \n", + "2 [It was the doctor's name \\n, They both had u... \n", + "3 [Hi, Bye, Mom, Dumb] \n", + "4 [14 days, 6 weeks \\n, A year \\n, 3 months] \n", + ".. ... \n", + "690 [Ross's Sister, Rachel's Sister, Emily's Siste... \n", + "691 [Rachel, Chandler, Monica, Joey] \n", + "692 [True, False, None of the above, All of the ab... \n", + "693 [David, Ross, Mike, Joey] \n", + "694 [Friends, Days Of Our Lives, How I Met Your Mo... \n", + "\n", + " Correct Answer Correct_Answer_no number_of_options \n", + "0 Three times B 4 \n", + "1 At the gym C 4 \n", + "2 It was on the janitor's name tag D 4 \n", + "3 Hi A 4 \n", + "4 6 weeks \\n B 4 \n", + ".. ... ... ... \n", + "690 Ross's Sister A 4 \n", + "691 Monica C 4 \n", + "692 False B 4 \n", + "693 Mike C 4 \n", + "694 Days Of Our Lives B 4 \n", + "\n", + "[694 rows x 5 columns]" + ] + }, + "execution_count": 166, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# !pip3 install swifter\n", "data" ] }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 37%|███▋ | 260/694 [2:14:16<3:44:07, 30.99s/it] \n" + ] + }, + { + "ename": "TimeoutError", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTimeoutError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn [9], line 47\u001b[0m\n\u001b[1;32m 45\u001b[0m loop \u001b[38;5;241m=\u001b[39m asyncio\u001b[38;5;241m.\u001b[39mnew_event_loop()\n\u001b[1;32m 46\u001b[0m asyncio\u001b[38;5;241m.\u001b[39mset_event_loop(loop)\n\u001b[0;32m---> 47\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m main()\n", + "Cell \u001b[0;32mIn [9], line 34\u001b[0m, in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mmain\u001b[39m():\n\u001b[0;32m---> 34\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m process_dataframe_concurrently(data)\n\u001b[1;32m 36\u001b[0m \u001b[38;5;66;03m# Extract results into separate lists\u001b[39;00m\n\u001b[1;32m 37\u001b[0m prediction, correct_format_rate, correct_answer_rate \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mzip\u001b[39m(\u001b[38;5;241m*\u001b[39mresults)\n", + "Cell \u001b[0;32mIn [9], line 9\u001b[0m, in \u001b[0;36mprocess_dataframe_concurrently\u001b[0;34m(data)\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m tqdm(total\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(data)) \u001b[38;5;28;01mas\u001b[39;00m pbar:\n\u001b[1;32m 8\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m idx, row \u001b[38;5;129;01min\u001b[39;00m data\u001b[38;5;241m.\u001b[39miterrows():\n\u001b[0;32m----> 9\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m get_mcq_answer_async(\n\u001b[1;32m 10\u001b[0m question\u001b[38;5;241m=\u001b[39mrow[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mQuestion\u001b[39m\u001b[38;5;124m'\u001b[39m],\n\u001b[1;32m 11\u001b[0m options\u001b[38;5;241m=\u001b[39mrow[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mOptions\u001b[39m\u001b[38;5;124m'\u001b[39m],\n\u001b[1;32m 12\u001b[0m reference\u001b[38;5;241m=\u001b[39mrow[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mCorrect_Answer_no\u001b[39m\u001b[38;5;124m'\u001b[39m],\n\u001b[1;32m 13\u001b[0m samples\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m5\u001b[39m\n\u001b[1;32m 14\u001b[0m )\n\u001b[1;32m 15\u001b[0m d\u001b[38;5;241m=\u001b[39m{\n\u001b[1;32m 16\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mquestion\u001b[39m\u001b[38;5;124m'\u001b[39m: row[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mQuestion\u001b[39m\u001b[38;5;124m'\u001b[39m],\n\u001b[1;32m 17\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124moption\u001b[39m\u001b[38;5;124m'\u001b[39m : row[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mOptions\u001b[39m\u001b[38;5;124m'\u001b[39m],\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 21\u001b[0m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcorrect_answer_rate\u001b[39m\u001b[38;5;124m'\u001b[39m : output[\u001b[38;5;241m2\u001b[39m]\n\u001b[1;32m 22\u001b[0m }\n\u001b[1;32m 23\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124meval_out\u001b[39m\u001b[38;5;132;01m{\u001b[39;00midx\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.json\u001b[39m\u001b[38;5;124m'\u001b[39m,\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mw\u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n", + "Cell \u001b[0;32mIn [8], line 82\u001b[0m, in \u001b[0;36mget_mcq_answer_async\u001b[0;34m(question, options, reference, samples)\u001b[0m\n\u001b[1;32m 80\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m answers:\n\u001b[1;32m 81\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m _ \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(samples):\n\u001b[0;32m---> 82\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m async_query(system_instruction, user_query)\n\u001b[1;32m 83\u001b[0m output \u001b[38;5;241m=\u001b[39m output[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcontent\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m 85\u001b[0m outputs\u001b[38;5;241m.\u001b[39mappend(output)\n", + "Cell \u001b[0;32mIn [8], line 16\u001b[0m, in \u001b[0;36masync_query\u001b[0;34m(system_instruction, user_query)\u001b[0m\n\u001b[1;32m 13\u001b[0m data \u001b[38;5;241m=\u001b[39m json\u001b[38;5;241m.\u001b[39mdumps(json_body)\n\u001b[1;32m 15\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m aiohttp\u001b[38;5;241m.\u001b[39mClientSession() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[0;32m---> 16\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mwith\u001b[39;00m session\u001b[38;5;241m.\u001b[39mpost(API_URL, data\u001b[38;5;241m=\u001b[39mdata) \u001b[38;5;28;01mas\u001b[39;00m response:\n\u001b[1;32m 17\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 18\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mawait\u001b[39;00m response\u001b[38;5;241m.\u001b[39mjson()\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/aiohttp/client.py:1187\u001b[0m, in \u001b[0;36m_BaseRequestContextManager.__aenter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1186\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39m__aenter__\u001b[39m(\u001b[39mself\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m _RetType:\n\u001b[0;32m-> 1187\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_resp \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_coro\n\u001b[1;32m 1188\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_resp\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/aiohttp/client.py:601\u001b[0m, in \u001b[0;36mClientSession._request\u001b[0;34m(self, method, str_or_url, params, data, json, cookies, headers, skip_auto_headers, auth, allow_redirects, max_redirects, compress, chunked, expect100, raise_for_status, read_until_eof, proxy, proxy_auth, timeout, verify_ssl, fingerprint, ssl_context, ssl, server_hostname, proxy_headers, trace_request_ctx, read_bufsize, auto_decompress, max_line_size, max_field_size)\u001b[0m\n\u001b[1;32m 599\u001b[0m resp \u001b[39m=\u001b[39m \u001b[39mawait\u001b[39;00m req\u001b[39m.\u001b[39msend(conn)\n\u001b[1;32m 600\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 601\u001b[0m \u001b[39mawait\u001b[39;00m resp\u001b[39m.\u001b[39mstart(conn)\n\u001b[1;32m 602\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m:\n\u001b[1;32m 603\u001b[0m resp\u001b[39m.\u001b[39mclose()\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/aiohttp/client_reqrep.py:980\u001b[0m, in \u001b[0;36mClientResponse.start\u001b[0;34m(self, connection)\u001b[0m\n\u001b[1;32m 978\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_continue \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 979\u001b[0m set_result(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_continue, \u001b[39mTrue\u001b[39;00m)\n\u001b[0;32m--> 980\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_continue \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 982\u001b[0m \u001b[39m# payload eof handler\u001b[39;00m\n\u001b[1;32m 983\u001b[0m payload\u001b[39m.\u001b[39mon_eof(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_response_eof)\n", + "File \u001b[0;32m~/Library/Python/3.8/lib/python/site-packages/aiohttp/helpers.py:735\u001b[0m, in \u001b[0;36mTimerContext.__exit__\u001b[0;34m(self, exc_type, exc_val, exc_tb)\u001b[0m\n\u001b[1;32m 732\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_tasks\u001b[39m.\u001b[39mpop()\n\u001b[1;32m 734\u001b[0m \u001b[39mif\u001b[39;00m exc_type \u001b[39mis\u001b[39;00m asyncio\u001b[39m.\u001b[39mCancelledError \u001b[39mand\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_cancelled:\n\u001b[0;32m--> 735\u001b[0m \u001b[39mraise\u001b[39;00m asyncio\u001b[39m.\u001b[39mTimeoutError \u001b[39mfrom\u001b[39;00m \u001b[39mNone\u001b[39m\n\u001b[1;32m 736\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mNone\u001b[39;00m\n", + "\u001b[0;31mTimeoutError\u001b[0m: " + ] + } + ], + "source": [ + "import asyncio\n", + "from tqdm import tqdm\n", + "\n", + "# Function to run the processing concurrently with a progress bar\n", + "async def process_dataframe_concurrently(data):\n", + " results = []\n", + " with tqdm(total=len(data)) as pbar:\n", + " for idx, row in data.iterrows():\n", + " output = await get_mcq_answer_async(\n", + " question=row['Question'],\n", + " options=row['Options'],\n", + " reference=row['Correct_Answer_no'],\n", + " samples=5\n", + " )\n", + " d={\n", + " 'question': row['Question'],\n", + " 'option' : row['Options'],\n", + " 'reference' : row['Correct_Answer_no'],\n", + " 'answer': output[0],\n", + " 'correct_format_rate' : output[1], \n", + " 'correct_answer_rate' : output[2]\n", + " }\n", + " with open(f'eval_out{idx}.json','w') as f:\n", + " json.dump(d, f)\n", + "\n", + " \n", + " results.append(output\n", + " )\n", + "\n", + " pbar.update(1)\n", + "\n", + "# Ensure the event loop is run in an asyncio context\n", + "async def main():\n", + " results = await process_dataframe_concurrently(data)\n", + "\n", + " # Extract results into separate lists\n", + " prediction, correct_format_rate, correct_answer_rate = zip(*results)\n", + " data['prediction'] = prediction\n", + " data['correct_format_rate'] = correct_format_rate\n", + " data['correct_answer_rate'] = correct_answer_rate\n", + "\n", + " return data\n", + "\n", + "# Create a new event loop and run the main function\n", + "loop = asyncio.new_event_loop()\n", + "asyncio.set_event_loop(loop)\n", + "data = await main()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
QuestionOptionsCorrect Answer
0How many times was Ross legally divorced? \\n[Twice, Three times, Five times, Six times]Three times
1Where did Carol first meet Susan? \\n[In college, At work, At the gym, At Central P...At the gym
2How did Susan and Ross come up with Ben’s name?[It was the doctor's name \\n, They both had u...It was on the janitor's name tag
3What were Ben's first words? \\n[Hi, Bye, Mom, Dumb]Hi
4How long did Ross and Emily date before they g...[14 days, 6 weeks \\n, A year \\n, 3 months]6 weeks \\n
............
692Who is Monica?[Ross's Sister, Rachel's Sister, Emily's Siste...Ross's Sister
693Who says it usually, \"I Know!\"[Rachel, Chandler, Monica, Joey]Monica
694Monica Kisses Joey[True, False]False
695Who did Phoebe marry?[David, Ross, Mike, Joey]Mike
696What was the name of the TV show joey worked for?[Friends, Days Of Our Lives, How I Met Your Mo...Days Of Our Lives
\n", + "

697 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " Question \\\n", + "0 How many times was Ross legally divorced? \\n \n", + "1 Where did Carol first meet Susan? \\n \n", + "2 How did Susan and Ross come up with Ben’s name? \n", + "3 What were Ben's first words? \\n \n", + "4 How long did Ross and Emily date before they g... \n", + ".. ... \n", + "692 Who is Monica? \n", + "693 Who says it usually, \"I Know!\" \n", + "694 Monica Kisses Joey \n", + "695 Who did Phoebe marry? \n", + "696 What was the name of the TV show joey worked for? \n", + "\n", + " Options \\\n", + "0 [Twice, Three times, Five times, Six times] \n", + "1 [In college, At work, At the gym, At Central P... \n", + "2 [It was the doctor's name \\n, They both had u... \n", + "3 [Hi, Bye, Mom, Dumb] \n", + "4 [14 days, 6 weeks \\n, A year \\n, 3 months] \n", + ".. ... \n", + "692 [Ross's Sister, Rachel's Sister, Emily's Siste... \n", + "693 [Rachel, Chandler, Monica, Joey] \n", + "694 [True, False] \n", + "695 [David, Ross, Mike, Joey] \n", + "696 [Friends, Days Of Our Lives, How I Met Your Mo... \n", + "\n", + " Correct Answer \n", + "0 Three times \n", + "1 At the gym \n", + "2 It was on the janitor's name tag \n", + "3 Hi \n", + "4 6 weeks \\n \n", + ".. ... \n", + "692 Ross's Sister \n", + "693 Monica \n", + "694 False \n", + "695 Mike \n", + "696 Days Of Our Lives \n", + "\n", + "[697 rows x 3 columns]" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.to_csv('friends_mcq_w_answers_evaluation.csv',index=None)" + ] + }, + { + "cell_type": "code", + "execution_count": 96, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2ddf4a3f2f53486992a3877ea2edb9dd", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Pandas Apply: 0%| | 0/695 [00:00