test_eval_multireg.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, glob, json, sys\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from src import data, utils, eval\n",
    "from typing import List\n",
    "\n",
    "sys.path.append('src')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the fine-tuned model and its tokenizer\n",
    "from transformers import AutoTokenizer\n",
    "\n",
    "tokenizer_name = 'hackathon-somos-nlp-2023/roberta-base-bne-finetuned-suicide-es'\n",
    "tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)\n",
    "with open('models/roberta-base-bne-finetuned.final-30.pkl', 'rb') as f:\n",
    "    model = pickle.load(f)\n",
    "# make the prediction function\n",
    "device = \"mps\"\n",
    "model.to(device)\n",
    "# predict = utils.make_predict(model.predict, tokenizer=tokenizer, device=device)\n",
    "def predict(msgs:List[str]):\n",
    "    predictions = np.zeros(len(msgs)).tolist()\n",
    "    for i,msg in enumerate(msgs):\n",
    "        predictions[i] = model.predict(msg, tokenizer=tokenizer, device=device)\n",
    "    return np.array(predictions)\n",
    "method = \"fine-tuning\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No sentence-transformers model found with name /Users/simon/.cache/torch/sentence_transformers/hackathon-somos-nlp-2023_roberta-base-bne-finetuned-suicide-es. Creating a new one with MEAN pooling.\n",
      "Some weights of the model checkpoint at /Users/simon/.cache/torch/sentence_transformers/hackathon-somos-nlp-2023_roberta-base-bne-finetuned-suicide-es were not used when initializing RobertaModel: ['classifier.out_proj.weight', 'classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias']\n",
      "- This IS expected if you are initializing RobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "#load the embeddings regression model and its sentece embeddings\n",
    "from sentence_transformers import SentenceTransformer\n",
    "from src.embeddings import EmbeddingsRegressor\n",
    "\n",
    "regressor_file = 'models/ridge_regressor-final.pkl'\n",
    "regressor_file = 'models/2d_ridge_roberta-suicide-regchain-pca-final.pkl'\n",
    "model_name = 'hackathon-somos-nlp-2023/roberta-base-bne-finetuned-suicide-es'\n",
    "tokenizer = SentenceTransformer(model_name)\n",
    "with open(regressor_file, 'rb') as f:\n",
    "    regressor = pickle.load(f)\n",
    "model = EmbeddingsRegressor(tokenizer, regressor, normalize_output=True)\n",
    "# make the prediction function\n",
    "predict = utils.make_predict(model.predict)\n",
    "# method = \"embeddings_multireg\"\n",
    "method = \"embeddings_chain-pca\""
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ABSOLUTE METRICS (predicting all observations):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load all the test set messages and concatenate then into a single string per subject id\n",
    "test_df = data.load('test')\n",
    "test_df = data.concat_messages(test_df)\n",
    "# get the predictions of each subject from the model\n",
    "results_df = test_df.assign(\n",
    "    label = lambda df: df.filter(regex=\"^d_\").values.tolist(),\n",
    ")\n",
    "results_df['predicted'] = predict(test_df.message.tolist()).tolist()\n",
    "# normalize\n",
    "# results_df['predicted'] = results_df['predicted'].apply(lambda x: (np.array(x)/sum(x)).tolist())\n",
    "results_df['predicted'] = utils.normalize(np.array(results_df.predicted.tolist())).tolist()\n",
    "results_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task A:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision (macro)</th>\n",
       "      <th>Recall (macro)</th>\n",
       "      <th>F1 (macro)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.711</td>\n",
       "      <td>0.753</td>\n",
       "      <td>0.727</td>\n",
       "      <td>0.707</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Accuracy  Precision (macro)  Recall (macro)  F1 (macro)\n",
       "0     0.711              0.753           0.727       0.707"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task B:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RMSE avg</th>\n",
       "      <th>R2 avg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.304</td>\n",
       "      <td>0.349</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   RMSE avg  R2 avg\n",
       "0     0.304   0.349"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task C:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision (macro)</th>\n",
       "      <th>Recall (macro)</th>\n",
       "      <th>F1 (macro)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.577</td>\n",
       "      <td>0.439</td>\n",
       "      <td>0.468</td>\n",
       "      <td>0.431</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Accuracy  Precision (macro)  Recall (macro)  F1 (macro)\n",
       "0     0.577              0.439           0.468       0.431"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task D:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RMSE avg</th>\n",
       "      <th>R2 avg</th>\n",
       "      <th>R2 control</th>\n",
       "      <th>R2 suffer_against</th>\n",
       "      <th>R2 suffer_in_favour</th>\n",
       "      <th>R2 suffer_other</th>\n",
       "      <th>RMSE control</th>\n",
       "      <th>RMSE suffer_against</th>\n",
       "      <th>RMSE suffer_in_favour</th>\n",
       "      <th>RMSE suffer_other</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.222</td>\n",
       "      <td>0.006</td>\n",
       "      <td>0.349</td>\n",
       "      <td>-0.144</td>\n",
       "      <td>0.358</td>\n",
       "      <td>-0.538</td>\n",
       "      <td>0.304</td>\n",
       "      <td>0.23</td>\n",
       "      <td>0.212</td>\n",
       "      <td>0.143</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   RMSE avg  R2 avg  R2 control  R2 suffer_against  R2 suffer_in_favour  \\\n",
       "0     0.222   0.006       0.349             -0.144                0.358   \n",
       "\n",
       "   R2 suffer_other  RMSE control  RMSE suffer_against  RMSE suffer_in_favour  \\\n",
       "0           -0.538         0.304                 0.23                  0.212   \n",
       "\n",
       "   RMSE suffer_other  \n",
       "0              0.143  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "predictions = results_df.predicted.apply(np.round,args=(5,)).tolist()\n",
    "preds_df = data.make_task_labels_from_d(predictions, include_d=True).rename(\n",
    "    columns={c:'d_'+c.replace('+','_').replace('|','_') for c in data.task_d_cols}\n",
    ")\n",
    "preds_df.to_csv(f'data/d_{method}_test_predictions.csv', index=False)\n",
    "results = eval.absolute_results(test_df, preds_df)\n",
    "print(\"Task A:\")\n",
    "display(results.taska.df.round(3))\n",
    "print(\"Task B:\")\n",
    "display(results.taskb.df.round(3))\n",
    "print(\"Task C:\")\n",
    "display(results.taskc.df.round(3))\n",
    "print(\"Task D:\")\n",
    "display(results.taskd.df.round(3))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "EARLY-RISK METRICS (round predictions/evaluation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[codecarbon INFO @ 12:52:19] [setup] RAM Tracking...\n",
      "[codecarbon INFO @ 12:52:19] [setup] GPU Tracking...\n",
      "[codecarbon INFO @ 12:52:19] No GPU found.\n",
      "[codecarbon INFO @ 12:52:19] [setup] CPU Tracking...\n",
      "[codecarbon WARNING @ 12:52:19] No CPU tracking mode found. Falling back on CPU constant mode.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[codecarbon INFO @ 12:52:19] CPU Model on constant consumption mode: Apple M1 Pro\n",
      "[codecarbon INFO @ 12:52:19] >>> Tracker's metadata:\n",
      "[codecarbon INFO @ 12:52:19]   Platform system: macOS-13.0-arm64-arm-64bit\n",
      "[codecarbon INFO @ 12:52:19]   Python version: 3.8.16\n",
      "[codecarbon INFO @ 12:52:19]   CodeCarbon version: 2.2.1\n",
      "[codecarbon INFO @ 12:52:19]   Available RAM : 16.000 GB\n",
      "[codecarbon INFO @ 12:52:19]   CPU count: 10\n",
      "[codecarbon INFO @ 12:52:19]   CPU model: Apple M1 Pro\n",
      "[codecarbon INFO @ 12:52:19]   GPU count: None\n",
      "[codecarbon INFO @ 12:52:19]   GPU model: None\n"
     ]
    }
   ],
   "source": [
    "import json, glob, os, re, tqdm\n",
    "from codecarbon import EmissionsTracker\n",
    "from src.class_eval import (\n",
    "    ClassRegressionEvaluation,\n",
    "    BinaryClassification,\n",
    "    BinaryMultiClassification,\n",
    "    ClassMultiRegressionEvaluation,\n",
    "    Emissions\n",
    ")\n",
    "\n",
    "# load the test set\n",
    "test_df = data.load('test').sort_values('round')\n",
    "# carbon emissions tracker\n",
    "tracker = EmissionsTracker(\n",
    "        save_to_file = True,\n",
    "        log_level= \"INFO\",\n",
    "        tracking_mode= \"process\",\n",
    "        output_dir= \"reports\")\n",
    "# relevant columns for the tracker\n",
    "relevant_cols = ['duration', 'emissions', 'cpu_energy', 'gpu_energy', 'ram_energy', \n",
    "            'energy_consumed', 'cpu_count', 'gpu_count', 'cpu_model', 'gpu_model', 'ram_total_size']\n",
    "\n",
    "# initialize helper variables\n",
    "prev_rounds_dfs = []\n",
    "round_preds = {}\n",
    "round_emmissions = {}\n",
    "preds_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(f'data/round_results/{method}', exist_ok=True)\n",
    "for round_num in tqdm.notebook.tqdm(sorted(test_df['round'].unique())):\n",
    "    # print(f\"Round {round_num} ======================\")\n",
    "    round_df = test_df[test_df['round'] == round_num]\n",
    "    rounds_df = pd.concat(prev_rounds_dfs+[round_df], axis=0).drop_duplicates('id_message')\n",
    "    rounds_df = rounds_df[rounds_df.subject_id.isin(round_df.subject_id.unique())]\n",
    "    rounds_df = data.concat_messages(rounds_df)\n",
    "    # make the predictions\n",
    "    tracker.start()\n",
    "    # pred_of_round = rounds_df.message.apply(predict).tolist()\n",
    "    pred_of_round = predict(rounds_df.message.tolist()).tolist()\n",
    "    # normalize on [0,1]\n",
    "    pred_of_round = np.array(pred_of_round).reshape(-1,4)\n",
    "    pred_of_round = pred_of_round / pred_of_round.sum(axis=1, keepdims=True)\n",
    "    pred_of_round_df = data.make_task_labels_from_d(pred_of_round.tolist(), include_d=True)\n",
    "    pred_of_round_df['nick'] = rounds_df['subject_id'].values\n",
    "    tracker.stop()\n",
    "    emissions_df = pd.read_csv('reports/emissions.csv') # pd.DataFrame([tracker.final_emissions_data])\n",
    "    emissions_dict = emissions_df[relevant_cols].iloc[-1].to_dict()\n",
    "    pred_of_round_df.columns = pred_of_round_df.columns.str.replace('[+|\\s]', '_', regex=True).str.lower()\n",
    "    pred_of_round_df['round'] = round_num\n",
    "    # save the predictions and the round data\n",
    "    round_preds[round_num] = pred_of_round_df\n",
    "    round_emmissions[round_num] = emissions_dict\n",
    "    preds_dict.update(pred_of_round_df.set_index('nick').to_dict(orient='index'))\n",
    "    round_preds[round_num].to_csv(f'data/round_results/{method}/round_{round_num}_preds.csv', index=False)\n",
    "    json.dump(round_emmissions[round_num],open(f\"data/round_results/{method}/round_{round_num}_emissions.json\",'w'))\n",
    "    prev_rounds_dfs.append(round_df)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the metrics:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.class_eval import (\n",
    "    ClassRegressionEvaluation,\n",
    "    BinaryClassification,\n",
    "    BinaryMultiClassification,\n",
    "    ClassMultiRegressionEvaluation,\n",
    "    Emissions\n",
    ")\n",
    "import glob, re\n",
    "import pandas as pd\n",
    "\n",
    "# method = \"embeddings\"\n",
    "# method = \"bert-fine-tuning\"\n",
    "\n",
    "# load round predictions obtained above:\n",
    "round_preds = {}\n",
    "round_preds_files = glob.glob(f'data/round_results/{method}/round_*_preds.csv')\n",
    "for f in round_preds_files:\n",
    "  round_num = re.search('round_(\\d+)_preds.csv',f).group(1)\n",
    "  round_preds[round_num] = pd.read_csv(f)#.drop(columns=['Unnamed: 0'])\n",
    "\n",
    "  # to make a dictionary of dataframes with the predictions of each subject for all the rounds\n",
    "round_preds_dict = {}\n",
    "round_preds_seq_df = round_preds['1'].set_index('nick').sort_index()\n",
    "for round_ in round_preds:\n",
    "    # get the predictions of the last rounds\n",
    "    prev_rounds_df = round_preds_seq_df.copy()\n",
    "    # update the predictions with the current round\n",
    "    this_round_df = round_preds[round_].set_index('nick').sort_index()\n",
    "    prev_rounds_df.loc[this_round_df.index.values,this_round_df.columns] = this_round_df.values\n",
    "    round_preds_dict[int(round_)] = prev_rounds_df.assign(round=int(round_)).reset_index()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Classification metrics:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "149 lines read in qrels file!\n",
      "\n",
      "\n",
      "\n",
      "149 lines read in qrels file!\n",
      "\n",
      "\n",
      "\n",
      "149 lines read in qrels file!\n",
      "\n",
      "\n",
      "\n",
      "150 lines read in qrels file!\n",
      "\n",
      "\n",
      "\n",
      "149 lines read in qrels file!\n",
      "\n",
      "\n",
      "\n",
      "149 lines read in qrels file!\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def preprocess_binary_preds(binary_preds_df, pred_col='pred'):\n",
    "  \"\"\" \n",
    "  preprocess the binary predictions dataframe to only keep the first round\n",
    "  when each user was labeled as positive (if any)\n",
    "  \"\"\"\n",
    "  results_dicts = []\n",
    "  for nick, nick_df in binary_preds_df.groupby('nick'):\n",
    "    df = nick_df.sort_values(['round'])\n",
    "    # get the instance when this user was labeled as positive for the first time \n",
    "    result_df = df[df[pred_col]==1]\n",
    "    if len(result_df) > 0: # if labeled positive in any round\n",
    "      results_dicts.append(result_df.iloc[0].to_dict())\n",
    "    else: # otherwise get the results of the last round\n",
    "      results_dicts.append(df.iloc[-1].to_dict())\n",
    "  return pd.DataFrame(results_dicts)\n",
    "\n",
    "\n",
    "# predictions of each round\n",
    "round_preds_df = pd.concat(round_preds.values()).sort_values(['nick','round'])\n",
    "\n",
    "# binary classification\n",
    "binary_preds_df = round_preds_df[['round','nick','a_label']].rename(columns={'a_label':'pred'})\n",
    "binary_preds_df = preprocess_binary_preds(binary_preds_df)\n",
    "binary_class = BinaryClassification(\n",
    "    '2',  data=binary_preds_df, qrels='data/test/golden_truth/task2_gold_a.txt'\n",
    "  )\n",
    "\n",
    "# binary regression\n",
    "binary_reg_df = round_preds_df[['round','nick','b_label']].rename(columns={'b_label':'pred'})\n",
    "binary_reg_std = ClassRegressionEvaluation('2',  # standard regression metrics\n",
    "                                       data=binary_reg_df.groupby(['nick']).last().reset_index(), \n",
    "                                       qrels='data/test/golden_truth/task2_gold_b.txt')\n",
    "binary_reg_preds_dict = {k:v[['round','nick','b_label']].rename(columns={'b_label':'pred'}) for k,v in round_preds_dict.items()}\n",
    "binary_reg_rank = ClassRegressionEvaluation('2',  # rank-based regression metrics\n",
    "                                        data=binary_reg_preds_dict,\n",
    "                                        qrels='data/test/golden_truth/task2_gold_b.txt')\n",
    "\n",
    "# multi-class classification\n",
    "multi_class_preds_df = round_preds_df[['round','nick','c_label']].rename(columns={'c_label':'pred'})\n",
    "# make the  binary predictions (1 if suffer, 0 otherwise)\n",
    "multi_class_preds_df = multi_class_preds_df.assign(pred_b = lambda df: df.pred.str.contains('suffer').astype(int))\n",
    "multi_class_preds_df = preprocess_binary_preds(multi_class_preds_df,pred_col='pred_b')\n",
    "multiclass = BinaryMultiClassification(\n",
    "    '2',  data=multi_class_preds_df,  qrels='data/test/golden_truth/task2_gold_c.txt'\n",
    ")\n",
    "del multiclass.qrels_multiclass['subject51'] # this user is not in the test set gold labels\n",
    "\n",
    "# multi-regression\n",
    "multi_reg_df = round_preds_df.set_index(['round','nick'])[['suffer_in_favour','suffer_against','suffer_other','control']].assign(\n",
    "  pred=lambda df: df.values.tolist()\n",
    ")\n",
    "multi_reg_preds_dict = {}\n",
    "for round_ in round_preds_dict:\n",
    "  multi_reg_preds_dict[round_] = round_preds_dict[round_].set_index(['nick'])[['suffer_in_favour','suffer_against','suffer_other','control']].assign(\n",
    "    pred=lambda df: df.values.tolist()\n",
    "  )\n",
    "multi_reg = ClassMultiRegressionEvaluation('2',  # standard regression metrics\n",
    "                                        data=multi_reg_df.groupby(['nick']).last().reset_index(),\n",
    "                                        qrels='data/test/golden_truth/task2_gold_d.txt')\n",
    "multi_reg_rank = ClassMultiRegressionEvaluation('2',  # rank-based regression metrics\n",
    "                                        data=multi_reg_preds_dict,\n",
    "                                        qrels='data/test/golden_truth/task2_gold_d.txt')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task A:\n",
      "===================================================\n",
      "DECISION-BASED EVALUATION:\n",
      "BINARY METRICS: =============================\n",
      "Accuracy:0.6912751677852349\n",
      "Macro precision:0.7113289760348583\n",
      "Macro recall:0.7553751645458534\n",
      "Macro f1:0.682332220986281\n",
      "Micro precision:0.6912751677852349\n",
      "Micro recall:0.6912751677852349\n",
      "Micro f1:0.6912751677852349\n",
      "LATENCY-BASED METRICS: =============================\n",
      "ERDE_5:0.2827975990970474\n",
      "ERDE_50:0.15548849150939317\n",
      "Median latency:3.0\n",
      "Speed:0.9821019115346772\n",
      "latency-weightedF1:0.7224657740025211\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Macro_P</th>\n",
       "      <th>Macro_R</th>\n",
       "      <th>Macro_F1</th>\n",
       "      <th>Micro_P</th>\n",
       "      <th>Micro_R</th>\n",
       "      <th>Micro_F1</th>\n",
       "      <th>ERDE5</th>\n",
       "      <th>ERDE30</th>\n",
       "      <th>ERDE50</th>\n",
       "      <th>latencyTP</th>\n",
       "      <th>speed</th>\n",
       "      <th>latency-weightedF1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.691</td>\n",
       "      <td>0.711</td>\n",
       "      <td>0.755</td>\n",
       "      <td>0.682</td>\n",
       "      <td>0.691</td>\n",
       "      <td>0.691</td>\n",
       "      <td>0.691</td>\n",
       "      <td>0.283</td>\n",
       "      <td>0.027</td>\n",
       "      <td>0.155</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.722</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Accuracy  Macro_P  Macro_R  Macro_F1  Micro_P  Micro_R  Micro_F1  ERDE5  \\\n",
       "0     0.691    0.711    0.755     0.682    0.691    0.691     0.691  0.283   \n",
       "\n",
       "   ERDE30  ERDE50  latencyTP  speed  latency-weightedF1  \n",
       "0   0.027   0.155        3.0  0.982               0.722  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task B:\n",
      "REGRESSION METRICS: =============================\n",
      "RMSE:0.2407803309766703\n",
      "Pearson correlation coefficient:0.7746648207201776\n",
      "===================================================\n",
      "RANK-BASED EVALUATION:\n",
      "Analizing ranking at round 1\n",
      "PRECISION AT K: =============================\n",
      "P@5:0.0\n",
      "P@10:0.0\n",
      "P@20:0.15\n",
      "P@30:0.1\n",
      "P@50:0.1\n",
      "Analizing ranking at round 25\n",
      "PRECISION AT K: =============================\n",
      "P@5:0.0\n",
      "P@10:0.2\n",
      "P@20:0.15\n",
      "P@30:0.13333333333333333\n",
      "P@50:0.08\n",
      "Analizing ranking at round 50\n",
      "PRECISION AT K: =============================\n",
      "P@5:0.0\n",
      "P@10:0.0\n",
      "P@20:0.15\n",
      "P@30:0.1\n",
      "P@50:0.1\n",
      "Analizing ranking at round 75\n",
      "PRECISION AT K: =============================\n",
      "P@5:0.0\n",
      "P@10:0.0\n",
      "P@20:0.15\n",
      "P@30:0.1\n",
      "P@50:0.1\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RMSE:</th>\n",
       "      <th>Pearson_coefficient</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.241</td>\n",
       "      <td>0.775</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   RMSE:  Pearson_coefficient\n",
       "0  0.241                0.775"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rank-based metrics:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>@5</th>\n",
       "      <th>@10</th>\n",
       "      <th>@20</th>\n",
       "      <th>@30</th>\n",
       "      <th>@50</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>round</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.100</td>\n",
       "      <td>0.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.133</td>\n",
       "      <td>0.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.100</td>\n",
       "      <td>0.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.100</td>\n",
       "      <td>0.10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        @5  @10   @20    @30   @50\n",
       "round                             \n",
       "1      0.0  0.0  0.15  0.100  0.10\n",
       "25     0.0  0.2  0.15  0.133  0.08\n",
       "50     0.0  0.0  0.15  0.100  0.10\n",
       "75     0.0  0.0  0.15  0.100  0.10"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task C:\n",
      "===================================================\n",
      "DECISION-BASED EVALUATION:\n",
      "BINARY METRICS: =============================\n",
      "Accuracy:0.5302013422818792\n",
      "Macro precision:0.4366506123058542\n",
      "Macro recall:0.41759500189732746\n",
      "Macro f1:0.39195194206714945\n",
      "Micro precision:0.5302013422818792\n",
      "Micro recall:0.5302013422818792\n",
      "Micro f1:0.5302013422818792\n",
      "LATENCY-BASED METRICS: =============================\n",
      "ERDE_5:0.28381917470174994\n",
      "ERDE_50:0.1565100671140957\n",
      "Median latency:3.0\n",
      "Speed:0.9821019115346772\n",
      "latency-weightedF1:0.7183373981510782\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Macro_P</th>\n",
       "      <th>Macro_R</th>\n",
       "      <th>Macro_F1</th>\n",
       "      <th>Micro_P</th>\n",
       "      <th>Micro_R</th>\n",
       "      <th>Micro_F1</th>\n",
       "      <th>ERDE5</th>\n",
       "      <th>ERDE30</th>\n",
       "      <th>ERDE50</th>\n",
       "      <th>latencyTP</th>\n",
       "      <th>speed</th>\n",
       "      <th>latency-weightedF1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.53</td>\n",
       "      <td>0.437</td>\n",
       "      <td>0.418</td>\n",
       "      <td>0.392</td>\n",
       "      <td>0.53</td>\n",
       "      <td>0.53</td>\n",
       "      <td>0.53</td>\n",
       "      <td>0.284</td>\n",
       "      <td>0.157</td>\n",
       "      <td>0.157</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.718</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Accuracy  Macro_P  Macro_R  Macro_F1  Micro_P  Micro_R  Micro_F1  ERDE5  \\\n",
       "0      0.53    0.437    0.418     0.392     0.53     0.53      0.53  0.284   \n",
       "\n",
       "   ERDE30  ERDE50  latencyTP  speed  latency-weightedF1  \n",
       "0   0.157   0.157        3.0  0.982               0.718  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task D:\n",
      "REGRESSION METRICS: =============================\n",
      "RMSE:0.17865016808534367\n",
      "Pearson correlation coefficient:\n",
      "Pearson sf:0.7472435224903471\n",
      "Pearson sa:0.4992946899714973\n",
      "Pearson so:0.37469662952285493\n",
      "Pearson c:0.7742933448336263\n",
      "===================================================\n",
      "PRECISION AT - EVALUATION:\n",
      "Analizing ranking at round 1\n",
      "PRECISION AT K: =============================\n",
      "P@5:0.55\n",
      "P@10:0.275\n",
      "P@20:0.25\n",
      "P@30:0.20833333333333331\n",
      "P@50:0.2\n",
      "Analizing ranking at round 25\n",
      "PRECISION AT K: =============================\n",
      "P@5:0.55\n",
      "P@10:0.275\n",
      "P@20:0.275\n",
      "P@30:0.25\n",
      "P@50:0.24\n",
      "Analizing ranking at round 50\n",
      "PRECISION AT K: =============================\n",
      "P@5:0.49999999999999994\n",
      "P@10:0.24999999999999997\n",
      "P@20:0.275\n",
      "P@30:0.24166666666666667\n",
      "P@50:0.23\n",
      "Analizing ranking at round 75\n",
      "PRECISION AT K: =============================\n",
      "P@5:0.49999999999999994\n",
      "P@10:0.24999999999999997\n",
      "P@20:0.275\n",
      "P@30:0.225\n",
      "P@50:0.23\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RMSE:</th>\n",
       "      <th>Pearson_mean</th>\n",
       "      <th>Pearson_sf</th>\n",
       "      <th>Pearson_sa</th>\n",
       "      <th>Pearson_so</th>\n",
       "      <th>Pearson_c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.179</td>\n",
       "      <td>0.599</td>\n",
       "      <td>0.747</td>\n",
       "      <td>0.499</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.774</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   RMSE:  Pearson_mean  Pearson_sf  Pearson_sa  Pearson_so  Pearson_c\n",
       "0  0.179         0.599       0.747       0.499       0.375      0.774"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rank-based metrics:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>@5</th>\n",
       "      <th>@10</th>\n",
       "      <th>@20</th>\n",
       "      <th>@30</th>\n",
       "      <th>@50</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>round</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.55</td>\n",
       "      <td>0.275</td>\n",
       "      <td>0.250</td>\n",
       "      <td>0.208</td>\n",
       "      <td>0.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.55</td>\n",
       "      <td>0.275</td>\n",
       "      <td>0.275</td>\n",
       "      <td>0.250</td>\n",
       "      <td>0.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.50</td>\n",
       "      <td>0.250</td>\n",
       "      <td>0.275</td>\n",
       "      <td>0.242</td>\n",
       "      <td>0.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>0.50</td>\n",
       "      <td>0.250</td>\n",
       "      <td>0.275</td>\n",
       "      <td>0.225</td>\n",
       "      <td>0.23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         @5    @10    @20    @30   @50\n",
       "round                                 \n",
       "1      0.55  0.275  0.250  0.208  0.20\n",
       "25     0.55  0.275  0.275  0.250  0.24\n",
       "50     0.50  0.250  0.275  0.242  0.23\n",
       "75     0.50  0.250  0.275  0.225  0.23"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"Task A:\")\n",
    "task_a_metrics = binary_class.eval_performance()\n",
    "task_a_metrics_df = pd.DataFrame([task_a_metrics]).round(3)\n",
    "display(task_a_metrics_df)\n",
    "print(\"Task B:\")\n",
    "task_b_metrics_std = binary_reg_std.eval_performance()\n",
    "task_b_metrics_rank = binary_reg_rank.eval_performance_rank_based()\n",
    "task_b_metrics_df_std = pd.DataFrame([task_b_metrics_std]).round(3)\n",
    "display(task_b_metrics_df_std)\n",
    "print('Rank-based metrics:')\n",
    "display(pd.DataFrame(task_b_metrics_rank).T.rename_axis('round').round(3))\n",
    "print(\"Task C:\")\n",
    "task_c_metrics = multiclass.eval_performance()\n",
    "task_c_metrics_df = pd.DataFrame([task_c_metrics]).round(3)\n",
    "display(task_c_metrics_df)\n",
    "print(\"Task D:\")\n",
    "task_d_metrics_std = multi_reg.eval_performance()\n",
    "task_d_metrics_rank = multi_reg_rank.eval_performance_rank_based()\n",
    "task_d_metrics_df_std = pd.DataFrame([task_d_metrics_std]).round(3)\n",
    "display(task_d_metrics_df_std)\n",
    "print('Rank-based metrics:')\n",
    "display(pd.DataFrame(task_d_metrics_rank).T.rename_axis('round').round(3))\n",
    "\n",
    "# save the results\n",
    "os.makedirs(f'reports/d_{method}', exist_ok=True)\n",
    "json.dump(task_a_metrics, open(f'reports/d_{method}/task_a_metrics.json', 'w'), indent=2)\n",
    "json.dump(task_b_metrics_std, open(f'reports/d_{method}/task_b_std_metrics.json', 'w'), indent=2)\n",
    "json.dump(task_b_metrics_rank, open(f'reports/d_{method}/task_b_rank_metrics.json', 'w'), indent=2)\n",
    "json.dump(task_c_metrics, open(f'reports/d_{method}/task_c_metrics.json', 'w'), indent=2)\n",
    "json.dump(task_d_metrics_std, open(f'reports/d_{method}/task_d_std_metrics.json', 'w'), indent=2)\n",
    "json.dump(task_d_metrics_rank, open(f'reports/d_{method}/task_d_rank_metrics.json', 'w'), indent=2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf-env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}