error_detection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make a Model\n",
    "- ~~seed 42, 456, 777, 109, 1004 총 5개의 seed로 앙상블 한 결과값 생성~~\n",
    "- ~~roberta-large 모델로 앙상블~~\n",
    "- ~~detection 의 detection~~"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import Dataset\n",
    "\n",
    "import evaluate\n",
    "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
    "from transformers import DataCollatorWithPadding\n",
    "from transformers import TrainingArguments, Trainer\n",
    "\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['TOKENIZERS_PARALLELISM'] = 'true'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED = 456\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.cuda.manual_seed(SEED)\n",
    "torch.cuda.manual_seed_all(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DEVICE = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
    "DEVICE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE_DIR = os.getcwd()\n",
    "DATA_DIR = os.path.join(BASE_DIR, './data')\n",
    "OUTPUT_DIR = os.path.join(BASE_DIR, './output')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at klue/bert-base were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at klue/bert-base and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model_name = 'klue/bert-base'\n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=7, cache_dir='/data/ephemeral/huggingface').to(DEVICE)\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name, cache_dir='/data/ephemeral/huggingface')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(os.path.join(DATA_DIR, 'train_aihub.csv'))\n",
    "dataset_train, dataset_valid = train_test_split(data, test_size=0.2, stratify=data['target'],random_state=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BERTDataset(Dataset):\n",
    "    def __init__(self, data, tokenizer):\n",
    "        input_texts = data['text']\n",
    "        targets = data['target']\n",
    "        self.inputs = []\n",
    "        self.labels = []\n",
    "        \n",
    "        for text, label in zip(input_texts, targets):\n",
    "            tokenized_input = tokenizer(text, padding='max_length', max_length=64, truncation=True, return_tensors='pt')\n",
    "            self.inputs.append(tokenized_input)\n",
    "            self.labels.append(torch.tensor(label))\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        return {\n",
    "            'input_ids': self.inputs[idx]['input_ids'].squeeze(0),\n",
    "            'token_type_ids': self.inputs[idx]['token_type_ids'].squeeze(0),\n",
    "            'attention_mask': self.inputs[idx]['attention_mask'].squeeze(0),\n",
    "            'labels': self.labels[idx].squeeze(0)\n",
    "        }\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train = BERTDataset(dataset_train, tokenizer)\n",
    "data_valid = BERTDataset(dataset_valid, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "f1 = evaluate.load('f1')\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    predictions = np.argmax(predictions, axis=1)\n",
    "    return f1.compute(predictions=predictions, references=labels, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "### for wandb setting\n",
    "os.environ['WANDB_DISABLED'] = 'true'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
      "Using the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).\n"
     ]
    }
   ],
   "source": [
    "training_args = TrainingArguments(\n",
    "    output_dir=OUTPUT_DIR,\n",
    "    overwrite_output_dir=True,\n",
    "    do_train=True,\n",
    "    do_eval=True,\n",
    "    do_predict=True,\n",
    "    logging_strategy='steps',\n",
    "    evaluation_strategy='steps',\n",
    "    save_strategy='steps',\n",
    "    logging_steps=0.05,\n",
    "    eval_steps=0.05,\n",
    "    save_steps=0.05,\n",
    "    save_total_limit=1,\n",
    "    learning_rate= 2e-05,\n",
    "    gradient_accumulation_steps=1,\n",
    "    adam_beta1 = 0.9,\n",
    "    adam_beta2 = 0.999,\n",
    "    adam_epsilon=1e-08,\n",
    "    weight_decay=0.01,\n",
    "    lr_scheduler_type='linear',\n",
    "    per_device_train_batch_size=16,\n",
    "    per_device_eval_batch_size=16,\n",
    "    num_train_epochs=2,\n",
    "    load_best_model_at_end=True,\n",
    "    metric_for_best_model='eval_f1',\n",
    "    greater_is_better=True,\n",
    "    seed=SEED\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=data_train,\n",
    "    eval_dataset=data_valid,\n",
    "    data_collator=data_collator,\n",
    "    compute_metrics=compute_metrics,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 94349\n",
      "  Num Epochs = 2\n",
      "  Instantaneous batch size per device = 16\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 11794\n",
      "  Number of trainable parameters = 110622727\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='11794' max='11794' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [11794/11794 22:21, Epoch 2/2]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.358200</td>\n",
       "      <td>0.795797</td>\n",
       "      <td>0.772574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2000</td>\n",
       "      <td>0.556100</td>\n",
       "      <td>0.610835</td>\n",
       "      <td>0.778774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3000</td>\n",
       "      <td>0.602500</td>\n",
       "      <td>0.554258</td>\n",
       "      <td>0.794589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4000</td>\n",
       "      <td>0.584900</td>\n",
       "      <td>0.547556</td>\n",
       "      <td>0.802880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5000</td>\n",
       "      <td>0.560400</td>\n",
       "      <td>0.541434</td>\n",
       "      <td>0.803530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6000</td>\n",
       "      <td>0.542900</td>\n",
       "      <td>0.540865</td>\n",
       "      <td>0.807547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7000</td>\n",
       "      <td>0.419200</td>\n",
       "      <td>0.553316</td>\n",
       "      <td>0.804258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8000</td>\n",
       "      <td>0.403600</td>\n",
       "      <td>0.565455</td>\n",
       "      <td>0.808688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9000</td>\n",
       "      <td>0.397700</td>\n",
       "      <td>0.552258</td>\n",
       "      <td>0.808941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10000</td>\n",
       "      <td>0.398200</td>\n",
       "      <td>0.546384</td>\n",
       "      <td>0.810033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11000</td>\n",
       "      <td>0.391600</td>\n",
       "      <td>0.548215</td>\n",
       "      <td>0.810103</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 23588\n",
      "  Batch size = 16\n",
      "Saving model checkpoint to /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-1000\n",
      "Configuration saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-1000/config.json\n",
      "Model weights saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-1000/pytorch_model.bin\n",
      "Deleting older checkpoint [/data/ephemeral/level2_datacentric/JHW/output/checkpoint-900] due to args.save_total_limit\n",
      "Deleting older checkpoint [/data/ephemeral/level2_datacentric/JHW/output/checkpoint-1200] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 23588\n",
      "  Batch size = 16\n",
      "Saving model checkpoint to /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-2000\n",
      "Configuration saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-2000/config.json\n",
      "Model weights saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-2000/pytorch_model.bin\n",
      "Deleting older checkpoint [/data/ephemeral/level2_datacentric/JHW/output/checkpoint-1300] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 23588\n",
      "  Batch size = 16\n",
      "Saving model checkpoint to /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-3000\n",
      "Configuration saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-3000/config.json\n",
      "Model weights saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-3000/pytorch_model.bin\n",
      "Deleting older checkpoint [/data/ephemeral/level2_datacentric/JHW/output/checkpoint-1000] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 23588\n",
      "  Batch size = 16\n",
      "Saving model checkpoint to /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-4000\n",
      "Configuration saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-4000/config.json\n",
      "Model weights saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-4000/pytorch_model.bin\n",
      "Deleting older checkpoint [/data/ephemeral/level2_datacentric/JHW/output/checkpoint-2000] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 23588\n",
      "  Batch size = 16\n",
      "Saving model checkpoint to /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-5000\n",
      "Configuration saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-5000/config.json\n",
      "Model weights saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-5000/pytorch_model.bin\n",
      "Deleting older checkpoint [/data/ephemeral/level2_datacentric/JHW/output/checkpoint-3000] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 23588\n",
      "  Batch size = 16\n",
      "Saving model checkpoint to /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-6000\n",
      "Configuration saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-6000/config.json\n",
      "Model weights saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-6000/pytorch_model.bin\n",
      "Deleting older checkpoint [/data/ephemeral/level2_datacentric/JHW/output/checkpoint-4000] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 23588\n",
      "  Batch size = 16\n",
      "Saving model checkpoint to /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-7000\n",
      "Configuration saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-7000/config.json\n",
      "Model weights saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-7000/pytorch_model.bin\n",
      "Deleting older checkpoint [/data/ephemeral/level2_datacentric/JHW/output/checkpoint-5000] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 23588\n",
      "  Batch size = 16\n",
      "Saving model checkpoint to /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-8000\n",
      "Configuration saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-8000/config.json\n",
      "Model weights saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-8000/pytorch_model.bin\n",
      "Deleting older checkpoint [/data/ephemeral/level2_datacentric/JHW/output/checkpoint-6000] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 23588\n",
      "  Batch size = 16\n",
      "Saving model checkpoint to /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-9000\n",
      "Configuration saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-9000/config.json\n",
      "Model weights saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-9000/pytorch_model.bin\n",
      "Deleting older checkpoint [/data/ephemeral/level2_datacentric/JHW/output/checkpoint-7000] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 23588\n",
      "  Batch size = 16\n",
      "Saving model checkpoint to /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-10000\n",
      "Configuration saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-10000/config.json\n",
      "Model weights saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-10000/pytorch_model.bin\n",
      "Deleting older checkpoint [/data/ephemeral/level2_datacentric/JHW/output/checkpoint-8000] due to args.save_total_limit\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 23588\n",
      "  Batch size = 16\n",
      "Saving model checkpoint to /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-11000\n",
      "Configuration saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-11000/config.json\n",
      "Model weights saved in /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-11000/pytorch_model.bin\n",
      "Deleting older checkpoint [/data/ephemeral/level2_datacentric/JHW/output/checkpoint-9000] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from /data/ephemeral/level2_datacentric/JHW/./output/checkpoint-11000 (score: 0.8101033081993375).\n",
      "Deleting older checkpoint [/data/ephemeral/level2_datacentric/JHW/output/checkpoint-10000] due to args.save_total_limit\n",
      "Deleting older checkpoint [/data/ephemeral/level2_datacentric/JHW/output/checkpoint-11000] due to args.save_total_limit\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=11794, training_loss=0.46935305725584114, metrics={'train_runtime': 1341.5966, 'train_samples_per_second': 140.652, 'train_steps_per_second': 8.791, 'total_flos': 6206344849363200.0, 'train_loss': 0.46935305725584114, 'epoch': 2.0})"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Making Probability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "117937it [16:41, 117.70it/s]\n"
     ]
    }
   ],
   "source": [
    "model.eval()\n",
    "logits_list = []\n",
    "for idx, sample in tqdm(data.iterrows()):\n",
    "    inputs = tokenizer(sample['text'], return_tensors=\"pt\").to(DEVICE)\n",
    "    with torch.no_grad():\n",
    "        logits = model(**inputs).logits\n",
    "        logits = logits.softmax(dim=-1)\n",
    "        logits_list.extend(logits.cpu().numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "probs = [list(item) for item in logits_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['probs'] = probs\n",
    "data.to_csv(f'./train_with_logits/electra_{SEED}.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensemble Label Prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "\n",
    "files = glob.glob('./train_with_logits/*.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "probs = [pd.read_csv(file)['probs'] for file in files]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "probs = [np.array(prob.apply(eval).to_list()) for prob in probs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "probs = np.array(probs).mean(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ERROR DETECT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "probs = np.array(probs)\n",
    "data = pd.read_csv('./data/train_aihub.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "id2label = {\n",
    "    0:'IT과학',\n",
    "    1:'경제',\n",
    "    2:'사회',\n",
    "    3:'생활문화',\n",
    "    4:'세계',\n",
    "    5:'스포츠',\n",
    "    6:'정치',\n",
    "}\n",
    "\n",
    "label2id = {\n",
    "    'IT과학':0,\n",
    "    '경제':1,\n",
    "    '사회':2,\n",
    "    '생활문화':3,\n",
    "    '세계':4,\n",
    "    '스포츠':5,\n",
    "    '정치':6,    \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input text: 주말 N 여행 제주권 만설 한라산…그 순수한 아름다움 속으로\n",
      "label: 세계\n",
      "input text: 리듬체조 유망주 서고은, '집사부일체' 깜짝 등장\n",
      "label: 스포츠\n",
      "input text: '복수해라' 윤현민 \"좋은 사람들과 동행...행복했던 시간\" 종영 소감\n",
      "label: IT과학\n",
      "input text: \"죄질이 좋지 않다\"…자가격리 두번 이탈 20대 첫 징역형\n",
      "label: 스포츠\n",
      "input text: '애로부부' 박철민·유경진 부부, 싸한 분위기 \"오는 길에도 싸웠다\"\n",
      "label: IT과학\n",
      "input text: 무릎 부상 기성용 선두 첼시전 출전 명단서 제외\n",
      "label: 경제\n",
      "input text: [철원]철원 모든 대학생에 장학금 지원\n",
      "label: 스포츠\n",
      "input text: [철원]철원장학회 장학생 선발\n",
      "label: 스포츠\n",
      "input text: 충북경찰 ‘앞담화’ 시간…“그냥 쉬고 싶은데 휴가 사유 왜 묻냐?”\n",
      "label: 경제\n",
      "input text: “코로나로 우울” 마약 밀수입 제주 30대 집행유예\n",
      "label: 경제\n",
      "total_issues: 5694\n"
     ]
    }
   ],
   "source": [
    "from cleanlab.filter import find_label_issues\n",
    "\n",
    "ordered_label_issues = find_label_issues(\n",
    "    labels=data['target'], #데이터셋 라벨\n",
    "    pred_probs=probs, #정답 예측 확률\n",
    "    return_indices_ranked_by='self_confidence',\n",
    ")\n",
    "\n",
    "head_issues=ordered_label_issues[:10]\n",
    "for issue in head_issues:\n",
    "    print('input text:',data.iloc[issue]['text'])\n",
    "    print('label:',id2label[data.iloc[issue]['target']])\n",
    "print('total_issues:',len(ordered_label_issues))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------------\n",
      "|  Generating a Cleanlab Dataset Health Summary            |\n",
      "|   for your dataset with 117,937 examples and 7 classes.  |\n",
      "|  Note, Cleanlab is not a medical doctor... yet.          |\n",
      "------------------------------------------------------------\n",
      "\n",
      "Overall Class Quality and Noise across your dataset (below)\n",
      "------------------------------------------------------------ \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class Name</th>\n",
       "      <th>Class Index</th>\n",
       "      <th>Label Issues</th>\n",
       "      <th>Inverse Label Issues</th>\n",
       "      <th>Label Noise</th>\n",
       "      <th>Inverse Label Noise</th>\n",
       "      <th>Label Quality Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>944</td>\n",
       "      <td>754</td>\n",
       "      <td>0.166784</td>\n",
       "      <td>0.137843</td>\n",
       "      <td>0.833216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1125</td>\n",
       "      <td>1626</td>\n",
       "      <td>0.053646</td>\n",
       "      <td>0.075727</td>\n",
       "      <td>0.946354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1931</td>\n",
       "      <td>1601</td>\n",
       "      <td>0.047062</td>\n",
       "      <td>0.039336</td>\n",
       "      <td>0.952938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>683</td>\n",
       "      <td>782</td>\n",
       "      <td>0.044928</td>\n",
       "      <td>0.051108</td>\n",
       "      <td>0.955072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>241</td>\n",
       "      <td>363</td>\n",
       "      <td>0.037209</td>\n",
       "      <td>0.055008</td>\n",
       "      <td>0.962791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>607</td>\n",
       "      <td>421</td>\n",
       "      <td>0.029036</td>\n",
       "      <td>0.020320</td>\n",
       "      <td>0.970964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>165</td>\n",
       "      <td>149</td>\n",
       "      <td>0.021454</td>\n",
       "      <td>0.019414</td>\n",
       "      <td>0.978546</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Class Name  Class Index  Label Issues  Inverse Label Issues  Label Noise   \n",
       "0           0            0           944                   754     0.166784  \\\n",
       "1           1            1          1125                  1626     0.053646   \n",
       "2           2            2          1931                  1601     0.047062   \n",
       "3           6            6           683                   782     0.044928   \n",
       "4           4            4           241                   363     0.037209   \n",
       "5           3            3           607                   421     0.029036   \n",
       "6           5            5           165                   149     0.021454   \n",
       "\n",
       "   Inverse Label Noise  Label Quality Score  \n",
       "0             0.137843             0.833216  \n",
       "1             0.075727             0.946354  \n",
       "2             0.039336             0.952938  \n",
       "3             0.051108             0.955072  \n",
       "4             0.055008             0.962791  \n",
       "5             0.020320             0.970964  \n",
       "6             0.019414             0.978546  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Class Overlap. In some cases, you may want to merge classes in the top rows (below)\n",
      "-----------------------------------------------------------------------------------\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class Name A</th>\n",
       "      <th>Class Name B</th>\n",
       "      <th>Class Index A</th>\n",
       "      <th>Class Index B</th>\n",
       "      <th>Num Overlapping Examples</th>\n",
       "      <th>Joint Probability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1286</td>\n",
       "      <td>0.010904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>858</td>\n",
       "      <td>0.007275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>839</td>\n",
       "      <td>0.007114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>550</td>\n",
       "      <td>0.004664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>543</td>\n",
       "      <td>0.004604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>277</td>\n",
       "      <td>0.002349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>212</td>\n",
       "      <td>0.001798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>178</td>\n",
       "      <td>0.001509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>135</td>\n",
       "      <td>0.001145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>106</td>\n",
       "      <td>0.000899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>102</td>\n",
       "      <td>0.000865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>95</td>\n",
       "      <td>0.000806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>93</td>\n",
       "      <td>0.000789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>88</td>\n",
       "      <td>0.000746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>71</td>\n",
       "      <td>0.000602</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>59</td>\n",
       "      <td>0.000500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>55</td>\n",
       "      <td>0.000466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>51</td>\n",
       "      <td>0.000432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>42</td>\n",
       "      <td>0.000356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>31</td>\n",
       "      <td>0.000263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>25</td>\n",
       "      <td>0.000212</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Class Name A  Class Name B  Class Index A  Class Index B   \n",
       "0              1             2              1              2  \\\n",
       "1              0             1              0              1   \n",
       "2              2             6              2              6   \n",
       "3              0             2              0              2   \n",
       "4              2             3              2              3   \n",
       "5              1             6              1              6   \n",
       "6              2             4              2              4   \n",
       "7              1             3              1              3   \n",
       "8              4             6              4              6   \n",
       "9              0             3              0              3   \n",
       "10             2             5              2              5   \n",
       "11             3             6              3              6   \n",
       "12             1             4              1              4   \n",
       "13             0             6              0              6   \n",
       "14             0             4              0              4   \n",
       "15             1             5              1              5   \n",
       "16             3             5              3              5   \n",
       "17             3             4              3              4   \n",
       "18             4             5              4              5   \n",
       "19             5             6              5              6   \n",
       "20             0             5              0              5   \n",
       "\n",
       "    Num Overlapping Examples  Joint Probability  \n",
       "0                       1286           0.010904  \n",
       "1                        858           0.007275  \n",
       "2                        839           0.007114  \n",
       "3                        550           0.004664  \n",
       "4                        543           0.004604  \n",
       "5                        277           0.002349  \n",
       "6                        212           0.001798  \n",
       "7                        178           0.001509  \n",
       "8                        135           0.001145  \n",
       "9                        106           0.000899  \n",
       "10                       102           0.000865  \n",
       "11                        95           0.000806  \n",
       "12                        93           0.000789  \n",
       "13                        88           0.000746  \n",
       "14                        71           0.000602  \n",
       "15                        59           0.000500  \n",
       "16                        55           0.000466  \n",
       "17                        51           0.000432  \n",
       "18                        42           0.000356  \n",
       "19                        31           0.000263  \n",
       "20                        25           0.000212  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " * Overall, about 5% (5,696 of the 117,937) labels in your dataset have potential issues.\n",
      " ** The overall label health score for this dataset is: 0.95.\n",
      "\n",
      "Generated with <3 from Cleanlab.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from cleanlab.dataset import health_summary\n",
    "\n",
    "class_names=[0,1,2,3,4,5,6]\n",
    "summary = health_summary(data['target'], probs, class_names=class_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Change Error by Threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_logits = np.array(probs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'f1': 0.05952908352452441}"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import evaluate\n",
    "\n",
    "# Error 이슈에 대한 macro f1 \n",
    "f1 = evaluate.load('f1')\n",
    "f1.compute(predictions=np.argmax(model_logits[ordered_label_issues], axis=1), references=data.iloc[ordered_label_issues]['target'].values, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1.2225989e-03 7.4410113e-04 8.2024140e-03 9.8873597e-01 1.7811389e-04\n",
      " 5.7038205e-04 3.4642071e-04]\n",
      "[0.59375054 0.38971087 0.00827854 0.00182783 0.00181179 0.00385392\n",
      " 0.00076651]\n"
     ]
    }
   ],
   "source": [
    "# model이 어느 정도의 확신으로 예측했는지\n",
    "# `ordered_label_issues` 는 모델의 라벨값과 가장 다른 순으로 정렬\n",
    "print(model_logits[ordered_label_issues][0])\n",
    "print(model_logits[ordered_label_issues][-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# logi2s의 threshold를 찾기\n",
    "i = 98\n",
    "print('Sentence:', data.iloc[ordered_label_issues[i]]['text'])\n",
    "print('Model Prob:', model_logits[ordered_label_issues[i]])\n",
    "print('Model Pred:', id2label[np.argmax(model_logits[ordered_label_issues[i]])])\n",
    "print('Train Label:', id2label[data.iloc[ordered_label_issues[i]]['target']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 모델이 99%로 예측하면 변경\n",
    "for idx in ordered_label_issues:\n",
    "    if model_logits[idx].max() >= 0.99:\n",
    "        data.iloc[idx, 2] = np.argmax(model_logits[idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "non_json = []\n",
    "\n",
    "for idx in ordered_label_issues:\n",
    "    try:\n",
    "        json_label = title2label[data.iloc[idx]['text']]\n",
    "        json_exists = True\n",
    "    except:\n",
    "        json_exists = False\n",
    "        non_json.append(idx)\n",
    "    \n",
    "    if json_exists:\n",
    "        data.iloc[idx, 2] = label2id[json_label]\n",
    "    else:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv('./data/train_aihub+LED.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "target\n",
       "2    41124\n",
       "1    20923\n",
       "3    20892\n",
       "6    15194\n",
       "5     7709\n",
       "4     6469\n",
       "0     5626\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.target.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentence: 자고나면 한 명씽 느느 녜비후보…열기 다라오르는 서부산\n",
      "Model Prob: [1.9274342e-03 9.9778979e-04 2.5337322e-03 9.9153203e-01 8.4386056e-04\n",
      " 1.4843867e-03 6.8072486e-04]\n",
      "Model Pred: 생활문화\n",
      "Train Label: 정치\n"
     ]
    }
   ],
   "source": [
    "# logits의 threshold를 찾기\n",
    "i = 111\n",
    "print('Sentence:', data.iloc[i]['text'])\n",
    "print('Model Prob:', model_logits[i])\n",
    "print('Model Pred:', id2label[np.argmax(model_logits[i])])\n",
    "print('Train Label:', id2label[data.iloc[i]['target']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[6458,\n",
       " 111,\n",
       " 797,\n",
       " 685,\n",
       " 2207,\n",
       " 2365,\n",
       " 4393,\n",
       " 6717,\n",
       " 5831,\n",
       " 5101,\n",
       " 2724,\n",
       " 2096,\n",
       " 1681,\n",
       " 2879,\n",
       " 6638,\n",
       " 841,\n",
       " 1382,\n",
       " 4738,\n",
       " 62,\n",
       " 157,\n",
       " 5953,\n",
       " 5364,\n",
       " 2316,\n",
       " 4122,\n",
       " 2649,\n",
       " 3768,\n",
       " 4329,\n",
       " 3749,\n",
       " 3866,\n",
       " 78,\n",
       " 6027,\n",
       " 3327,\n",
       " 5497,\n",
       " 3957,\n",
       " 2515,\n",
       " 4163,\n",
       " 5245,\n",
       " 4048,\n",
       " 1458,\n",
       " 6546,\n",
       " 1865,\n",
       " 5958,\n",
       " 3189,\n",
       " 4369,\n",
       " 2355,\n",
       " 6327,\n",
       " 370,\n",
       " 2949,\n",
       " 3406,\n",
       " 4275,\n",
       " 4523,\n",
       " 2299,\n",
       " 5107,\n",
       " 2182,\n",
       " 2402,\n",
       " 3655,\n",
       " 1607,\n",
       " 4360,\n",
       " 6845,\n",
       " 6978,\n",
       " 5535,\n",
       " 405,\n",
       " 4206,\n",
       " 5754,\n",
       " 631,\n",
       " 248,\n",
       " 5092,\n",
       " 1957,\n",
       " 6666,\n",
       " 6113,\n",
       " 4718,\n",
       " 3523,\n",
       " 3119,\n",
       " 6481,\n",
       " 6297,\n",
       " 5887,\n",
       " 6375,\n",
       " 3585,\n",
       " 312,\n",
       " 6634,\n",
       " 297,\n",
       " 2921,\n",
       " 3448,\n",
       " 1377,\n",
       " 4133,\n",
       " 5352,\n",
       " 2530,\n",
       " 1346,\n",
       " 2540,\n",
       " 6650,\n",
       " 4548,\n",
       " 3925,\n",
       " 5084,\n",
       " 6829,\n",
       " 2835,\n",
       " 5386,\n",
       " 266,\n",
       " 664,\n",
       " 2873,\n",
       " 6519,\n",
       " 4319,\n",
       " 2967,\n",
       " 4556,\n",
       " 5738,\n",
       " 5938,\n",
       " 4493,\n",
       " 6674,\n",
       " 234,\n",
       " 2227,\n",
       " 6140,\n",
       " 6368,\n",
       " 447,\n",
       " 4730,\n",
       " 414,\n",
       " 6114,\n",
       " 4799,\n",
       " 5895,\n",
       " 508,\n",
       " 4189,\n",
       " 5240,\n",
       " 2913,\n",
       " 6109,\n",
       " 764,\n",
       " 5437,\n",
       " 4906,\n",
       " 708,\n",
       " 4357,\n",
       " 3673,\n",
       " 2051,\n",
       " 3505,\n",
       " 1254,\n",
       " 432,\n",
       " 5963,\n",
       " 4153,\n",
       " 2572,\n",
       " 5659,\n",
       " 4517,\n",
       " 707]"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "non_json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_data = pd.read_csv('./data/train.csv')\n",
    "\n",
    "for i in range(154):\n",
    "    new_data.iloc[ordered_label_issues[i], 2] = np.argmax(model_logits[ordered_label_issues[i]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_data.to_csv('./data/train_LED_roberta-large.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}