code/BERT_finetuning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "34bcd448",
   "metadata": {},
   "source": [
    "This code is inspired from the following sources:\n",
    "- WEAT analysis: [Social Bias Argumentation](https://github.com/webis-de/argmining20-social-bias-argumentation)\n",
    "- Finetuning BERT: [HuggingFace tutorials](https://huggingface.co/course/chapter7/3?fw=tf)\n",
    "and several other similar sources."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2c12ee49",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import TrainingArguments, Trainer\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
    "from transformers import BertModel, AutoModelForSeq2SeqLM\n",
    "from transformers import DataCollatorForLanguageModeling\n",
    "\n",
    "import sys\n",
    "import numpy as np\n",
    "import re\n",
    "import json\n",
    "from sklearn.model_selection import train_test_split\n",
    "import string\n",
    "\n",
    "import torch \n",
    "from deep_translator import GoogleTranslator\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "datadir = \"wordlists/\"\n",
    "\n",
    "# Replace the path here with your desired results folder\n",
    "!mkdir results_folder\n",
    "results_folder = 'folder_name'\n",
    "\n",
    "# Replace the path here with your desired finetuning file\n",
    "path = 'raw_text_file.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e91643bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train dataset length: 2698\n",
      "Test dataset length: 477\n"
     ]
    }
   ],
   "source": [
    "\n",
    "with open(path, encoding='UTF8') as f:\n",
    "    contents = f.read()\n",
    "\n",
    "def cleanhtml(raw_html):\n",
    "  cleanr = re.compile('<.*?>')\n",
    "  cleantext = re.sub(cleanr, '', raw_html)\n",
    "  cleantext = re.sub(r'http\\S+', '', cleantext)\n",
    "  cleantext = cleantext.translate(str.maketrans('', '', string.punctuation))\n",
    "  return cleantext.split('\\n')\n",
    "\n",
    "cleaned_contents = cleanhtml(contents)\n",
    "\n",
    "def build_text_files(data_json, dest_path):\n",
    "    f = open(dest_path, 'w')\n",
    "    data = data_json\n",
    "    for texts in data_json:\n",
    "        f.write(texts + \" \")\n",
    "\n",
    "train, test = train_test_split(cleaned_contents,test_size=0.15) \n",
    "\n",
    "build_text_files(train,'train_dataset.txt')\n",
    "build_text_files(test,'test_dataset.txt')\n",
    "\n",
    "print(\"Train dataset length: \"+str(len(train)))\n",
    "print(\"Test dataset length: \"+ str(len(test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e58ea8d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-german-cased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-german-cased\")\n",
    "\n",
    "from transformers import AutoModelForMaskedLM\n",
    "\n",
    "model_checkpoint = \"bert-base-german-cased\"\n",
    "model = AutoModelForMaskedLM.from_pretrained(model_checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8037e84f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['input_ids', 'token_type_ids', 'attention_mask', 'labels'])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs = tokenizer(cleaned_contents, return_tensors='pt', max_length=512, truncation=True, padding='max_length')\n",
    "inputs['labels'] = inputs.input_ids.detach().clone()\n",
    "inputs.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "035e5554",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create random array of floats with equal dimensions to input_ids tensor\n",
    "rand = torch.rand(inputs.input_ids.shape)\n",
    "# create mask array\n",
    "mask_arr = (rand < 0.15) * (inputs.input_ids != 101) * \\\n",
    "           (inputs.input_ids != 102) * (inputs.input_ids != 0)\n",
    "\n",
    "selection = []\n",
    "\n",
    "for i in range(inputs.input_ids.shape[0]):\n",
    "    selection.append(\n",
    "        torch.flatten(mask_arr[i].nonzero()).tolist()\n",
    "    )\n",
    "for i in range(inputs.input_ids.shape[0]):\n",
    "    inputs.input_ids[i, selection[i]] = 103"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ff516611",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReviewsDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, encodings):\n",
    "        self.encodings = encodings\n",
    "    def __getitem__(self, idx):\n",
    "        return {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
    "    def __len__(self):\n",
    "        return len(self.encodings.input_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "88a96e6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = ReviewsDataset(inputs)\n",
    "loader = torch.utils.data.DataLoader(dataset, batch_size=16, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "994dbc91",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "args = TrainingArguments(\n",
    "    output_dir='out',\n",
    "    per_device_train_batch_size=64,\n",
    "    num_train_epochs=10\n",
    ")\n",
    "\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    train_dataset=dataset\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5bf27e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.9/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 3175\n",
      "  Num Epochs = 10\n",
      "  Instantaneous batch size per device = 64\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 64\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 500\n",
      "/var/folders/4g/n21d8sp530qfh0_kpw6vk_b40000gp/T/ipykernel_6560/1319218758.py:5: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  return {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2' max='500' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [  2/500 : < :, Epoch 0.02/10]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51d6bc95",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model, path+'_bert.pt')\n",
    "checkpoint = torch.load('finetuned_german_bert.pt')\n",
    "checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "d7fc29a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=0.15)\n",
    "trans = GoogleTranslator('de','en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "794e8949",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cosine_similarity(vec1, len1, vec2, len2):\n",
    "    \"\"\"calculates the cosine similarity between two vectors\"\"\"\n",
    "    if type(len1) == list or type(len2) == list:\n",
    "        return 0\n",
    "    dot_product = np.dot(vec1, vec2)\n",
    "    return dot_product / (len1 * len2)\n",
    "\n",
    "def loadwordlist(test_num):\n",
    "    \"\"\"loads and returns all words in the given file.  Omits words not in the\n",
    "    word embedding matrix\"\"\"\n",
    "    if test_num > 9:\n",
    "        print('Invalid WEAT Test.')\n",
    "        return\n",
    "    with open(datadir+\"weat_tests_german.json\") as f:\n",
    "        weat_german = json.load(f)\n",
    "        test_string = \"test\" + str(test_num)\n",
    "        weat_test = weat_german[test_string]\n",
    "    return weat_test['X'], weat_test['Y'], weat_test['A'], weat_test['B']\n",
    "\n",
    "def loadwordnames(test_num):\n",
    "    if test_num > 9:\n",
    "        print('Invalid WEAT Test.')\n",
    "        return\n",
    "    with open(datadir+\"weat_tests_german.json\") as f:\n",
    "        weat_german = json.load(f)\n",
    "        test_string = \"test\" + str(test_num)\n",
    "        weat_test = weat_german[test_string]\n",
    "        attribute_names = weat_test['attribute_words'].split(' vs. ')\n",
    "        word_names = weat_test['target_words'].split(' vs. ')\n",
    "    return word_names[0], word_names[1], attribute_names[0], attribute_names[1]\n",
    "\n",
    "def getAverageSimilarity(targetVec, targetLength, attrVectors, attrLengths):\n",
    "    \"\"\"Calculates average similarity between words in the target list and attribute\n",
    "    list\"\"\"\n",
    "    return np.average([cosine_similarity(targetVec, targetLength, attrVectors[i], attrLengths[i]) for i in range(attrLengths.shape[0])])\n",
    "\n",
    "def getListData(conceptWords, tokenizer, transformer):\n",
    "    vectors = []\n",
    "    lengths = []\n",
    "    biggest_shape = []\n",
    "    for target_idx in range(len(conceptWords)):\n",
    "        if len(conceptWords[target_idx]) > 0:\n",
    "            if 'bert' in model_name:\n",
    "                word_id = tokenizer.encode(conceptWords[target_idx])[1]\n",
    "                v = transformer.embeddings.word_embeddings.weight[word_id].detach().numpy()\n",
    "            if 't5' in model_name:\n",
    "                word_id = tokenizer(conceptWords[target_idx], return_tensors=\"pt\")\n",
    "\n",
    "                # forward pass through encoder only\n",
    "                output = transformer.encoder(\n",
    "                    input_ids=word_id[\"input_ids\"], \n",
    "                    attention_mask=word_id[\"attention_mask\"], \n",
    "                    return_dict=True\n",
    "                )\n",
    "                # get the final hidden states\n",
    "                v = output.last_hidden_state.detach().numpy()\n",
    "                biggest_shape.append(v.shape)\n",
    "            if 'gpt2' in model_name:\n",
    "                word_id = tokenizer.encode(conceptWords[target_idx])\n",
    "                v = model.transformer.wte.weight[word_id,:].detach().numpy()\n",
    "                biggest_shape.append(v.shape)\n",
    "            vectors.append(v)\n",
    "            lengths.append(np.linalg.norm(v))\n",
    "    if 't5' in model_name:\n",
    "        biggest_shape = np.maximum.reduce([np.array(b) for b in biggest_shape])\n",
    "        padded_vectors = []\n",
    "        for v in vectors:\n",
    "            vec_zeros = np.zeros(biggest_shape)\n",
    "            vec_zeros[:v.shape[0],:v.shape[1],:v.shape[2]] = v\n",
    "            padded_vectors.append(vec_zeros.reshape(-1))\n",
    "        vectors = padded_vectors\n",
    "    if 'gpt2' in model_name:\n",
    "        biggest_shape = np.maximum.reduce([np.array(b) for b in biggest_shape])\n",
    "        padded_vectors = []\n",
    "        for v in vectors:\n",
    "            vec_zeros = np.zeros(biggest_shape)\n",
    "            vec_zeros[:v.shape[0],:v.shape[1]] = v\n",
    "            padded_vectors.append(vec_zeros.reshape(-1))\n",
    "        vectors = padded_vectors\n",
    "    vectors = np.array(vectors)\n",
    "    lengths = np.array(lengths)\n",
    "    return (vectors, lengths)\n",
    "\n",
    "def rankAttributes(targetData, targetLengths, attrData, attrLengths, attrWords, n= 5):\n",
    "    \"\"\"Return the n highest similarity scores between the target and all attr\"\"\"\n",
    "    attrSims = [getAverageSimilarity(attrData[i], attrLengths[i], targetData, targetLengths) for i in range(attrLengths.shape[0])]\n",
    "    return attrWords[np.argsort(attrSims)[-n:]]\n",
    "\n",
    "def run_weat(rundirect, male_word_lists=False):\n",
    "    #parse inputs, load in glove vectors and wordlists\n",
    "    tokenizer = AutoTokenizer.from_pretrained(rundirect[0])\n",
    "    model = BertModel.from_pretrained(rundirect[0])\n",
    "\n",
    "    test_num = rundirect[1]\n",
    "\n",
    "    target1,target2,attribute1,attribute2 = loadwordlist(test_num)\n",
    "    target1Name,target2Name,attribute1Name,attribute2Name = loadwordnames(test_num)\n",
    "\n",
    "    print('WEAT Test: ', test_num)\n",
    "    print('targets: ', target1Name, target2Name)\n",
    "    print('attributes: ', attribute1Name, attribute2Name)\n",
    "\n",
    "    target1Vecs, target1Lengths = getListData(target1, tokenizer, model)\n",
    "    target2Vecs, target2Lengths = getListData(target2, tokenizer, model)\n",
    "    attr1Data, attr1Lengths = getListData(attribute1, tokenizer, model)\n",
    "    attr2Data, attr2Lengths = getListData(attribute2, tokenizer, model)\n",
    "    \n",
    "    print(len(target1Vecs), len(target1Lengths))\n",
    "    \n",
    "    #Find more similar attribute words for each target list\n",
    "    print()\n",
    "    print(\"Top 5 most similar attribute words to %s:\" % target1Name)\n",
    "\n",
    "    topWordsT1 = rankAttributes(target1Vecs, target1Lengths, np.concatenate([attr1Data, attr2Data]),\n",
    "            np.concatenate([attr1Lengths, attr2Lengths]), np.array([attribute1[0], attribute2[0]]))\n",
    "    for word in topWordsT1[::-1]:\n",
    "        print(\"\\t\"+word)\n",
    "\n",
    "    print()\n",
    "    print(\"Top 5 most similar attribute words to %s:\" % target2Name)\n",
    "\n",
    "    topWordsT2 = rankAttributes(target1Vecs, target1Lengths, np.concatenate([attr1Data, attr2Data]),\n",
    "            np.concatenate([attr1Lengths, attr2Lengths]), np.array([attribute1[1], attribute2[1]]))\n",
    "    for word in topWordsT2[::-1]:\n",
    "        print(\"\\t\"+word)\n",
    "\n",
    "    print()\n",
    "    #calculate similarities between target 1 and both attributes\n",
    "    targ1attr1Sims = [getAverageSimilarity( target1Vecs[i], target1Lengths[i], attr1Data, attr1Lengths)\n",
    "        for i in range( target1Vecs.shape[0])]\n",
    "    targ1attr2Sims = [getAverageSimilarity( target1Vecs[i], target1Lengths[i], attr2Data, attr2Lengths)\n",
    "        for i in range( target1Vecs.shape[0])]\n",
    "    targ1SimDiff = np.subtract(targ1attr1Sims, targ1attr2Sims)\n",
    "\n",
    "    #calculate similarities between target 2 and both attributes\n",
    "    targ2attr1Sims = [getAverageSimilarity( target2Vecs[i], target2Lengths[i], attr1Data, attr1Lengths)\n",
    "        for i in range( target2Vecs.shape[0])]\n",
    "    targ2attr2Sims = [getAverageSimilarity( target2Vecs[i], target2Lengths[i], attr2Data, attr2Lengths)\n",
    "        for i in range( target2Vecs.shape[0])]\n",
    "    targ2SimDiff = np.subtract(targ2attr1Sims, targ2attr2Sims)\n",
    "\n",
    "    #effect size is avg difference in similarities divided by standard dev\n",
    "    d = (np.average(targ1SimDiff) - np.average(targ2SimDiff))/np.std(np.concatenate((targ1SimDiff,targ2SimDiff)))\n",
    "\n",
    "\n",
    "    print()\n",
    "    print(\"Calculating effect size.  The score is between +2.0 and -2.0.  \")\n",
    "    print(\"Positive scores indicate that %s is more associated with %s than %s.\" % (target1Name, attribute1Name, target2Name))\n",
    "    print(\"Or, equivalently, %s is more associated with %s than %s.\" % (target2Name, attribute2Name, target1Name))\n",
    "    print(\"Negative scores have the opposite relationship.\")\n",
    "    print(\"Scores close to 0 indicate little to no effect.\")\n",
    "    print()\n",
    "    print(\"Effect size: %.2f\" % d)\n",
    "\n",
    "    print()\n",
    "    print(\"Plotting similarity scores...\")\n",
    "\n",
    "    fig = plt.figure(figsize=(16,8))\n",
    "    ax1 = fig.add_subplot(121)\n",
    "    ax2 = fig.add_subplot(122)\n",
    "    ax1.set_title(\"Similarities Scores for Target/Attribute Pairs\")\n",
    "    ax2.set_title(\"Difference Scores For Each Target\")\n",
    "\n",
    "    # Box plot of pairwise similarity scores\n",
    "    df = pd.DataFrame()\n",
    "    df[\"Similarity\"] = np.concatenate([targ1attr1Sims,targ1attr2Sims,targ2attr1Sims,targ2attr2Sims])\n",
    "    df[\"Pairs\"] = [target1Name+\"-\"+attribute1Name]*len(targ1attr1Sims)+[target1Name+\"-\"+attribute2Name]*len(targ1attr2Sims)+[target2Name+\"-\"+attribute1Name]*len(targ2attr1Sims) \\\n",
    "      +[target2Name+\"-\"+attribute2Name]*len(targ2attr2Sims)\n",
    "    df[\"Target\"] = [target1Name]*len(targ1attr1Sims+targ1attr2Sims)+[target2Name]*len(targ2attr1Sims+targ2attr2Sims)\n",
    "    df[\"Attribute\"] = [attribute1Name]*len(targ1attr1Sims)+[attribute2Name]*len(targ1attr2Sims)+[attribute1Name]*len(targ2attr1Sims) +[attribute2Name]*len(targ2attr2Sims)\n",
    "    sns.boxplot(x=\"Target\", y=\"Similarity\", hue=\"Attribute\",data=df, ax=ax1)\n",
    "    #Box plot of target bias in similarities\n",
    "    df = pd.DataFrame()\n",
    "    df[\"Difference\"] = np.concatenate([targ1SimDiff, targ2SimDiff])\n",
    "    df[\"Target\"] = [target1Name]*len(targ1SimDiff) + [target2Name]*len(targ2SimDiff)\n",
    "    ax = sns.boxplot(x=\"Target\", y=\"Difference\", data=df, ax=ax2)\n",
    "\n",
    "\n",
    "    ticks = ax1.get_yticks()\n",
    "    mx = max(abs(ticks[0]),ticks[-1])\n",
    "    mx = int(mx*10+.99)/10.0\n",
    "    ax1.yaxis.set_ticks(np.arange(-mx,mx+.1,.1))\n",
    "    ticks = ax2.get_yticks()\n",
    "    mx = max(abs(ticks[0]),ticks[-1])\n",
    "    mx = int(mx*10+.99)/10.0\n",
    "    ax2.yaxis.set_ticks(np.arange(-mx,mx+.1,.1))\n",
    "\n",
    "    fig.subplots_adjust(wspace=0.5)\n",
    "    fig.canvas.draw()\n",
    "\n",
    "    labels = [item.get_text() for item in ax1.get_yticklabels()]\n",
    "    labels[0] = \"(less similar) \" + labels[0]\n",
    "    labels[-1] = \"(more similar) \" + labels[-1]\n",
    "    ax1.set_yticklabels(labels)\n",
    "    labels = [item.get_text() for item in ax2.get_yticklabels()]\n",
    "    labels[0] = \"(%s) \" % attribute2Name + labels[0]\n",
    "    labels[len(labels)//2] = \"(neutral) 0.0\"\n",
    "    labels[-1] = \"(%s) \" % attribute1Name + labels[-1]\n",
    "    ax2.set_yticklabels(labels)\n",
    "\n",
    "    return plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "912f9df2",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(1,10):\n",
    "    rundirect = ['bert_ft', i]\n",
    "    test_name = folder + '/bert_WEAT_test_ft_'+str(rundirect[1])\n",
    "\n",
    "    orig_stdout = sys.stdout\n",
    "    f = open(test_name +'.txt', 'w')\n",
    "    sys.stdout = f\n",
    "    \n",
    "    model_name = rundirect[0]\n",
    "    if 'bert' in model_name:\n",
    "        if 'ft' not in model_name:\n",
    "            tokenizer = AutoTokenizer.from_pretrained('bert-base-german-cased')\n",
    "            model = BertModel.from_pretrained('bert-base-german-cased')\n",
    "    if 't5' in model_name:\n",
    "        tokenizer = AutoTokenizer.from_pretrained(\"ml6team/mt5-small-german-finetune-mlsum\")\n",
    "        model = AutoModelForSeq2SeqLM.from_pretrained(\"ml6team/mt5-small-german-finetune-mlsum\")\n",
    "    if 'gpt2' in model_name:\n",
    "        tokenizer = AutoTokenizer.from_pretrained(\"dbmdz/german-gpt2\")\n",
    "        model = AutoModelForCausalLM.from_pretrained(\"dbmdz/german-gpt2\")\n",
    "\n",
    "    test_num = rundirect[1]\n",
    "\n",
    "    target1,target2,attribute1,attribute2 = loadwordlist(test_num)\n",
    "    target1Name,target2Name,attribute1Name,attribute2Name = loadwordnames(test_num)\n",
    "\n",
    "    print('WEAT Test: ', test_num)\n",
    "    print('targets: ', target1Name, target2Name)\n",
    "    print('attributes: ', attribute1Name, attribute2Name)\n",
    "\n",
    "    target1Vecs, target1Lengths = getListData(target1, tokenizer, model)\n",
    "    target2Vecs, target2Lengths = getListData(target2, tokenizer, model)\n",
    "    attr1Data, attr1Lengths = getListData(attribute1, tokenizer, model)\n",
    "    attr2Data, attr2Lengths = getListData(attribute2, tokenizer, model)\n",
    "    \n",
    "    print([target1Vecs.shape, target2Vecs.shape, attr1Data.shape, attr2Data.shape])\n",
    "\n",
    "    if 't5' or 'gpt2' in model_name:\n",
    "        max_shape = np.maximum.reduce([target1Vecs.shape, target2Vecs.shape, attr1Data.shape, attr2Data.shape])\n",
    "        \n",
    "        zero_attr = np.zeros(max_shape)\n",
    "        zero_attr[:target1Vecs.shape[0],:target1Vecs.shape[1]] = target1Vecs\n",
    "        target1Vecs = zero_attr\n",
    "\n",
    "        zero_attr = np.zeros(max_shape)\n",
    "        zero_attr[:target2Vecs.shape[0],:target2Vecs.shape[1]] = target2Vecs\n",
    "        target2Vecs = zero_attr\n",
    "\n",
    "        zero_attr = np.zeros(max_shape)\n",
    "        zero_attr[:attr2Data.shape[0], :attr2Data.shape[1]] = attr2Data\n",
    "        attr2Data = zero_attr\n",
    "\n",
    "        zero_attr = np.zeros(max_shape)\n",
    "        zero_attr[:attr1Data.shape[0], :attr1Data.shape[1]] = attr1Data\n",
    "        attr1Data = zero_attr\n",
    "\n",
    "#     Find more similar attribute words for each target list\n",
    "    print(\"Top 5 most similar attribute words to %s:\" % target1Name+ \", \" + trans.translate(target1Name))\n",
    "\n",
    "    topWordsT1 = rankAttributes(target1Vecs, target1Lengths, np.concatenate([attr1Data, attr2Data]),\n",
    "            np.concatenate([attr1Lengths, attr2Lengths]), np.concatenate([attribute1, attribute2]))\n",
    "    for word in topWordsT1[::-1]:\n",
    "        print(\"\\t\"+word + \", \" + trans.translate(word))\n",
    "\n",
    "    print()\n",
    "    print(\"Top 5 most similar attribute words to %s:\" % target2Name + \", \" + trans.translate(target2Name))\n",
    "\n",
    "    topWordsT2 = rankAttributes(target2Vecs, target2Lengths, np.concatenate([attr1Data, attr2Data]),\n",
    "            np.concatenate([attr1Lengths, attr2Lengths]), np.concatenate([attribute1, attribute2]))\n",
    "    for word in topWordsT2[::-1]:\n",
    "        print(\"\\t\"+word + \", \" + trans.translate(word))\n",
    "\n",
    "    print()\n",
    "    #calculate similarities between target 1 and both attributes\n",
    "    targ1attr1Sims = [getAverageSimilarity( target1Vecs[i], target1Lengths[i], attr1Data, attr1Lengths)\n",
    "        for i in range( target1Lengths.shape[0])]\n",
    "    targ1attr2Sims = [getAverageSimilarity( target1Vecs[i], target1Lengths[i], attr2Data, attr2Lengths)\n",
    "        for i in range( target1Lengths.shape[0])]\n",
    "    targ1SimDiff = np.subtract(targ1attr1Sims, targ1attr2Sims)\n",
    "\n",
    "    #calculate similarities between target 2 and both attributes\n",
    "    targ2attr1Sims = [getAverageSimilarity( target2Vecs[i], target2Lengths[i], attr1Data, attr1Lengths)\n",
    "        for i in range( target2Lengths.shape[0])]\n",
    "    targ2attr2Sims = [getAverageSimilarity( target2Vecs[i], target2Lengths[i], attr2Data, attr2Lengths)\n",
    "        for i in range( target2Lengths.shape[0])]\n",
    "    targ2SimDiff = np.subtract(targ2attr1Sims, targ2attr2Sims)\n",
    "\n",
    "    #effect size is avg difference in similarities divided by standard dev\n",
    "    d = (np.average(targ1SimDiff) - np.average(targ2SimDiff))/np.std(np.concatenate((targ1SimDiff,targ2SimDiff)))\n",
    "\n",
    "\n",
    "    print()\n",
    "    print(\"Calculating effect size.  The score is between +2.0 and -2.0.  \")\n",
    "    print(\"Positive scores indicate that %s is more associated with %s than %s.\" % (target1Name, attribute1Name, target2Name))\n",
    "    print(\"Or, equivalently, %s is more associated with %s than %s.\" % (target2Name, attribute2Name, target1Name))\n",
    "    print(\"Negative scores have the opposite relationship.\")\n",
    "    print(\"Scores close to 0 indicate little to no effect.\")\n",
    "    print()\n",
    "    print(\"Effect size: %.2f\" % d)\n",
    "\n",
    "    print()\n",
    "    print(\"Plotting similarity scores...\")\n",
    "\n",
    "    fig = plt.figure(figsize=(16,8))\n",
    "    ax1 = fig.add_subplot(121)\n",
    "    ax2 = fig.add_subplot(122)\n",
    "    ax1.set_title(\"Similarities Scores for Target/Attribute Pairs\")\n",
    "    ax2.set_title(\"Difference Scores For Each Target\")\n",
    "\n",
    "    # Box plot of pairwise similarity scores\n",
    "    df = pd.DataFrame()\n",
    "    df[\"Similarity\"] = np.concatenate([targ1attr1Sims,targ1attr2Sims,targ2attr1Sims,targ2attr2Sims])\n",
    "    df[\"Pairs\"] = [target1Name+\"-\"+attribute1Name]*len(targ1attr1Sims)+[target1Name+\"-\"+attribute2Name]*len(targ1attr2Sims)+[target2Name+\"-\"+attribute1Name]*len(targ2attr1Sims) \\\n",
    "      +[target2Name+\"-\"+attribute2Name]*len(targ2attr2Sims)\n",
    "    df[\"Target\"] = [target1Name]*len(targ1attr1Sims+targ1attr2Sims)+[target2Name]*len(targ2attr1Sims+targ2attr2Sims)\n",
    "    df[\"Attribute\"] = [attribute1Name]*len(targ1attr1Sims)+[attribute2Name]*len(targ1attr2Sims)+[attribute1Name]*len(targ2attr1Sims) +[attribute2Name]*len(targ2attr2Sims)\n",
    "    sns.boxplot(x=\"Target\", y=\"Similarity\", hue=\"Attribute\",data=df, ax=ax1)\n",
    "    #Box plot of target bias in similarities\n",
    "    df = pd.DataFrame()\n",
    "    df[\"Difference\"] = np.concatenate([targ1SimDiff, targ2SimDiff])\n",
    "    df[\"Target\"] = [target1Name]*len(targ1SimDiff) + [target2Name]*len(targ2SimDiff)\n",
    "    ax = sns.boxplot(x=\"Target\", y=\"Difference\", data=df, ax=ax2)\n",
    "\n",
    "\n",
    "    ticks = ax1.get_yticks()\n",
    "    mx = max(abs(ticks[0]),ticks[-1])\n",
    "    mx = int(mx*10+.99)/10.0\n",
    "    ax1.yaxis.set_ticks(np.arange(-mx,mx+.1,.1))\n",
    "    ticks = ax2.get_yticks()\n",
    "    mx = max(abs(ticks[0]),ticks[-1])\n",
    "    mx = int(mx*10+.99)/10.0\n",
    "    ax2.yaxis.set_ticks(np.arange(-mx,mx+.1,.1))\n",
    "\n",
    "    fig.subplots_adjust(wspace=0.5)\n",
    "    fig.canvas.draw()\n",
    "\n",
    "    labels = [item.get_text() for item in ax1.get_yticklabels()]\n",
    "    labels[0] = \"(less similar) \" + labels[0]\n",
    "    labels[-1] = \"(more similar) \" + labels[-1]\n",
    "    ax1.set_yticklabels(labels)\n",
    "    labels = [item.get_text() for item in ax2.get_yticklabels()]\n",
    "    labels[0] = \"(%s) \" % attribute2Name + labels[0]\n",
    "    labels[len(labels)//2] = \"(neutral) 0.0\"\n",
    "    labels[-1] = \"(%s) \" % attribute1Name + labels[-1]\n",
    "    ax2.set_yticklabels(labels)\n",
    "\n",
    "    fig.savefig(test_name+'.png')\n",
    "\n",
    "    sys.stdout = orig_stdout\n",
    "    f.close()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}