adversarial_training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Adversairal training\n",
    "Exercise shows how to increase robustness of model by adversarial training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:43:53.218829Z",
     "start_time": "2019-05-28T12:43:48.715677Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.\n"
     ]
    }
   ],
   "source": [
    "from utils import load_news20\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import numpy as np\n",
    "from wildnlp.aspects import *\n",
    "from wildnlp.aspects.utils import compose\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:43:53.811626Z",
     "start_time": "2019-05-28T12:43:53.221678Z"
    }
   },
   "outputs": [],
   "source": [
    "# download the 20 News Group dataset, this may take a minute\n",
    "train_data, test_data, class_names = load_news20()\n",
    "\n",
    "# extract train and test datasets\n",
    "X_raw_train, y_train = train_data\n",
    "X_raw_test, y_test = test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:43:53.821069Z",
     "start_time": "2019-05-28T12:43:53.814477Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['atheism',\n",
       " 'graphics',\n",
       " 'ms-windows.misc',\n",
       " 'pc.hardware',\n",
       " 'mac.hardware',\n",
       " 'ms-windows.x',\n",
       " 'misc.forsale',\n",
       " 'autos',\n",
       " 'motorcycles',\n",
       " 'baseball',\n",
       " 'hockey',\n",
       " 'crypt',\n",
       " 'electronics',\n",
       " 'med',\n",
       " 'space',\n",
       " 'christian',\n",
       " 'guns',\n",
       " 'mideast',\n",
       " 'politics.misc',\n",
       " 'religion.misc']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Corrupt train data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a composed corruptor function.\n",
    "# Functions will be applied in the same order they appear.\n",
    "composed = compose(QWERTY(words_percentage=30), Swap(transform_percentage=30), RemoveChar(words_percentage=30))\n",
    "X_raw_train_corrupt = [composed(x) for x in X_raw_train]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Concatenate clear and corrupt data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_raw_train_new =  X_raw_train + X_raw_train_corrupt\n",
    "y_train_new = np.concatenate((y_train, y_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Corrupt test data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:46:01.582260Z",
     "start_time": "2019-05-28T12:43:53.823636Z"
    }
   },
   "outputs": [],
   "source": [
    "# Create a composed corruptor function.\n",
    "# Functions will be applied in the same order they appear.\n",
    "composed = compose(QWERTY(words_percentage=30), Swap(transform_percentage=30), RemoveChar(words_percentage=30))\n",
    "X_raw_test_corrupt = [composed(x) for x in X_raw_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:46:01.589593Z",
     "start_time": "2019-05-28T12:46:01.586204Z"
    }
   },
   "outputs": [],
   "source": [
    "example_index = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Clear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:46:01.601531Z",
     "start_time": "2019-05-28T12:46:01.591585Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'From: Rick Miller <rick@ee.uwm.edu>\\nSubject: X-Face?\\nOrganization: Just me.\\nLines: 17\\nDistribution: world\\nNNTP-Posting-Host: 129.89.2.33\\nSummary: Go ahead... swamp me.  <EEP!>\\n\\nI\\'m not familiar at all with the format of these \"X-Face:\" thingies, but\\nafter seeing them in some folks\\' headers, I\\'ve *got* to *see* them (and\\nmaybe make one of my own)!\\n\\nI\\'ve got \"dpg-view\" on my Linux box (which displays \"uncompressed X-Faces\")\\nand I\\'ve managed to compile [un]compface too... but now that I\\'m *looking*\\nfor them, I can\\'t seem to find any X-Face:\\'s in anyones news headers!  :-(\\n\\nCould you, would you, please send me your \"X-Face:\" header?\\n\\nI *know* I\\'ll probably get a little swamped, but I can handle it.\\n\\n\\t...I hope.\\n\\nRick Miller  <rick@ee.uwm.edu> | <ricxjo@discus.mil.wi.us>   Ricxjo Muelisto\\nSend a postcard, get one back! | Enposxtigu bildkarton kaj vi ricevos alion!\\n          RICK MILLER // 16203 WOODS // MUSKEGO, WIS. 53150 // USA\\n'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_raw_test[example_index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:46:01.799155Z",
     "start_time": "2019-05-28T12:46:01.604692Z"
    }
   },
   "source": [
    "Corrupt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:47:01.018045Z",
     "start_time": "2019-05-28T12:47:01.013984Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Fo5m: Rick Millre < rick @ ee. ume. edu > Suvejct: X-Face? Organizatoin: Just m3. Lines: 17 Disrinution: word NNTP-Positng-Host: 192. 89. . 33 Sumamry: Go aeda. .. swmap e. < EEP! > Im\\' nt fsmiliar t al wihh teh fomrat 0f thsee\" XF-ace: \" tingeis, but after seein them i xoem fokle\\' haeders, I\\'ve *got* go *see* thmr (adn maybe maek on og my own)! I\\'ve gto\" dpg-view\" n y Linu bxo (hwich dislpays\" uncompressd X-Fzecs\" ) afn I\\'be managed o cmopile [ un ] compfacd oi. .. btu nwo th6a I\\'m *looikng* fro thme, I can\\'t serm to find aym XF-ace: \\'s in anones nes yeaders! : -( Coud io, wolud yku, pleae sner n yor\" X-Face: \" heaer? I *knwo* Il\\'l probbaly get litt. swamepd, bu I na hanle it. .. . I npo. Ric Milelr < dcik @ ed. uwm. eud > | < rixjxo @ dicsud. mil. wi. uw > Ricxjo Meuliso Sesn z psotxard, yte oen bcak! | Snposxitgu bildarton aj vi riecvos aino! ICK MILLER / / 12603 WO9DS / / MUSLEOG, WIS. 53150 / / USA'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_raw_test_corrupt[example_index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare clear and corrupt data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:47:14.879985Z",
     "start_time": "2019-05-28T12:47:03.391171Z"
    }
   },
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-b13bd67fa257>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# vectorize the data using tfidf, this time we'll use the whole dataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mtfidf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTfidfVectorizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmax_features\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_features\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstop_words\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'english'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mX_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtfidf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_raw_train_new\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0mX_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtfidf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_raw_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mX_test_corrupt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtfidf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_raw_test_corrupt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mMemoryError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# maximum vocabulary size \n",
    "max_features = 20000\n",
    "\n",
    "# vectorize the data using tfidf, this time we'll use the whole dataset\n",
    "tfidf = TfidfVectorizer(max_features=max_features, stop_words='english')\n",
    "X_train = tfidf.fit_transform(np.array(X_raw_train_new))\n",
    "X_test = tfidf.transform(X_raw_test)\n",
    "X_test_corrupt = tfidf.transform(X_raw_test_corrupt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T09:39:26.085449Z",
     "start_time": "2019-05-28T09:39:26.081596Z"
    }
   },
   "source": [
    "# Prepare and test model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:47:14.939555Z",
     "start_time": "2019-05-28T12:47:14.882715Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.pipeline import make_pipeline\n",
    "\n",
    "# initialize and train a naive bayes model\n",
    "model = MultinomialNB(alpha=1)\n",
    "model.fit(X=X_train, y=y_train_new)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate its performance on the clear test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:47:14.966747Z",
     "start_time": "2019-05-28T12:47:14.941627Z"
    }
   },
   "outputs": [],
   "source": [
    "y_pred = model.predict(X_test)\n",
    "accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:46:01.808086Z",
     "start_time": "2019-05-28T12:44:07.585Z"
    }
   },
   "source": [
    "Evaluate its performance on the corrupt test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T13:01:32.372188Z",
     "start_time": "2019-05-28T13:01:32.350206Z"
    }
   },
   "outputs": [],
   "source": [
    "y_pred_corrupt = model.predict(X_test_corrupt)\n",
    "accuracy_score(y_test, y_pred_corrupt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:47:23.689246Z",
     "start_time": "2019-05-28T12:47:23.686242Z"
    }
   },
   "outputs": [],
   "source": [
    "# we'll make a simple pipeline to fallback to raw text\n",
    "pipe = make_pipeline(tfidf, model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T09:41:15.279874Z",
     "start_time": "2019-05-28T09:41:15.277235Z"
    }
   },
   "source": [
    "# Black-box explainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:47:24.644443Z",
     "start_time": "2019-05-28T12:47:24.641476Z"
    }
   },
   "outputs": [],
   "source": [
    "from utils import most_important_words_black"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:50:21.198031Z",
     "start_time": "2019-05-28T12:50:21.193710Z"
    }
   },
   "outputs": [],
   "source": [
    "# feel free to change to any of the above indices\n",
    "idx_to_explain = 21\n",
    "x = X_raw_test_corrupt[idx_to_explain]\n",
    "y = y_test[idx_to_explain]\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:50:21.927925Z",
     "start_time": "2019-05-28T12:50:21.923922Z"
    }
   },
   "outputs": [],
   "source": [
    "class_names[y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:50:22.339455Z",
     "start_time": "2019-05-28T12:50:22.319790Z"
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "words_imp = most_important_words_black(x, y, pipe.predict_proba)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:50:22.690898Z",
     "start_time": "2019-05-28T12:50:22.682269Z"
    }
   },
   "outputs": [],
   "source": [
    "for ind, word, imp in words_imp:\n",
    "    print(ind, word, imp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T10:09:52.767159Z",
     "start_time": "2019-05-28T10:09:52.764690Z"
    }
   },
   "source": [
    "# Lime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:50:25.700695Z",
     "start_time": "2019-05-28T12:50:25.697750Z"
    }
   },
   "outputs": [],
   "source": [
    "from lime.lime_text import LimeTextExplainer\n",
    "\n",
    "\n",
    "explainer = LimeTextExplainer(class_names=class_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:50:27.183753Z",
     "start_time": "2019-05-28T12:50:26.152528Z"
    }
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "exp = explainer.explain_instance(x, pipe.predict_proba,  num_features=10, top_labels=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T12:50:27.295121Z",
     "start_time": "2019-05-28T12:50:27.187313Z"
    }
   },
   "outputs": [],
   "source": [
    "exp.show_in_notebook()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Geval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-28T13:13:15.606120Z",
     "start_time": "2019-05-28T13:13:15.589386Z"
    }
   },
   "outputs": [],
   "source": [
    "np.savetxt(\"geval_adv_train/out.tsv\",y_pred, fmt='%i')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! geval -t  geval_adv_train --tokenizer 13a  --metric Accuracy --worst-features > worst_features_adv_train.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}