02_basic_imitation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "import il_tutorial.cost_graphs as cg\n",
    "import il_tutorial.util as util\n",
    "from IPython.display import HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script>\n",
       "code_show=true; \n",
       "function code_toggle() {\n",
       " if (code_show){\n",
       " $('div.input').hide();\n",
       " } else {\n",
       " $('div.input').show();\n",
       " }\n",
       " code_show = !code_show\n",
       "} \n",
       "$( document ).ready(code_toggle);\n",
       "</script>\n",
       "The raw code for this IPython notebook is by default hidden for easier reading.\n",
       "To toggle on/off the raw code, click <a href=\"javascript:code_toggle()\">here</a>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "HTML('''<script>\n",
    "code_show=true; \n",
    "function code_toggle() {\n",
    " if (code_show){\n",
    " $('div.input').hide();\n",
    " } else {\n",
    " $('div.input').show();\n",
    " }\n",
    " code_show = !code_show\n",
    "} \n",
    "$( document ).ready(code_toggle);\n",
    "</script>\n",
    "The raw code for this IPython notebook is by default hidden for easier reading.\n",
    "To toggle on/off the raw code, click <a href=\"javascript:code_toggle()\">here</a>.''')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<center>\n",
    "<h2>Imitation learning</h2>\n",
    "</center>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Imitation learning for part-of-speech tagging\n",
    "\n",
    "<table style=\"border-style: hidden; border-collapse: collapse; padding: 50px\">\n",
    "<thead>\n",
    "<tr>\n",
    "<th>I</th>\n",
    "<th>can</th>\n",
    "<th>fly</th>\n",
    "</tr>\n",
    "</thead>\n",
    "<tbody>\n",
    "<tr>\n",
    "<td><span>Pronoun</span></td>\n",
    "<td><span>Modal</span></td>\n",
    "<td><span>Verb</span></td>\n",
    "</tr>\n",
    "</tbody>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    " **Task loss**: <span class=\"fragment\">Hamming loss: number of incorrectly predicted tags</span>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "**Transition system**: <span class=\"fragment\">Tag each token left-to-right</span>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "**Expert policy**: <span class=\"fragment\">Return the next tag from the gold standard</span>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<h3>Gold standard in search space</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div id=\"a0933420-1867-11e7-bd69-df4f424074db\" class=\"carousel\" data-ride=\"carousel\" data-interval=\"false\">\n",
       "          <div class=\"carousel-inner\" role=\"listbox\">\n",
       "          <div class=\"item active\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id27\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "<div class=\"item\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id28\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id28)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "<div class=\"item\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id29\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id29)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><line marker-end=\"url(#id29)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"200\" x2=\"300\" y1=\"200\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "<div class=\"item\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id30\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id30)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><line marker-end=\"url(#id30)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"200\" x2=\"300\" y1=\"200\" y2=\"150\" /><line marker-end=\"url(#id30)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"300\" x2=\"400\" y1=\"150\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "          </div>\n",
       "          <!-- Controls -->\n",
       "          <a href=\"#a0933420-1867-11e7-bd69-df4f424074db\" role=\"button2\" data-slide=\"prev\">Previous</a>\n",
       "          &nbsp\n",
       "          <a  href=\"#a0933420-1867-11e7-bd69-df4f424074db\" role=\"button2\" data-slide=\"next\">Next</a>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<il_tutorial.util.Carousel at 0x7f5c38598278>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paths = [[],[(0,4),(1,3)],[(0,4),(1,3),(2,2)],[(0,4),(1,3),(2,2),(3,1)]]\n",
    "rows = ['Noun', 'Verb', 'Modal', 'Pronoun','NULL']\n",
    "columns = ['NULL','I', 'can', 'fly']\n",
    "cbs = []\n",
    "for path in paths:\n",
    "    cbs.append(cg.draw_cost_breakdown(rows, columns, path))\n",
    "util.Carousel(cbs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "<p>\n",
    "<ul>\n",
    "<li>Three actions to complete the output</li>\n",
    "<li>Expert policy replicates the gold standard</li>\n",
    "</ul>\n",
    "</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "<h3>Training a classifier<span class=\"fragment\" data-fragment-index=\"1\"> with structure features </span></h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id31\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id31)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><line marker-end=\"url(#id31)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"200\" x2=\"300\" y1=\"200\" y2=\"150\" /><line marker-end=\"url(#id31)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"300\" x2=\"400\" y1=\"150\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gold_path = [(0,4),(1,3),(2,2),(3,1)]\n",
    "cb_gold = cg.draw_cost_breakdown(rows, columns, gold_path)\n",
    "cb_gold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "<table style=\"font-size:100%; border-style:hidden; border-collapse:collapse; padding:50px; float:left;\">\n",
    "<thead>\n",
    "<tr>\n",
    "<th>timestep</th>\n",
    "<th>label ($\\alpha_t$)</th>\n",
    "<th>features ($\\phi(S_{t-1},\\mathbf{x})$)</th>\n",
    "</tr>\n",
    "</thead>\n",
    "<tbody>\n",
    "<tr>\n",
    "<td> $t=1$ </td>\n",
    "<td><b>Pronoun</b></td>\n",
    "<td>token=I, ...<span class=\"fragment\" data-fragment-index=\"1\">, prev=<b>NULL</b></span></td>\n",
    "</tr>\n",
    "<tr>\n",
    "<td> $t=2$ </td>\n",
    "<td><b>Modal</b></td>\n",
    "<td>token=can, ...<span class=\"fragment\" data-fragment-index=\"1\">, prev=<b>Pronoun</b></span></td>\n",
    "</tr>\n",
    "<tr>\n",
    "<td> $t=3$ </td>\n",
    "<td><b>Verb</b></td>\n",
    "<td>token=fly, ...<span class=\"fragment\" data-fragment-index=\"1\">, prev=<b>Modal</b></span></td>\n",
    "</tr>\n",
    "</tbody>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "source": [
    "The feature restricition though is needed only to be able to use dynamic programming (i.e. Viterbi) for efficient joint inference. In incremental model though this is not needed thus features can use all previous tags.\n",
    "<p style=\"float:left; font-size: 100%\" class=\"fragment\" data-fragment-index=\"2\">We learn how to imitate the expert assuming no deviations</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Algorithm\n",
    "\n",
    "<p style=\"border:3px; border-radius: 25px; background-color:lightgrey; border-style:solid; border-color:black; padding: 0.3em; font-size: 80%\">\n",
    "\\begin{align}\n",
    "& \\textbf{Input:} \\; D_{train} = \\{(\\mathbf{x}^1,\\mathbf{y}^1)...(\\mathbf{x}^M,\\mathbf{y}^M)\\}, \\; \\text{expert}\\; \\pi^{\\star}, \\; \\text{classifier} \\; H\\\\\n",
    "& \\text{set training examples}\\; \\cal E = \\emptyset\\\\\n",
    "& \\mathbf{for} \\; (\\mathbf{x},\\mathbf{y}) \\in D_{train} \\; \\mathbf{do}\\\\\n",
    "& \\quad \\text{generate expert trajectory} \\; \\alpha_1^{\\star}\\dots \\alpha_T^{\\star}  = \\pi^{\\star}(\\mathbf{x},\\mathbf{y})\\\\\n",
    "& \\quad \\mathbf{for} \\; \\alpha^{\\star}_t \\in \\alpha_1^{\\star}\\dots \\alpha_T^{\\star} \\; \\mathbf{do}\\\\\n",
    "& \\quad \\quad \\text{extract features}\\; \\mathit{feat}=\\phi(\\mathbf{x},S_{t-1}) \\\\\n",
    "& \\quad \\quad \\cal E = \\cal E \\cup (\\mathit{feat},\\alpha^{\\star}_t)\\\\\n",
    "& \\text{learn} \\; H\\; \\text{from}\\; \\cal E\\\\\n",
    "\\end{align}\n",
    "</p>\n",
    "\n",
    "\n",
    "<p style=\"float:left; font-size: 100%\" class=\"fragment\" data-fragment-index=\"2\">With logistic regression and $k$ previous tags: training a $kth$-order Maximum Entropy Markov Model (<a href=\"http://people.csail.mit.edu/mcollins/6864/slides/memm.pdf\">McCallum et al., 2000</a>)</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Exposure bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div id=\"a58d8a2a-1867-11e7-bd69-df4f424074db\" class=\"carousel\" data-ride=\"carousel\" data-interval=\"false\">\n",
       "          <div class=\"carousel-inner\" role=\"listbox\">\n",
       "          <div class=\"item active\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id31\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id31)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><line marker-end=\"url(#id31)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"200\" x2=\"300\" y1=\"200\" y2=\"150\" /><line marker-end=\"url(#id31)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"300\" x2=\"400\" y1=\"150\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "<div class=\"item\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id32\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id32)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><line marker-end=\"url(#id32)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"200\" x2=\"300\" y1=\"200\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "          </div>\n",
       "          <!-- Controls -->\n",
       "          <a href=\"#a58d8a2a-1867-11e7-bd69-df4f424074db\" role=\"button2\" data-slide=\"prev\">Previous</a>\n",
       "          &nbsp\n",
       "          <a  href=\"#a58d8a2a-1867-11e7-bd69-df4f424074db\" role=\"button2\" data-slide=\"next\">Next</a>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<il_tutorial.util.Carousel at 0x7f5c3861d080>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wrong_path = [(0,4),(1,3),(2,1)]\n",
    "cb_wrong = cg.draw_cost_breakdown(rows, columns, wrong_path)\n",
    "util.Carousel([cb_gold, cb_wrong])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "<p style=\"float: left; font-size: 80%\">We had seen: &nbsp;&nbsp; \n",
    "<table style=\"float: left; border-style: hidden; border-collapse: collapse; font-size: 80%\">\n",
    "<thead>\n",
    "<tr>\n",
    "<th>timestep</th>\n",
    "<th>label</th>\n",
    "<th>features</th>\n",
    "</tr>\n",
    "</thead>\n",
    "<tbody>\n",
    "<tr>\n",
    "<td>t=3</td>\n",
    "<td><b>Verb</b></td>\n",
    "<td>token=fly,..., prev=<b>Modal</b></td>\n",
    "</tr>\n",
    "</tbody>\n",
    "</table>\n",
    "</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "<p style=\"float: left; font-size: 80%\">but not: &nbsp;&nbsp;\n",
    "<table style=\"float: left; border-style: hidden; border-collapse: collapse; font-size: 80%\">\n",
    "<thead>\n",
    "<tr>\n",
    "<th>timestep</th>\n",
    "<th>label</th>\n",
    "<th>features</th>\n",
    "</tr>\n",
    "</thead>\n",
    "<tbody>\n",
    "<tr>\n",
    "<td>t=3</td>\n",
    "<td><b>Verb</b></td>\n",
    "<td>token=fly,..., <span style=\"color:red\">prev=<b>Verb</b></span></td>\n",
    "</tr>\n",
    "</tbody>\n",
    "</table></p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Addressing exposure with Rollins\n",
    "\n",
    "<p style=\"float: left;\">Allow the classifier to guide the learning<br></p>  <a href=\"https://www.pinterest.com/explore/affordable-driving-school/\"><img src=\"images/driving_mix.jpg\" style=\"width:35%; float: right;\"></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Define a **rollin** policy that sometimes uses the expert $\\pi^{\\star}$ and other times the classifier $H$:\n",
    "\n",
    "$$\\pi^{in} = \\beta\\pi^{\\star} + (1-\\beta)H$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### DAgger algorithm\n",
    "\n",
    "<p style=\"border:3px; border-radius: 25px; background-color:lightgrey; border-style:solid; border-color:black; padding: 0.3em; font-size: 80%\">\n",
    "\\begin{align}\n",
    "& \\textbf{Input:} \\; D_{train} = \\{(\\mathbf{x}^1,\\mathbf{y}^1)...(\\mathbf{x}^M,\\mathbf{y}^M)\\}, \\; \\text{expert}\\; \\pi^{\\star}, \\; \\text{classifier} \\; H\\\\\n",
    "& \\text{set training examples}\\; \\cal E = \\emptyset ,\\; \\color{red}{\\pi^{\\star}\\; \\mathrm{probability}\\; \\beta=1}\\\\\n",
    "& \\mathbf{while}\\; \\text{termination condition not reached}\\; \\mathbf{do}\\\\\n",
    "& \\quad \\color{red}{\\text{set rollin policy} \\; \\pi^{in} = \\beta\\pi^{\\star} + (1-\\beta)H}\\\\\n",
    "& \\quad \\mathbf{for} \\; (\\mathbf{x},\\mathbf{y}) \\in D_{train} \\; \\mathbf{do}\\\\\n",
    "& \\quad \\quad \\color{red}{\\text{generate trajectory} \\; \\hat \\alpha_1\\dots\\hat \\alpha_T  = \\pi^{in}(\\mathbf{x},\\mathbf{y})}\\\\\n",
    "& \\quad \\quad \\mathbf{for} \\; \\hat \\alpha_t \\in \\hat \\alpha_1\\dots\\hat \\alpha_T \\; \\mathbf{do}\\\\\n",
    "& \\quad \\quad \\quad \\color{red}{\\text{ask expert for best action}\\; \\alpha^{\\star} = \\pi^{\\star}(\\mathbf{x},S_{t-1})} \\\\\n",
    "& \\quad \\quad \\quad \\text{extract features} \\; \\mathit{feat}=\\phi(\\mathbf{x},S_{t-1}) \\\\\n",
    "& \\quad \\quad \\quad \\cal E = \\cal E \\cup (\\mathit{feat},\\alpha^{\\star})\\\\\n",
    "& \\quad \\text{learn}\\; H \\; \\text{from}\\; \\cal E\\\\\n",
    "& \\quad \\color{red}{\\text{decrease} \\; \\beta}\\\\\n",
    "\\end{align}\n",
    "</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "Proposed by [Ross et al. (2011)](http://www.cs.cmu.edu/~sross1/publications/Ross-AIStats11-NoRegret.pdf) motivated by robotics\n",
    "- first iteration is standard classification training\n",
    "- task loss and gold standard are implicitly considered via the expert\n",
    "- DAgger: the Datasets in each iteration are Aggregated"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "**rollins** expose to previous mistakes. Future ones?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "**rollouts**: expose the classifier to future mistakes!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Training labels as costs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id5\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id5)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><line marker-end=\"url(#id5)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"200\" x2=\"300\" y1=\"200\" y2=\"150\" /><line marker-end=\"url(#id5)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"300\" x2=\"400\" y1=\"150\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cb_gold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "source": [
    "<table style=\"float: left; border-style: hidden; border-collapse: collapse; float:left;\">\n",
    "<thead>\n",
    "<tr>\n",
    "<th>timestep</th>\n",
    "<th><b>Pronoun</b></th>\n",
    "<th><b>Modal</b></th>\n",
    "<th><b>Verb</b></th>\n",
    "<th><b>Noun</b></th>\n",
    "<th>features</th>\n",
    "</tr>\n",
    "</thead>\n",
    "<tbody>\n",
    "<tr>\n",
    "<td>t=1</td>\n",
    "<td>0</td>\n",
    "<td>1</td>\n",
    "<td>1</td>\n",
    "<td>1</td>\n",
    "<td>token=I, prev=<b>NULL</b>...</td>\n",
    "</tr>\n",
    "<tr>\n",
    "<td>t=2</td>\n",
    "<td>1</td>\n",
    "<td>0</td>\n",
    "<td>1</td>\n",
    "<td>1</td>\n",
    "<td>token=can, prev=<b>Pronoun</b>...</td>\n",
    "</tr>\n",
    "<tr>\n",
    "<td>t=3</td>\n",
    "<td>1</td>\n",
    "<td>1</td>\n",
    "<td>0</td>\n",
    "<td>1</td>\n",
    "<td>token=fly, prev=<b>Modal</b>...</td>\n",
    "</tr>\n",
    "</tbody>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "<h3>Cost break down</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div id=\"a92a7990-1867-11e7-bd69-df4f424074db\" class=\"carousel\" data-ride=\"carousel\" data-interval=\"false\">\n",
       "          <div class=\"carousel-inner\" role=\"listbox\">\n",
       "          <div class=\"item active\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id33\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id33)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"185\" y=\"185\">rollin</text><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "<div class=\"item\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id34\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id34)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><line marker-end=\"url(#id34)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"200\" x2=\"300\" y1=\"200\" y2=\"50\" /><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"185\" y=\"185\">rollin</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"285\" y=\"35\">explore</text><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "<div class=\"item\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id35\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id35)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><line marker-end=\"url(#id35)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"200\" x2=\"300\" y1=\"200\" y2=\"50\" /><line marker-end=\"url(#id35)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"300\" x2=\"400\" y1=\"50\" y2=\"100\" /><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"185\" y=\"185\">rollin</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"285\" y=\"35\">explore</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"385\" y=\"35\">rollout</text><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "<div class=\"item\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id36\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id36)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><line marker-end=\"url(#id36)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"200\" x2=\"300\" y1=\"200\" y2=\"50\" /><line marker-end=\"url(#id36)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"300\" x2=\"400\" y1=\"50\" y2=\"100\" /><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"410\" y=\"100\">cost = 1</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"185\" y=\"185\">rollin</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"285\" y=\"35\">explore</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"385\" y=\"35\">rollout</text><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "<div class=\"item\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id37\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id37)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><line marker-end=\"url(#id37)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"200\" x2=\"300\" y1=\"200\" y2=\"100\" /><line marker-end=\"url(#id37)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"300\" x2=\"400\" y1=\"100\" y2=\"100\" /><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"410\" y=\"100\">cost = 1</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"185\" y=\"185\">rollin</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"285\" y=\"85\">explore</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"385\" y=\"35\">rollout</text><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "<div class=\"item\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id38\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id38)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><line marker-end=\"url(#id38)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"200\" x2=\"300\" y1=\"200\" y2=\"150\" /><line marker-end=\"url(#id38)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"300\" x2=\"400\" y1=\"150\" y2=\"100\" /><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"410\" y=\"100\">cost = 0</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"185\" y=\"185\">rollin</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"285\" y=\"135\">explore</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"385\" y=\"35\">rollout</text><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "<div class=\"item\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id39\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id39)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><line marker-end=\"url(#id39)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"200\" x2=\"300\" y1=\"200\" y2=\"200\" /><line marker-end=\"url(#id39)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"300\" x2=\"400\" y1=\"200\" y2=\"100\" /><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"410\" y=\"100\">cost = 1</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"185\" y=\"185\">rollin</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"285\" y=\"185\">explore</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"385\" y=\"35\">rollout</text><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "          </div>\n",
       "          <!-- Controls -->\n",
       "          <a href=\"#a92a7990-1867-11e7-bd69-df4f424074db\" role=\"button2\" data-slide=\"prev\">Previous</a>\n",
       "          &nbsp\n",
       "          <a  href=\"#a92a7990-1867-11e7-bd69-df4f424074db\" role=\"button2\" data-slide=\"next\">Next</a>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<il_tutorial.util.Carousel at 0x7f5c38551668>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = gold_path.copy()\n",
    "cost = 1\n",
    "cb_costs = []\n",
    "cb_costs.append(cg.draw_cost_breakdown(rows, columns, [(0,4),(1,3)], roll_in_cell=p[1]))\n",
    "cb_costs.append(cg.draw_cost_breakdown(rows, columns, [(0,4),(1,3),(2,0)], roll_in_cell=p[1], explore_cell=(2,0)))\n",
    "cb_costs.append(cg.draw_cost_breakdown(rows, columns, [(0,4),(1,3),(2,0),(3,1)], roll_in_cell=p[1], explore_cell=(2,0),roll_out_cell=(3,0)))\n",
    "cb_costs.append(cg.draw_cost_breakdown(rows, columns, [(0,4),(1,3),(2,0),(3,1)], cost, p[3], roll_in_cell=p[1], explore_cell=(2,0),roll_out_cell=(3,0)))\n",
    "for i in range(1,4):\n",
    "    p = gold_path.copy()\n",
    "    p[2] = (gold_path[2][0],i)\n",
    "    if p == gold_path:\n",
    "        cost = 0\n",
    "    else:\n",
    "        cost = 1\n",
    "    cb_costs.append(cg.draw_cost_breakdown(rows, columns, p, cost, p[3], roll_in_cell=p[1],roll_out_cell=(3,0), explore_cell=p[2]))\n",
    "util.Carousel(cb_costs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "<table style=\"border-style: hidden; border-collapse: collapse; font-size: 90%\">\n",
    "<thead>\n",
    "<tr>\n",
    "<th>step</th>\n",
    "<th><b>Pronoun</b></th>\n",
    "<th><b>Modal</b></th>\n",
    "<th><b>Verb</b></th>\n",
    "<th><b>Noun</b></th>\n",
    "<th>features</th>\n",
    "</tr>\n",
    "</thead>\n",
    "<tbody>\n",
    "<tr>\n",
    "<td>t=2</td>\n",
    "<td>1</td>\n",
    "<td>0</td>\n",
    "<td>1</td>\n",
    "<td>1</td>\n",
    "<td>token=can, prev=<b>Pronoun</b>...</td>\n",
    "</tr>\n",
    "</tbody>\n",
    "</table>\n",
    "<ul style=\"font-size: 90%\">\n",
    "<li><b>rollin</b> to a point in the sentence</li>\n",
    "<li><b>explore</b> each action: <b>rollout</b> and cost with task loss</li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Mixed rollouts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true,
    "slideshow": {
     "slide_type": "-"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div id=\"ab083388-1867-11e7-bd69-df4f424074db\" class=\"carousel\" data-ride=\"carousel\" data-interval=\"false\">\n",
       "          <div class=\"carousel-inner\" role=\"listbox\">\n",
       "          <div class=\"item active\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id40\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id40)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"185\" y=\"185\">rollin</text><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "<div class=\"item\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id41\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id41)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><line marker-end=\"url(#id41)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"200\" x2=\"300\" y1=\"200\" y2=\"50\" /><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"185\" y=\"185\">rollin</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"285\" y=\"35\">explore</text><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "<div class=\"item\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id42\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id42)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><line marker-end=\"url(#id42)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"200\" x2=\"300\" y1=\"200\" y2=\"50\" /><line marker-end=\"url(#id42)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"300\" x2=\"400\" y1=\"50\" y2=\"100\" /><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"410\" y=\"100\">cost = 1</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"185\" y=\"185\">rollin</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"285\" y=\"35\">explore</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"385\" y=\"35\">rollout</text><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "<div class=\"item\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id43\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id43)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><line marker-end=\"url(#id43)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"200\" x2=\"300\" y1=\"200\" y2=\"100\" /><line marker-end=\"url(#id43)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"300\" x2=\"400\" y1=\"100\" y2=\"50\" /><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"410\" y=\"50\">cost = 2</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"185\" y=\"185\">rollin</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"285\" y=\"85\">explore</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"385\" y=\"35\">rollout</text><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "<div class=\"item\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id44\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id44)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><line marker-end=\"url(#id44)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"200\" x2=\"300\" y1=\"200\" y2=\"150\" /><line marker-end=\"url(#id44)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"300\" x2=\"400\" y1=\"150\" y2=\"100\" /><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"410\" y=\"100\">cost = 0</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"185\" y=\"185\">rollin</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"285\" y=\"135\">explore</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"385\" y=\"35\">rollout</text><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "<div class=\"item\"><svg baseProfile=\"full\" height=\"315.0\" version=\"1.1\" width=\"500\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:ev=\"http://www.w3.org/2001/xml-events\" xmlns:xlink=\"http://www.w3.org/1999/xlink\"><defs><marker id=\"id45\" markerHeight=\"10\" markerWidth=\"10\" orient=\"auto\" refX=\"5\" refY=\"5\"><path d=\"M 0 0 L 10 5 L 0 10 z\" fill=\"blue\" /></marker></defs><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"50\">Noun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"100\">Verb</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"150\">Modal</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"200\">Pronoun</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"0\" y=\"250\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"100\" y=\"300\">NULL</text><text style=\"font-size:25px; font-family:Lato\" x=\"200\" y=\"300\">I</text><text style=\"font-size:25px; font-family:Lato\" x=\"300\" y=\"300\">can</text><text style=\"font-size:25px; font-family:Lato\" x=\"400\" y=\"300\">fly</text><line marker-end=\"url(#id45)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"100\" x2=\"200\" y1=\"250\" y2=\"200\" /><line marker-end=\"url(#id45)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"200\" x2=\"300\" y1=\"200\" y2=\"200\" /><line marker-end=\"url(#id45)\" stroke=\"rgb(10%,10%,16%)\" stroke-width=\"2.5\" x1=\"300\" x2=\"400\" y1=\"200\" y2=\"100\" /><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"410\" y=\"100\">cost = 1</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"185\" y=\"185\">rollin</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"285\" y=\"185\">explore</text><text style=\"font-size:25px; font-weight:bold; font-family:Lato\" x=\"385\" y=\"35\">rollout</text><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"105\" x2=\"195\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"205\" x2=\"295\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"50\" y2=\"50\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"100\" y2=\"100\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"150\" y2=\"150\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"200\" y2=\"200\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"305\" x2=\"395\" y1=\"250\" y2=\"250\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"100\" x2=\"100\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"200\" x2=\"200\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"300\" x2=\"300\" y1=\"205\" y2=\"245\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"55\" y2=\"95\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"105\" y2=\"145\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"155\" y2=\"195\" /><line stroke=\"rgb(10%,10%,16%)\" stroke-dasharray=\"5 5\" x1=\"400\" x2=\"400\" y1=\"205\" y2=\"245\" /></svg></div>\n",
       "          </div>\n",
       "          <!-- Controls -->\n",
       "          <a href=\"#ab083388-1867-11e7-bd69-df4f424074db\" role=\"button2\" data-slide=\"prev\">Previous</a>\n",
       "          &nbsp\n",
       "          <a  href=\"#ab083388-1867-11e7-bd69-df4f424074db\" role=\"button2\" data-slide=\"next\">Next</a>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<il_tutorial.util.Carousel at 0x7f5c385a4f60>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cb_mix_costs = []\n",
    "cb_mix_costs.append(cg.draw_cost_breakdown(rows, columns, [(0,4),(1,3)], roll_in_cell=p[1]))\n",
    "cb_mix_costs.append(cg.draw_cost_breakdown(rows, columns, [(0,4),(1,3),(2,0)], roll_in_cell=p[1], explore_cell=(2,0)))\n",
    "for i in range(4):\n",
    "    p = gold_path.copy()\n",
    "    p[2] = (gold_path[2][0],i)\n",
    "    if p == gold_path:\n",
    "        cost = 0\n",
    "    elif i==1:\n",
    "        cost =2\n",
    "        p[3] = (3,0)\n",
    "    else:\n",
    "        cost = 1\n",
    "    cb_mix_costs.append(cg.draw_cost_breakdown(rows, columns, p, cost, p[3], roll_in_cell=p[1],roll_out_cell=(3,0), explore_cell=p[2]))\n",
    "util.Carousel(cb_mix_costs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "<table style=\"border-style: hidden; border-collapse: collapse; padding: 50px;\">\n",
    "<thead>\n",
    "<tr>\n",
    "<th><b>Pronoun</b></th>\n",
    "<th><b>Modal</b></th>\n",
    "<th><b>Verb</b></th>\n",
    "<th><b>Noun</b></th>\n",
    "<th>features</th>\n",
    "</tr>\n",
    "</thead>\n",
    "<tbody>\n",
    "<tr>\n",
    "<td>1</td>\n",
    "<td>0</td>\n",
    "<td>2</td>\n",
    "<td>1</td>\n",
    "<td>token=can, prev=<b>Pronoun</b>...</td>\n",
    "</tr>\n",
    "</tbody>\n",
    "</table>\n",
    "\n",
    "<p>\n",
    "**Mixed rollout** with the expert $\\pi^{\\star}$ and the classifier $H$:\n",
    "\n",
    "$$\\pi^{out} = \\beta\\pi^{\\star} + (1-\\beta)H$$\n",
    "</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### DAgger with roll-outs\n",
    "\n",
    "<p style=\"border:3px; border-radius: 25px; background-color:lightgrey; border-style:solid; border-color:black; padding: 0.3em; font-size: 70%\">\n",
    "\\begin{align}\n",
    "& \\textbf{Input:} \\; D_{train} = \\{(\\mathbf{x}^1,\\mathbf{y}^1)...(\\mathbf{x}^M,\\mathbf{y}^M)\\}, \\; \\text{expert}\\; \\pi^{\\star}, \\; \\text{classifier} \\; H, \\; \\text{loss} \\; L\\\\\n",
    "& \\text{set training examples}\\; \\cal E = \\emptyset, \\; \\pi^{\\star}\\; \\mathrm{probability}\\; \\beta=1\\\\\n",
    "& \\mathbf{while}\\; \\text{termination condition not reached}\\; \\mathbf{do}\\\\\n",
    "& \\quad \\color{red}{\\text{set rollin/out policy} \\; \\pi^{in/out} = \\beta\\pi^{\\star} + (1-\\beta)H}\\\\\n",
    "& \\quad \\mathbf{for} \\; (\\mathbf{x},\\mathbf{y}) \\in D_{train} \\; \\mathbf{do}\\\\\n",
    "& \\quad \\quad \\text{rollin to predict} \\; \\hat \\alpha_1\\dots\\hat \\alpha_T  = \\pi^{in/out}(\\mathbf{x},\\mathbf{y})\\\\\n",
    "& \\quad \\quad \\mathbf{for} \\; \\hat \\alpha_t \\in \\hat \\alpha_1\\dots\\hat \\alpha_T \\; \\mathbf{do}\\\\\n",
    "& \\quad \\quad \\quad \\mathbf{for} \\; \\alpha \\in {\\cal A} \\; \\mathbf{do}\\\\\n",
    "& \\quad \\quad \\quad \\quad \\color{red}{\\text{rollout} \\; S_{final} = \\pi^{in/out}(S_{t-1}, \\alpha, \\mathbf{x})}\\\\\n",
    "& \\quad \\quad \\quad \\quad \\color{red}{\\text{cost}\\; c_{\\alpha}=L(S_{final}, \\mathbf{y})}\\\\\n",
    "& \\quad \\quad \\quad \\text{extract features}\\; \\mathit{feat}=\\phi(\\mathbf{x}, S_{t-1}) \\\\\n",
    "& \\quad \\quad \\quad \\cal E = \\cal E \\cup (\\mathit{feat},\\mathbf{c})\\\\\n",
    "& \\quad \\text{learn} \\;H \\; \\text{from}\\; \\cal E\\\\\n",
    "& \\quad \\text{decrease} \\; \\beta\\\\\n",
    "\\end{align}\n",
    "</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Roll-outs\n",
    "\n",
    "- can learn with sub-optimal experts\n",
    "- expensive when there are many actions and long sequences to complete outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "- first proposed in SEARN ([Daumé III et al., 2009](http://hunch.net/~jl/projects/reductions/searn/searn.pdf))\n",
    "\n",
    "- used to hybridise DAgger by [Vlachos and Clark (2014)](http://www.aclweb.org/anthology/Q14-1042), referred to later as V-DAgger ([Goodman et al. 2016](http://aclweb.org/anthology/P16-1001))\n",
    "\n",
    "- also proposed as look-aheads ([Tsuruoka et al. 2011](http://www.anthology.aclweb.org/W/W11/W11-0328.pdf))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### LoLS\n",
    "\n",
    "Locally Optimal Learning to Search ([Chang et al., 2015](https://arxiv.org/pdf/1502.02206.pdf))\n",
    "<img src=\"images/lols.png\" style=\"width:60%;\">\n",
    "- rollin always with the classifier\n",
    "- each rollout uses only the expert or the classifier "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "<h3>Generic imitation learning</h3>\n",
    "\n",
    "<p style=\"border:3px; border-radius: 25px; background-color:lightgrey; border-style:solid; border-color:black; padding: 0.3em; font-size: 75%\">\n",
    "\\begin{align}\n",
    "& \\textbf{Input:} \\; D_{train} = \\{(\\mathbf{x}^1,\\mathbf{y}^1)...(\\mathbf{x}^M,\\mathbf{y}^M)\\}, \\; \\text{expert}\\; \\pi^{\\star}, \\text{classifier} \\; H, \\; \\text{loss} \\; L\\\\\n",
    "& \\text{set training examples}\\; \\cal E = \\emptyset\\\\\n",
    "& \\mathbf{while}\\; \\text{termination condition not reached}\\; \\mathbf{do}\\\\\n",
    "& \\quad \\color{red}{\\text{set rollin policy} \\; \\pi^{in} = mix(H,\\pi^{\\star})}\\\\\n",
    "& \\quad \\color{red}{\\text{set rollout policy} \\; \\pi^{out} = mix(H,\\pi^{\\star})}\\\\\n",
    "& \\quad \\mathbf{for} \\; (\\mathbf{x},\\mathbf{y}) \\in D_{train} \\; \\mathbf{do}\\\\\n",
    "& \\quad \\quad \\color{red}{\\text{rollin to predict} \\; \\hat \\alpha_1\\dots\\hat \\alpha_T  = \\pi^{in}(\\mathbf{x},\\mathbf{y})}\\\\\n",
    "& \\quad \\quad \\mathbf{for} \\; \\hat \\alpha_t \\in \\hat \\alpha_1\\dots\\hat \\alpha_T \\; \\mathbf{do}\\\\\n",
    "& \\quad \\quad \\quad \\color{red}{\\text{rollout to obtain costs}\\; c \\; \\text{for all possible actions using}\\; L}\\\\\n",
    "& \\quad \\quad \\quad \\text{extract features}\\; \\mathit{feat}=\\phi(\\mathbf{x},S_{t-1}) \\\\\n",
    "& \\quad \\quad \\quad \\cal E = \\cal E \\cup (\\mathit{feat},c)\\\\\n",
    "& \\quad \\text{learn}\\; H \\; \\text{from}\\; \\cal E\\\\\n",
    "\\end{align}\n",
    "</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "### Overview\n",
    "\n",
    "<table style=\"border-style: hidden\">\n",
    "<thead>\n",
    "<tr>\n",
    "<th style=\"padding: 10px;\">Method</th>\n",
    "<th style=\"padding: 10px;\">rollin</th>\n",
    "<th style=\"padding: 10px;\">rollout</th>\n",
    "<th style=\"padding: 10px;\">loss</th>\n",
    "<th style=\"padding: 10px;\">expert decay</th>\n",
    "<th style=\"padding: 10px;\">training data</th>\n",
    "</tr>\n",
    "</thead>\n",
    "<tbody>\n",
    "<tr>\n",
    "<td style=\"padding: 10px;\">classification</td>\n",
    "<td style=\"padding: 10px;\">expert</td>\n",
    "<td style=\"padding: 10px;\">N/A</td>\n",
    "<td style=\"padding: 10px;\">0/1</td>\n",
    "<td style=\"padding: 10px;\">N/A</td>\n",
    "<td style=\"padding: 10px;\">single iteration</td>\n",
    "</tr>\n",
    "<tr>\n",
    "<td style=\"padding: 10px;\">DAgger</td>\n",
    "<td style=\"padding: 10px;\">mix</td>\n",
    "<td style=\"padding: 10px;\">N/A</td>\n",
    "<td style=\"padding: 10px;\">0/1</td>\n",
    "<td style=\"padding: 10px;\">decrease</td>\n",
    "<td style=\"padding: 10px;\">all iterations</td>\n",
    "</tr>\n",
    "<tr>\n",
    "<td style=\"padding: 10px;\">V-DAgger</td>\n",
    "<td style=\"padding: 10px;\">mix</td>\n",
    "<td style=\"padding: 10px;\">mix</td>\n",
    "<td style=\"padding: 10px;\">task</td>\n",
    "<td style=\"padding: 10px;\">exponential</td>\n",
    "<td style=\"padding: 10px;\">all iterations</td>\n",
    "</tr>\n",
    "<tr>\n",
    "<td style=\"padding: 10px;\">LOLS</td>\n",
    "<td style=\"padding: 10px;\">classifier</td>\n",
    "<td style=\"padding: 10px;\">action-level mix</td>\n",
    "<td style=\"padding: 10px;\">task</td>\n",
    "<td style=\"padding: 10px;\">no decay</td>\n",
    "<td style=\"padding: 10px;\">averaged across iterations</td>\n",
    "</tr>\n",
    "<tr>\n",
    "<td style=\"padding: 10px;\">SEARN</td>\n",
    "<td style=\"padding: 10px;\">mix</td>\n",
    "<td style=\"padding: 10px;\">mix</td>\n",
    "<td style=\"padding: 10px;\">task</td>\n",
    "<td style=\"padding: 10px;\">exponential</td>\n",
    "<td style=\"padding: 10px;\">weighted averaged across iterations</td>\n",
    "</tr>\n",
    "</tbody>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Summary so far\n",
    "\n",
    "- basic intuition behind IL\n",
    "- rollin and the DAgger algorithm \n",
    "- rollouts, V-DAgger and LOLS\n",
    "- generic imitation learning recipe"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "livereveal": {
   "height": 768,
   "start_slideshow_at": "selected",
   "theme": "solarized",
   "transition": "slide",
   "width": 1024
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}