main.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "book-name-extraction",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3.9.4 64-bit ('textbook-name-recognition')",
      "name": "python3810jvsc74a57bd0bf856c2005ed093fea5f62566ff4fc7c62693020706d5e5ed84212489212ec15"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.10"
    },
    "metadata": {
      "interpreter": {
        "hash": "bf856c2005ed093fea5f62566ff4fc7c62693020706d5e5ed84212489212ec15"
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KGKrY0pVptfR"
      },
      "source": [
        "# Book Name Extraction Project\n",
        "\n",
        "By Jai K. Smith (Jai.K.Smith.22@dartmouth.edu)  \n",
        "Computational Linguistics - Spring 2021\n",
        "\n",
        "**Note:** This notebook is configured to run on Google's Colab environment, and the first step (Clone Repository) downloads the rest of this repository into Colab's memory, assuming you've opened solely the `ipynb` file. If you are running this locally, you can skip the that step and begin with Creating Model."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a7ts3RVO0aSK"
      },
      "source": [
        "### Clone Repository"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CZFDnA4O0ZT-",
        "outputId": "62a7aca3-f5d1-41b3-fea6-f1ea6cb52a82"
      },
      "source": [
        "!rm -rf *\n",
        "!git clone https://github.com/jaismith/book-name-extraction\n",
        "!mv book-name-extraction/** .\n",
        "!rm -rf book-name-extraction main.ipynb"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'book-name-extraction'...\n",
            "remote: Enumerating objects: 69, done.\u001b[K\n",
            "remote: Counting objects:   1% (1/69)\u001b[K\rremote: Counting objects:   2% (2/69)\u001b[K\rremote: Counting objects:   4% (3/69)\u001b[K\rremote: Counting objects:   5% (4/69)\u001b[K\rremote: Counting objects:   7% (5/69)\u001b[K\rremote: Counting objects:   8% (6/69)\u001b[K\rremote: Counting objects:  10% (7/69)\u001b[K\rremote: Counting objects:  11% (8/69)\u001b[K\rremote: Counting objects:  13% (9/69)\u001b[K\rremote: Counting objects:  14% (10/69)\u001b[K\rremote: Counting objects:  15% (11/69)\u001b[K\rremote: Counting objects:  17% (12/69)\u001b[K\rremote: Counting objects:  18% (13/69)\u001b[K\rremote: Counting objects:  20% (14/69)\u001b[K\rremote: Counting objects:  21% (15/69)\u001b[K\rremote: Counting objects:  23% (16/69)\u001b[K\rremote: Counting objects:  24% (17/69)\u001b[K\rremote: Counting objects:  26% (18/69)\u001b[K\rremote: Counting objects:  27% (19/69)\u001b[K\rremote: Counting objects:  28% (20/69)\u001b[K\rremote: Counting objects:  30% (21/69)\u001b[K\rremote: Counting objects:  31% (22/69)\u001b[K\rremote: Counting objects:  33% (23/69)\u001b[K\rremote: Counting objects:  34% (24/69)\u001b[K\rremote: Counting objects:  36% (25/69)\u001b[K\rremote: Counting objects:  37% (26/69)\u001b[K\rremote: Counting objects:  39% (27/69)\u001b[K\rremote: Counting objects:  40% (28/69)\u001b[K\rremote: Counting objects:  42% (29/69)\u001b[K\rremote: Counting objects:  43% (30/69)\u001b[K\rremote: Counting objects:  44% (31/69)\u001b[K\rremote: Counting objects:  46% (32/69)\u001b[K\rremote: Counting objects:  47% (33/69)\u001b[K\rremote: Counting objects:  49% (34/69)\u001b[K\rremote: Counting objects:  50% (35/69)\u001b[K\rremote: Counting objects:  52% (36/69)\u001b[K\rremote: Counting objects:  53% (37/69)\u001b[K\rremote: Counting objects:  55% (38/69)\u001b[K\rremote: Counting objects:  56% (39/69)\u001b[K\rremote: Counting objects:  57% (40/69)\u001b[K\rremote: Counting objects:  59% (41/69)\u001b[K\rremote: Counting objects:  60% (42/69)\u001b[K\rremote: Counting objects:  62% (43/69)\u001b[K\rremote: Counting objects:  63% (44/69)\u001b[K\rremote: Counting objects:  65% (45/69)\u001b[K\rremote: Counting objects:  66% (46/69)\u001b[K\rremote: Counting objects:  68% (47/69)\u001b[K\rremote: Counting objects:  69% (48/69)\u001b[K\rremote: Counting objects:  71% (49/69)\u001b[K\rremote: Counting objects:  72% (50/69)\u001b[K\rremote: Counting objects:  73% (51/69)\u001b[K\rremote: Counting objects:  75% (52/69)\u001b[K\rremote: Counting objects:  76% (53/69)\u001b[K\rremote: Counting objects:  78% (54/69)\u001b[K\rremote: Counting objects:  79% (55/69)\u001b[K\rremote: Counting objects:  81% (56/69)\u001b[K\rremote: Counting objects:  82% (57/69)\u001b[K\rremote: Counting objects:  84% (58/69)\u001b[K\rremote: Counting objects:  85% (59/69)\u001b[K\rremote: Counting objects:  86% (60/69)\u001b[K\rremote: Counting objects:  88% (61/69)\u001b[K\rremote: Counting objects:  89% (62/69)\u001b[K\rremote: Counting objects:  91% (63/69)\u001b[K\rremote: Counting objects:  92% (64/69)\u001b[K\rremote: Counting objects:  94% (65/69)\u001b[K\rremote: Counting objects:  95% (66/69)\u001b[K\rremote: Counting objects:  97% (67/69)\u001b[K\rremote: Counting objects:  98% (68/69)\u001b[K\rremote: Counting objects: 100% (69/69)\u001b[K\rremote: Counting objects: 100% (69/69), done.\u001b[K\n",
            "remote: Compressing objects:   1% (1/54)\u001b[K\rremote: Compressing objects:   3% (2/54)\u001b[K\rremote: Compressing objects:   5% (3/54)\u001b[K\rremote: Compressing objects:   7% (4/54)\u001b[K\rremote: Compressing objects:   9% (5/54)\u001b[K\rremote: Compressing objects:  11% (6/54)\u001b[K\rremote: Compressing objects:  12% (7/54)\u001b[K\rremote: Compressing objects:  14% (8/54)\u001b[K\rremote: Compressing objects:  16% (9/54)\u001b[K\rremote: Compressing objects:  18% (10/54)\u001b[K\rremote: Compressing objects:  20% (11/54)\u001b[K\rremote: Compressing objects:  22% (12/54)\u001b[K\rremote: Compressing objects:  24% (13/54)\u001b[K\rremote: Compressing objects:  25% (14/54)\u001b[K\rremote: Compressing objects:  27% (15/54)\u001b[K\rremote: Compressing objects:  29% (16/54)\u001b[K\rremote: Compressing objects:  31% (17/54)\u001b[K\rremote: Compressing objects:  33% (18/54)\u001b[K\rremote: Compressing objects:  35% (19/54)\u001b[K\rremote: Compressing objects:  37% (20/54)\u001b[K\rremote: Compressing objects:  38% (21/54)\u001b[K\rremote: Compressing objects:  40% (22/54)\u001b[K\rremote: Compressing objects:  42% (23/54)\u001b[K\rremote: Compressing objects:  44% (24/54)\u001b[K\rremote: Compressing objects:  46% (25/54)\u001b[K\rremote: Compressing objects:  48% (26/54)\u001b[K\rremote: Compressing objects:  50% (27/54)\u001b[K\rremote: Compressing objects:  51% (28/54)\u001b[K\rremote: Compressing objects:  53% (29/54)\u001b[K\rremote: Compressing objects:  55% (30/54)\u001b[K\rremote: Compressing objects:  57% (31/54)\u001b[K\rremote: Compressing objects:  59% (32/54)\u001b[K\rremote: Compressing objects:  61% (33/54)\u001b[K\rremote: Compressing objects:  62% (34/54)\u001b[K\rremote: Compressing objects:  64% (35/54)\u001b[K\rremote: Compressing objects:  66% (36/54)\u001b[K\rremote: Compressing objects:  68% (37/54)\u001b[K\rremote: Compressing objects:  70% (38/54)\u001b[K\rremote: Compressing objects:  72% (39/54)\u001b[K\rremote: Compressing objects:  74% (40/54)\u001b[K\rremote: Compressing objects:  75% (41/54)\u001b[K\rremote: Compressing objects:  77% (42/54)\u001b[K\rremote: Compressing objects:  79% (43/54)\u001b[K\rremote: Compressing objects:  81% (44/54)\u001b[K\rremote: Compressing objects:  83% (45/54)\u001b[K\rremote: Compressing objects:  85% (46/54)\u001b[K\rremote: Compressing objects:  87% (47/54)\u001b[K\rremote: Compressing objects:  88% (48/54)\u001b[K\rremote: Compressing objects:  90% (49/54)\u001b[K\rremote: Compressing objects:  92% (50/54)\u001b[K\rremote: Compressing objects:  94% (51/54)\u001b[K\rremote: Compressing objects:  96% (52/54)\u001b[K\rremote: Compressing objects:  98% (53/54)\u001b[K\rremote: Compressing objects: 100% (54/54)\u001b[K\rremote: Compressing objects: 100% (54/54), done.\u001b[K\n",
            "Unpacking objects:   1% (1/69)   \rUnpacking objects:   2% (2/69)   \rUnpacking objects:   4% (3/69)   \rUnpacking objects:   5% (4/69)   \rUnpacking objects:   7% (5/69)   \rUnpacking objects:   8% (6/69)   \rUnpacking objects:  10% (7/69)   \rUnpacking objects:  11% (8/69)   \rUnpacking objects:  13% (9/69)   \rUnpacking objects:  14% (10/69)   \rUnpacking objects:  15% (11/69)   \rUnpacking objects:  17% (12/69)   \rUnpacking objects:  18% (13/69)   \rUnpacking objects:  20% (14/69)   \rUnpacking objects:  21% (15/69)   \rUnpacking objects:  23% (16/69)   \rUnpacking objects:  24% (17/69)   \rUnpacking objects:  26% (18/69)   \rUnpacking objects:  27% (19/69)   \rUnpacking objects:  28% (20/69)   \rUnpacking objects:  30% (21/69)   \rUnpacking objects:  31% (22/69)   \rUnpacking objects:  33% (23/69)   \rUnpacking objects:  34% (24/69)   \rUnpacking objects:  36% (25/69)   \rremote: Total 69 (delta 25), reused 57 (delta 13), pack-reused 0\u001b[K\n",
            "Unpacking objects:  37% (26/69)   \rUnpacking objects:  39% (27/69)   \rUnpacking objects:  40% (28/69)   \rUnpacking objects:  42% (29/69)   \rUnpacking objects:  43% (30/69)   \rUnpacking objects:  44% (31/69)   \rUnpacking objects:  46% (32/69)   \rUnpacking objects:  47% (33/69)   \rUnpacking objects:  49% (34/69)   \rUnpacking objects:  50% (35/69)   \rUnpacking objects:  52% (36/69)   \rUnpacking objects:  53% (37/69)   \rUnpacking objects:  55% (38/69)   \rUnpacking objects:  56% (39/69)   \rUnpacking objects:  57% (40/69)   \rUnpacking objects:  59% (41/69)   \rUnpacking objects:  60% (42/69)   \rUnpacking objects:  62% (43/69)   \rUnpacking objects:  63% (44/69)   \rUnpacking objects:  65% (45/69)   \rUnpacking objects:  66% (46/69)   \rUnpacking objects:  68% (47/69)   \rUnpacking objects:  69% (48/69)   \rUnpacking objects:  71% (49/69)   \rUnpacking objects:  72% (50/69)   \rUnpacking objects:  73% (51/69)   \rUnpacking objects:  75% (52/69)   \rUnpacking objects:  76% (53/69)   \rUnpacking objects:  78% (54/69)   \rUnpacking objects:  79% (55/69)   \rUnpacking objects:  81% (56/69)   \rUnpacking objects:  82% (57/69)   \rUnpacking objects:  84% (58/69)   \rUnpacking objects:  85% (59/69)   \rUnpacking objects:  86% (60/69)   \rUnpacking objects:  88% (61/69)   \rUnpacking objects:  89% (62/69)   \rUnpacking objects:  91% (63/69)   \rUnpacking objects:  92% (64/69)   \rUnpacking objects:  94% (65/69)   \rUnpacking objects:  95% (66/69)   \rUnpacking objects:  97% (67/69)   \rUnpacking objects:  98% (68/69)   \rUnpacking objects: 100% (69/69)   \rUnpacking objects: 100% (69/69), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q5KQdHU1WaZG"
      },
      "source": [
        "# Creating Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "elZjgqlsWaZI"
      },
      "source": [
        "!pip install -q -U tensorflow-text\n",
        "!pip install -q tf-models-official\n",
        "\n",
        "import json\n",
        "import os\n",
        "import shutil\n",
        "from bs4 import BeautifulSoup\n",
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "import tensorflow_text as text\n",
        "from official.nlp import optimization\n",
        "from requests import get\n",
        "import csv\n",
        "from datetime import datetime\n",
        "import random\n",
        "import json\n",
        "import re\n",
        "import numpy as np\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import constants\n",
        "from modules import loader, preprocessing"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P_6bx5QdiFlc"
      },
      "source": [
        "## Load timetable textbook fields"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L3EVIZWiiHdI"
      },
      "source": [
        "text_data = json.loads(open('data/raw/source.json', 'r').read())\n",
        "\n",
        "data = []\n",
        "for snippet in text_data:\n",
        "  soup = BeautifulSoup(snippet, 'html.parser')\n",
        "  segments = filter(\n",
        "    lambda s:\n",
        "      not s.startswith('Textbook information not yet submitted by department/program for ') and\n",
        "      not s.startswith('Textbook(s) Required for '),\n",
        "    soup.get_text(separator='<<AND>>', strip=True).split('<<AND>>')\n",
        "  )\n",
        "\n",
        "  data += list(preprocessing.standardize(segment) for segment in segments)\n",
        "\n",
        "data = preprocessing.rule_based_clumping(data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W6quV_6lWaZJ"
      },
      "source": [
        "## Download ORC data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i8CNL4BOWaZJ"
      },
      "source": [
        "if not os.path.isfile('data/raw/orc.json'):\n",
        "    loader.fetch_source(constants.DATA_SOURCE, 'orc.json')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4jN4IzHOWaZJ"
      },
      "source": [
        "## Prepare dataset for training\n",
        "\n",
        "Before running this step, please create the `data/processed` folder containing `positive/` and `negative/` subfolders."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W77hRif1WaZJ",
        "outputId": "d24ae32d-993f-46d8-94c7-631dddabc509"
      },
      "source": [
        "# clear and recreate processed dir (empty)\n",
        "if os.path.isdir('data/processed'):\n",
        "    shutil.rmtree('data/processed', ignore_errors=True)\n",
        "os.makedirs('data/processed/train/positive')\n",
        "os.mkdir('data/processed/train/negative')\n",
        "os.makedirs('data/processed/test/positive')\n",
        "os.mkdir('data/processed/test/negative')\n",
        "\n",
        "# generate dataset\n",
        "preprocessing.generate_positive()\n",
        "preprocessing.generate_negative()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Generated 11707 positive samples\n",
            "Generated 11707 negative samples\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zkDoQJVDWaZK"
      },
      "source": [
        "## Configure tf dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GaNBXr-1WaZK",
        "outputId": "93772501-7711-470b-8bc3-09ac09376c9a"
      },
      "source": [
        "AUTOTUNE = tf.data.AUTOTUNE\n",
        "batch_size = 32\n",
        "seed = 42\n",
        "\n",
        "# training set\n",
        "raw_train_ds = tf.keras.preprocessing.text_dataset_from_directory(\n",
        "    'data/processed/train',\n",
        "    batch_size=batch_size,\n",
        "    validation_split=0.2,\n",
        "    subset='training',\n",
        "    seed=seed\n",
        ")\n",
        "\n",
        "class_names = raw_train_ds.class_names\n",
        "train_ds = raw_train_ds.cache().prefetch(buffer_size=AUTOTUNE)\n",
        "\n",
        "# validation set\n",
        "val_ds = tf.keras.preprocessing.text_dataset_from_directory(\n",
        "    'data/processed/train',\n",
        "    batch_size=batch_size,\n",
        "    validation_split=0.2,\n",
        "    subset='validation',\n",
        "    seed=seed\n",
        ")\n",
        "\n",
        "val_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)\n",
        "\n",
        "# testing set\n",
        "test_ds = tf.keras.preprocessing.text_dataset_from_directory(\n",
        "    'data/processed/test',\n",
        "    batch_size=batch_size\n",
        ")\n",
        "\n",
        "test_ds = test_ds.cache().prefetch(buffer_size=AUTOTUNE)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 19900 files belonging to 2 classes.\n",
            "Using 15920 files for training.\n",
            "Found 19900 files belonging to 2 classes.\n",
            "Using 3980 files for validation.\n",
            "Found 3514 files belonging to 2 classes.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4jcXTFObWaZK"
      },
      "source": [
        "## Select BERT model\n",
        "\n",
        "[see options here](https://www.tensorflow.org/text/tutorials/classify_text_with_bert#choose_a_bert_model_to_fine-tune)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ozx8uVMkWaZL"
      },
      "source": [
        "# # use ALBERT\n",
        "# tfhub_handle_encoder = 'https://tfhub.dev/tensorflow/albert_en_base/2'\n",
        "# tfhub_handle_preprocess = 'https://tfhub.dev/tensorflow/albert_en_preprocess/3'\n",
        "\n",
        "# use small bert\n",
        "tfhub_handle_encoder = 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1'\n",
        "tfhub_handle_preprocess = 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3'\n",
        "\n",
        "bert_preprocess_model = hub.KerasLayer(tfhub_handle_preprocess)\n",
        "bert_model = hub.KerasLayer(tfhub_handle_encoder)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8yU3od9TWaZL"
      },
      "source": [
        "## Define model\n",
        "\n",
        "Based off binary text classification example [here](https://www.tensorflow.org/text/tutorials/classify_text_with_bert)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S8T8gB4IWaZL"
      },
      "source": [
        "def build_classifier_model():\n",
        "  text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text')\n",
        "  preprocessing_layer = hub.KerasLayer(tfhub_handle_preprocess, name='preprocessing')\n",
        "  encoder_inputs = preprocessing_layer(text_input)\n",
        "  encoder = hub.KerasLayer(tfhub_handle_encoder, trainable=True, name='BERT_encoder')\n",
        "  outputs = encoder(encoder_inputs)\n",
        "  net = outputs['pooled_output']\n",
        "  net = tf.keras.layers.Dropout(0.1)(net)\n",
        "  net = tf.keras.layers.Dense(1, activation=None, name='classifier')(net)\n",
        "  return tf.keras.Model(text_input, net)\n",
        "\n",
        "classifier_model = build_classifier_model()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MyYvs3n1ZwKz"
      },
      "source": [
        "## Define loss function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hvrdiwu4ZzOB"
      },
      "source": [
        "loss = tf.keras.losses.BinaryCrossentropy(from_logits=True)\n",
        "metrics = tf.metrics.BinaryAccuracy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vEErUWFyZ-IT"
      },
      "source": [
        "## Define optimizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UBNkAfABaBmt"
      },
      "source": [
        "epochs = 5\n",
        "steps_per_epoch = tf.data.experimental.cardinality(train_ds).numpy()\n",
        "num_train_steps = steps_per_epoch * epochs\n",
        "num_warmup_steps = int(0.1 * num_train_steps)\n",
        "\n",
        "init_lr = 3e-5\n",
        "optimizer = optimization.create_optimizer(init_lr=init_lr,\n",
        "                                          num_train_steps=num_train_steps,\n",
        "                                          num_warmup_steps=num_warmup_steps,\n",
        "                                          optimizer_type='adamw')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B39nkftuaFRC"
      },
      "source": [
        "## Load BERT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1f3VSK10aQ2B"
      },
      "source": [
        "classifier_model.compile(optimizer=optimizer,\n",
        "                         loss=loss,\n",
        "                         metrics=metrics)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7AvM3SqVaRle"
      },
      "source": [
        "## Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "xJaW5-IgaUBl",
        "outputId": "9b66c3f8-d023-4612-f354-f9c9b633a675"
      },
      "source": [
        "print(f'Training model with {tfhub_handle_encoder}')\n",
        "history = classifier_model.fit(x=train_ds,\n",
        "                               validation_data=val_ds,\n",
        "                               epochs=epochs)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training model with https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1\n",
            "Epoch 1/5\n",
            "498/498 [==============================] - 122s 229ms/step - loss: 0.1111 - binary_accuracy: 0.9564 - val_loss: 0.0271 - val_binary_accuracy: 0.9915\n",
            "Epoch 2/5\n",
            "498/498 [==============================] - 111s 222ms/step - loss: 0.0169 - binary_accuracy: 0.9950 - val_loss: 0.0216 - val_binary_accuracy: 0.9955\n",
            "Epoch 3/5\n",
            "498/498 [==============================] - 112s 224ms/step - loss: 0.0079 - binary_accuracy: 0.9977 - val_loss: 0.0222 - val_binary_accuracy: 0.9957\n",
            "Epoch 4/5\n",
            "498/498 [==============================] - 112s 226ms/step - loss: 0.0047 - binary_accuracy: 0.9987 - val_loss: 0.0232 - val_binary_accuracy: 0.9955\n",
            "Epoch 5/5\n",
            "498/498 [==============================] - 112s 226ms/step - loss: 0.0031 - binary_accuracy: 0.9991 - val_loss: 0.0251 - val_binary_accuracy: 0.9955\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OkGVtstYe014"
      },
      "source": [
        "## Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rHnD1qBKe2Rq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e93c13d0-adc7-41d3-8cb6-11662d0ac12f"
      },
      "source": [
        "loss, accuracy = classifier_model.evaluate(test_ds)\n",
        "\n",
        "print(f'Loss: {loss}')\n",
        "print(f'Accuracy: {accuracy}')"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "110/110 [==============================] - 11s 98ms/step - loss: 0.0302 - binary_accuracy: 0.9946\n",
            "Loss: 0.030168350785970688\n",
            "Accuracy: 0.99459308385849\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eCfYpQH8fCRZ"
      },
      "source": [
        "## Training stats"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zCIu95IifG5r",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 421
        },
        "outputId": "387ab47b-bf1a-48d5-909f-3c0cf6775f80"
      },
      "source": [
        "history_dict = history.history\n",
        "\n",
        "acc = history_dict['binary_accuracy']\n",
        "val_acc = history_dict['val_binary_accuracy']\n",
        "loss = history_dict['loss']\n",
        "val_loss = history_dict['val_loss']\n",
        "\n",
        "epochs = range(1, len(acc) + 1)\n",
        "fig = plt.figure(figsize=(10, 6))\n",
        "fig.tight_layout()\n",
        "\n",
        "plt.subplot(2, 1, 1)\n",
        "# \"bo\" is for \"blue dot\"\n",
        "plt.plot(epochs, loss, 'r', label='Training loss')\n",
        "# b is for \"solid blue line\"\n",
        "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
        "plt.title('Training and validation loss')\n",
        "# plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "plt.legend()\n",
        "\n",
        "plt.subplot(2, 1, 2)\n",
        "plt.plot(epochs, acc, 'r', label='Training acc')\n",
        "plt.plot(epochs, val_acc, 'b', label='Validation acc')\n",
        "plt.title('Training and validation accuracy')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.legend(loc='lower right')"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f0a3c26db50>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmcAAAGDCAYAAABuj7cYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeZgU1dn38e89Cwz7OiqLEVwQURF0wAVBBBUQBSUaRUWJu49GjdFEzQIxMZvGqG80xgTXqKAm8hBBiYqIWxIGQlRQE1R4RFGRfd/mfv84NdC0PTM9zPRUT8/vc119dXXVqaq7unD69pxT55i7IyIiIiLZIS/uAERERERkJyVnIiIiIllEyZmIiIhIFlFyJiIiIpJFlJyJiIiIZBElZyIiIiJZRMmZiFTJzJ4zswtqu2yczGyRmZ2QgeO6me0fLd9nZj9Mp+xunOdcM/vb7sZZyXEHmtmS2j6uiKSvIO4ARCQzzGxdwsemwGZge/T5Mnd/LN1jufuwTJTNde5+eW0cx8y6AB8Bhe6+LTr2Y0Da91BE6g8lZyI5yt2bly+b2SLgYnd/MbmcmRWU/+CLiEj81Kwp0sCUN1uZ2ffM7DPgQTNrY2bPmtkyM1sZLXdO2GemmV0cLY81s9fM7Pao7EdmNmw3y3Y1s1lmttbMXjSze8zsTxXEnU6MPzGz16Pj/c3M2idsH2Nmi81suZl9v5Lv50gz+8zM8hPWnW5mb0XLfc3sTTNbZWZLzey3ZtaogmM9ZGY/Tfh8Q7TPp2Z2YVLZ4Wb2LzNbY2Yfm9n4hM2zovdVZrbOzI4u/24T9j/GzGab2ero/Zh0v5vKmNlB0f6rzGy+mY1I2HaymS2IjvmJmV0frW8f3Z9VZrbCzF41M/3eiKRJ/7GINEx7AW2BfYBLCX8LHow+fw3YCPy2kv2PBN4H2gO/AiaYme1G2ceBfwLtgPHAmErOmU6M5wDfBPYAGgHlyUIP4HfR8TtG5+tMCu7+D2A9MCjpuI9Hy9uBb0fXczQwGPifSuImimFoFM+JwAFAcn+39cD5QGtgOHCFmZ0WbRsQvbd29+bu/mbSsdsCU4G7o2u7A5hqZu2SruEr300VMRcCfwX+Fu33LeAxMzswKjKB0ETeAjgEmBGt/w6wBCgG9gRuBjRXoEialJyJNExlwDh33+zuG919ubv/2d03uPta4FbguEr2X+zuf3D37cDDQAfCj3DaZc3sa0Af4EfuvsXdXwOmVHTCNGN80N3/4+4bgSeBXtH6M4Bn3X2Wu28Gfhh9BxV5AhgNYGYtgJOjdbj7HHf/u7tvc/dFwO9TxJHKN6L43nH39YRkNPH6Zrr72+5e5u5vRedL57gQkrn/uvujUVxPAO8BpyaUqei7qcxRQHPgF9E9mgE8S/TdAFuBHmbW0t1XuvvchPUdgH3cfau7v+qayFkkbUrORBqmZe6+qfyDmTU1s99HzX5rCM1orROb9pJ8Vr7g7huixebVLNsRWJGwDuDjigJOM8bPEpY3JMTUMfHYUXK0vKJzEWrJRplZY2AUMNfdF0dxdIua7D6L4vgZoRatKrvEACxOur4jzezlqNl2NXB5msctP/bipHWLgU4Jnyv6bqqM2d0TE9nE436dkLguNrNXzOzoaP1twELgb2b2oZndmN5liAgoORNpqJJrMb4DHAgc6e4t2dmMVlFTZW1YCrQ1s6YJ6/aupHxNYlyaeOzonO0qKuzuCwhJyDB2bdKE0Dz6HnBAFMfNuxMDoWk20eOEmsO93b0VcF/CcauqdfqU0Nyb6GvAJ2nEVdVx907qL7bjuO4+291HEpo8JxNq5HD3te7+HXffFxgBXGdmg2sYi0iDoeRMRABaEPpwrYr6L43L9AmjmqhSYLyZNYpqXU6tZJeaxPg0cIqZHRt13r+Fqv/+PQ5cQ0gCn0qKYw2wzsy6A1ekGcOTwFgz6xElh8nxtyDUJG4ys76EpLDcMkIz7L4VHHsa0M3MzjGzAjM7C+hBaIKsiX8Qatm+a2aFZjaQcI8mRvfsXDNr5e5bCd9JGYCZnWJm+0d9C1cT+ulV1owsIgmUnIkIwJ1AE+BL4O/A83V03nMJneqXAz8FJhHGY0tlt2N09/nAlYSEaymwktBhvTLlfb5muPuXCeuvJyROa4E/RDGnE8Nz0TXMIDT5zUgq8j/ALWa2FvgRUS1UtO8GQh+716MnII9KOvZy4BRC7eJy4LvAKUlxV5u7byEkY8MI3/u9wPnu/l5UZAywKGrevZxwPyE88PAisA54E7jX3V+uSSwiDYmpj6aIZAszmwS85+4Zr7kTEclWqjkTkdiYWR8z28/M8qKhJkYS+i6JiDRYmiFAROK0F/AXQuf8JcAV7v6veEMSEYmXmjVFREREsoiaNUVERESyiJIzERERkSySM33O2rdv7126dIk7DBEREZEqzZkz50t3L061LWeSsy5dulBaWhp3GCIiIiJVMrPkKdd2ULOmiIiISBZRciYiIiKSRZSciYiIiGSRnOlzVieeeAKOOQb22SfuSEREpAHbunUrS5YsYdOmTXGHIlUoKiqic+fOFBYWpr2PkrN0rVoFV10FTZvC3/4GBx0Ud0QiItJALVmyhBYtWtClSxfMLO5wpALuzvLly1myZAldu3ZNez81a6ardWt4+WXYuhX694fZs+OOSEREGqhNmzbRrl07JWZZzsxo165dtWs4lZxVR8+e8Prr0LIlDBoEM2bEHZGIiDRQSszqh925T0rOqmu//eC116BLFxg2DCZPjjsiERGROrV8+XJ69epFr1692GuvvejUqdOOz1u2bKl039LSUq6++uoqz3HMMcfUSqwzZ87klFNOqZVj1RX1OdsdHTvCK6/A8OHw9a/DH/8I3/xm3FGJiIjUiXbt2jFv3jwAxo8fT/Pmzbn++ut3bN+2bRsFBalTjJKSEkpKSqo8xxtvvFE7wdZDqjnbXW3bwosvwgknwIUXwh13xB2RiIhIbMaOHcvll1/OkUceyXe/+13++c9/cvTRR9O7d2+OOeYY3n//fWDXmqzx48dz4YUXMnDgQPbdd1/uvvvuHcdr3rz5jvIDBw7kjDPOoHv37px77rm4OwDTpk2je/fuHHHEEVx99dVV1pCtWLGC0047jZ49e3LUUUfx1ltvAfDKK6/sqPnr3bs3a9euZenSpQwYMIBevXpxyCGH8Oqrr9b6d1YR1ZzVRLNmMGUKjBkD3/kOLF8OP/0pqB+AiIjUlWuvhagWq9b06gV33lnt3ZYsWcIbb7xBfn4+a9as4dVXX6WgoIAXX3yRm2++mT//+c9f2ee9997j5ZdfZu3atRx44IFcccUVXxl24l//+hfz58+nY8eO9OvXj9dff52SkhIuu+wyZs2aRdeuXRk9enSV8Y0bN47evXszefJkZsyYwfnnn8+8efO4/fbbueeee+jXrx/r1q2jqKiI+++/nyFDhvD973+f7du3s2HDhmp/H7tLyVlNNW4cxj9r0wZ+9rOQoN1zD+Tnxx2ZiIhInTrzzDPJj37/Vq9ezQUXXMB///tfzIytW7em3Gf48OE0btyYxo0bs8cee/D555/TuXPnXcr07dt3x7pevXqxaNEimjdvzr777rtjiIrRo0dz//33Vxrfa6+9tiNBHDRoEMuXL2fNmjX069eP6667jnPPPZdRo0bRuXNn+vTpw4UXXsjWrVs57bTT6NWrV42+m+pQclYb8vPhvvugXTv4+c/DmGiPPAKNGsUdmYiI5LrdqOHKlGbNmu1Y/uEPf8jxxx/PM888w6JFixg4cGDKfRo3brxjOT8/n23btu1WmZq48cYbGT58ONOmTaNfv35Mnz6dAQMGMGvWLKZOncrYsWO57rrrOP/882v1vBVRn7PaYhZqzm67DSZNghEjYP36uKMSERGJxerVq+nUqRMADz30UK0f/8ADD+TDDz9k0aJFAEyaNKnKffr3789jjz0GhL5s7du3p2XLlnzwwQcceuihfO9736NPnz689957LF68mD333JNLLrmEiy++mLlz59b6NVREyVltu/768PTmCy/AiSfCypVxRyQiIlLnvvvd73LTTTfRu3fvWq/pAmjSpAn33nsvQ4cO5YgjjqBFixa0atWq0n3Gjx/PnDlz6NmzJzfeeCMPP/wwAHfeeSeHHHIIPXv2pLCwkGHDhjFz5kwOO+wwevfuzaRJk7jmmmtq/RoqYuVPPGTk4GZDgbuAfOCP7v6LpO0DgDuBnsDZ7v50wrYLgB9EH3/q7g9Xdq6SkhIvLS2tzfBr5i9/gdGj4cADYfp06NAh7ohERCRHvPvuuxykaQRZt24dzZs3x9258sorOeCAA/j2t78dd1hfkep+mdkcd085pkjGas7MLB+4BxgG9ABGm1mPpGL/B4wFHk/aty0wDjgS6AuMM7M2mYo1I0aNgmnT4MMP4dhjw7uIiIjUmj/84Q/06tWLgw8+mNWrV3PZZZfFHVKtyGSzZl9gobt/6O5bgInAyMQC7r7I3d8CypL2HQK84O4r3H0l8AIwNIOxZsbgwWGKp1WroF8/ePvtuCMSERHJGd/+9reZN28eCxYs4LHHHqNp06Zxh1QrMpmcdQI+Tvi8JFpXa/ua2aVmVmpmpcuWLdvtQDOqb1949dXwROeAAdCARzwWERGRqtXrBwLc/X53L3H3kuLi4rjDqViPHmE+zuLi8JDA9OlxRyQiIiJZKpPJ2SfA3gmfO0frMr1vdurSJdSgdesGp54KTz4Zd0QiIiKShTKZnM0GDjCzrmbWCDgbmJLmvtOBk8ysTfQgwEnRuvptzz1h5kw46ig4+2yoYiRjERERaXgylpy5+zbgKkJS9S7wpLvPN7NbzGwEgJn1MbMlwJnA781sfrTvCuAnhARvNnBLtK7+a9UKnn8eTj4ZLrsszCiQweFMREREatvxxx/P9KQuOnfeeSdXXHFFhfsMHDiQ8iGvTj75ZFatWvWVMuPHj+f222+v9NyTJ09mwYIFOz7/6Ec/4sUXX6xO+CklTsget4xO3+Tu04BpSet+lLA8m9BkmWrfB4AHMhlfbJo2hWeegbFj4eabw3yct92mCdNFRKReGD16NBMnTmTIkCE71k2cOJFf/epXae0/bdq0qgtVYPLkyZxyyin06BFG57rlllt2+1jZql4/EFCvFRbCo4/CVVfBr38NF18MGRhBWUREpLadccYZTJ06lS1btgCwaNEiPv30U/r3788VV1xBSUkJBx98MOPGjUu5f5cuXfjyyy8BuPXWW+nWrRvHHnss77///o4yf/jDH+jTpw+HHXYYX//619mwYQNvvPEGU6ZM4YYbbqBXr1588MEHjB07lqefDmPYv/TSS/Tu3ZtDDz2UCy+8kM2bN+8437hx4zj88MM59NBDee+99yq9vhUrVnDaaafRs2dPjjrqKN566y0AXnnlFXr16kWvXr3o3bs3a9euZenSpQwYMIBevXpxyCGH8Oqrr9bsy0UTn8crLw/uvjtMmP7jH4epnh5/HIqK4o5MRETqiWuvhXnzaveYvXpVPp9627Zt6du3L8899xwjR45k4sSJfOMb38DMuPXWW2nbti3bt29n8ODBvPXWW/Ts2TPlcebMmcPEiROZN28e27Zt4/DDD+eII44AYNSoUVxyySUA/OAHP2DChAl861vfYsSIEZxyyimcccYZuxxr06ZNjB07lpdeeolu3bpx/vnn87vf/Y5rr70WgPbt2zN37lzuvfdebr/9dv74xz9WeH3jxo2jd+/eTJ48mRkzZnD++eczb948br/9du655x769evHunXrKCoq4v7772fIkCF8//vfZ/v27WzYsKE6X3VKqjmLmxmMHw933RWaOocPh7Vr445KRESkUuVNmxCaNEePHg3Ak08+yeGHH07v3r2ZP3/+Lv3Dkr366qucfvrpNG3alJYtWzJixIgd29555x369+/PoYceymOPPcb8+fMrjef999+na9eudOvWDYALLriAWbNm7dg+atQoAI444ogdk6VX5LXXXmPMmDEADBo0iOXLl7NmzRr69evHddddx913382qVasoKCigT58+PPjgg4wfP563336bFi1aVHrsdKjmLFtcfTW0aQPf/GaYWWDaNGjfPu6oREQky1VWw5VJI0eO5Nvf/jZz585lw4YNHHHEEXz00UfcfvvtzJ49mzZt2jB27Fg2bdq0W8cfO3YskydP5rDDDuOhhx5i5syZNYq3cePGAOTn5+/2ROw33ngjw4cPZ9q0afTr14/p06czYMAAZs2axdSpUxk7dizXXXcd559/fo1iVc1ZNhkzJtSevf12mE1gyZK4IxIREUmpefPmHH/88Vx44YU7as3WrFlDs2bNaNWqFZ9//jnPPfdcpccYMGAAkydPZuPGjaxdu5a//vWvO7atXbuWDh06sHXrVh577LEd61u0aMHaFC1MBx54IIsWLWLhwoUAPProoxx33HG7dW39+/ffcc6ZM2fSvn17WrZsyQcffMChhx7K9773Pfr06cN7773H4sWL2XPPPbnkkku4+OKLmTt37m6dM5GSs2xz6qlhBoFPPgnzcf7nP3FHJCIiktLo0aP597//vSM5O+yww+jduzfdu3fnnHPOoV+/fpXuf/jhh3PWWWdx2GGHMWzYMPr06bNj209+8hOOPPJI+vXrR/fu3XesP/vss7ntttvo3bs3H3zwwY71RUVFPPjgg5x55pkceuih5OXlcfnll+/WdY0fP545c+bQs2dPbrzxRh5++GEgDBdyyCGH0LNnTwoLCxk2bBgzZ87ccd2TJk3immuu2a1zJjLPkTG2SkpKvHz8lJwwdy4MjeZ6nz4deveONx4REcka7777LgcddFDcYUiaUt0vM5vj7iWpyqvmLFsdfniYj7NJExg4EBI6NYqIiEjuUnKWzbp1Cwlax44wZAg8+2zcEYmIiEiGKTnLdnvvHSZMP+QQOO00SOgUKSIiIrlHyVl90L49zJgBxx0H550H/+//xR2RiIjELFf6jOe63blPSs7qixYtYOrUUHt29dVh4Fr9hyki0iAVFRWxfPlyJWhZzt1Zvnw5RdWc+UeD0NYnRUXw1FNwySVhuqcVK8Log3nKsUVEGpLOnTuzZMkSli1bFncoUoWioiI6d+5crX2UnNU3BQUwYQK0bQt33BHm43zggTCRuoiINAiFhYV07do17jAkQ5Sc1Ud5eXD77aEv2s03w6pV8OSTYdgNERERqdfUHlZfmcFNN8Hvfhf6og0ZAqtXxx2ViIiI1FBGkzMzG2pm75vZQjO7McX2xmY2Kdr+DzPrEq0vNLOHzextM3vXzG7KZJz12uWXwxNPwJtvhsFqP/887ohERESkBjKWnJlZPnAPMAzoAYw2sx5JxS4CVrr7/sBvgF9G688EGrv7ocARwGXliZukcNZZ8Ne/wvvvQ//+sHhx3BGJiIjIbspkzVlfYKG7f+juW4CJwMikMiOBh6Plp4HBZmaAA83MrABoAmwB1mQw1vpv6FB48UVYtixMmL5gQdwRiYiIyG7IZHLWCfg44fOSaF3KMu6+DVgNtCMkauuBpcD/Abe7+4rkE5jZpWZWamalepwYOOYYeOUV2L4dBgyAf/4z7ohERESkmrL1gYC+wHagI9AV+I6Z7ZtcyN3vd/cSdy8pLi6u6xizU8+eYT7Oli1h0CB46aW4IxIREZFqyGRy9gmwd8LnztG6lGWiJsxWwHLgHOB5d9/q7l8ArwMlGYw1t+y3X0jQunaFk0+GZ56JOyIRERFJUyaTs9nAAWbW1cwaAWcDU5LKTAEuiJbPAGZ4mIvi/4BBAGbWDDgKeC+Dseaejh1DE+cRR8AZZ4SBakVERCTrZSw5i/qQXQVMB94FnnT3+WZ2i5mNiIpNANqZ2ULgOqB8uI17gOZmNp+Q5D3o7m9lKtac1bYtvPACnHACXHQR/PrXcUckIiIiVbBcmTS1pKTES0tL4w4jO23ZAmPGhFkEbroJbr01DGIrIiIisTCzOe6essuWpm9qCBo1gscfh9at4ec/DxOm33MP5OfHHZmIiIgkUXLWUOTnw333Qbt2IUFbuRIefTQkbiIiIpI1lJw1JGbws5+Fvmg33BAmTP/LX6BZs7gjExERkUi2jnMmmXT99TBhQphR4MQTQzOniIiIZAUlZw3VhRfCU0/BnDlw3HGwdGncEYmIiAhKzhq2UaNg2jT46KMwH+cHH8QdkYiISIOn5KyhGzwYZsyANWvg2GPh7bfjjkhERKRBU3Im0LcvzJoVnugcMADeeCPuiERERBosJWcS9OgBr78OxcVhRoHnn487IhERkQZJyZnstM8+8OqrcOCBMGIETJoUd0QiIiINjpIz2dWee8LMmXDUUTB6NPz+93FHJCIi0qAoOZOvatUqNGuefDJcfnkYuDZH5mAVERHJdkrOJLWmTeGZZ+Dcc+H73w8D1ypBExERyThN3yQVKyyERx4J0z3dcUeYj/P++6FA/2xEREQyJa1fWTNrBmx09zIz6wZ0B55z960ZjU7il5cHd90VErQf/zgkaE88AUVFcUcmIiKSk9Jt1pwFFJlZJ+BvwBjgoUwFJVnGDMaPD0na5MkwfDisXRt3VCIiIjkp3eTM3H0DMAq4193PBA6uciezoWb2vpktNLMbU2xvbGaTou3/MLMuCdt6mtmbZjbfzN42M1XVxO3qq+HRR+GVV2DQIPjyy7gjEhERyTlpJ2dmdjRwLjA1WpdfxQ75wD3AMKAHMNrMeiQVuwhY6e77A78BfhntWwD8Cbjc3Q8GBgJqQs0G550XHhR45x3o3x8+/jjuiERERHJKusnZtcBNwDPuPt/M9gVermKfvsBCd//Q3bcAE4GRSWVGAg9Hy08Dg83MgJOAt9z93wDuvtzdt6cZq2TaqafC9Onw6adhPs7//CfuiERERHJGWsmZu7/i7iPc/Zdmlgd86e5XV7FbJyCxWmVJtC5lGXffBqwG2gHdADez6WY218y+m+oEZnapmZWaWemyZcvSuRSpLQMGwMsvw8aNIUGbOzfuiERERHJCWsmZmT1uZi2jpzbfARaY2Q0ZjKsAOJbQjHoscLqZDU4u5O73u3uJu5cUFxdnMBxJ6fDD4bXXoEkTGDgw9EUTERGRGkm3WbOHu68BTgOeA7oSntiszCfA3gmfO0frUpaJ+pm1ApYTatlmufuX0YMI04DD04xV6lK3bmHC9E6dYOhQ+Otf445IRESkXks3OSs0s0JCcjYlGt+squHiZwMHmFlXM2sEnA1MSSozBbggWj4DmOHuDkwHDjWzplHSdhywIM1Ypa517hwmTD/kEDj9dPjTn+KOSEREpN5KNzn7PbAIaAbMMrN9gDWV7RD1IbuKkGi9CzwZPUxwi5mNiIpNANqZ2ULgOuDGaN+VwB2EBG8eMNfdpyafQ7JI+/YwYwYcdxyMGQN33x13RCIiIvWS+W7Ol2hmBVEClhVKSkq8tLQ07jBk0yYYPToMVjtuXHiZxR2ViIhIVjGzOe5ekmpbug8EtDKzO8qfjDSzXxNq0UR2VVQETz0F3/xmmO7p6quhrCzuqEREROqNdGewfoDwlOY3os9jgAcJMwaI7KqgACZMCPNx/vrXYT7OBx8ME6mLiIhIpdJNzvZz968nfP6xmc3LRECSI8zgttugXTu4+WZYtSrUqDVpEndkIiIiWS3dBwI2mtmx5R/MrB+wMTMhSc4wg5tugt/9DqZNgyFDQpImIiIiFUq35uxy4BEzaxV9XsnOITBEKnf55dCmTXiK8/jj4fnnYc89445KREQkK6U7fdO/3f0woCfQ0917A4MyGpnklrPOCgPU/uc/YbqnRYvijkhERCQrpdusCYC7r4lmCoAwLplI+oYMgRdegC+/DAnaAo0rLCIikqxayVkSDV4l1XfMMWEOzu3boX9/+Oc/445IREQkq9QkOdu90WtFevYME6a3bg2DBsGLL8YdkYiISNaoNDkzs7VmtibFay3QsY5ilFy0334hQevaFYYPh7/8Je6IREREskKlyZm7t3D3lileLdw93Sc9RVLr0AFmzYIjjoAzz4QHHog7IhERkdjVpFlTpObatAkPCZxwAlx0Edx+e9wRiYiIxErJmcSvWbMwzMY3vgE33BAGrnV1aRQRkYZJTZOSHRo1gscfDzVpv/gFrFgB994L+flxRyYiIlKnlJxJ9sjPD1M9tWsHP/tZmDD9T38KiZuIiEgDoeRMsosZ3HortG0L118Pq1eHJzmbNYs7MhERkTqR0T5nZjbUzN43s4VmdmOK7Y3NbFK0/R9m1iVp+9fMbJ2ZXZ/JOCULfec7MGFCGAPthBNCM6eIiEgDkLHkzMzygXuAYUAPYLSZ9UgqdhGw0t33B34D/DJp+x3Ac5mKUbLchRfC00/D3Llw3HHw6adxRyQiIpJxmaw56wssdPcP3X0LMBEYmVRmJPBwtPw0MNjMDMDMTgM+AuZnMEbJdqefDtOmhYnSjz0WPvgg7ohEREQyKpPJWSfg44TPS6J1Kcu4+zZgNdDOzJoD3wN+XNkJzOxSMys1s9Jly5bVWuCSZQYPhhkzYM2akKC99VbcEYmIiGRMto5zNh74jbuvq6yQu9/v7iXuXlJcXFw3kUk8+vQJswnk54cmztdfjzsiERGRjMhkcvYJsHfC587RupRlzKwAaAUsB44EfmVmi4BrgZvN7KoMxir1QY8eISkrLoYTT4Tnn487IhERkVqXyeRsNnCAmXU1s0bA2cCUpDJTgAui5TOAGR70d/cu7t4FuBP4mbv/NoOxSn2xzz7w6qtw4IFw6qkwcWLcEYmIiNSqjCVnUR+yq4DpwLvAk+4+38xuMbMRUbEJhD5mC4HrgK8MtyHyFXvuCTNnwtFHwznnwH33xR2RiIhIrTHPkTkMS0pKvLS0NO4wpC5t3Bjm43z2WfjpT+Hmm8MgtiIiIlnOzOa4e0mqbdn6QIBI1Zo0CbMHnHsu/OAHYeDasrK4oxIREakRTd8k9VthITzySJju6Te/CfNx/uEPUKB/2iIiUj/pF0zqv7w8uOuuMGH6+PGwahU88QQUFcUdmYiISLWpWVNygxmMGwd33w2TJ8PJJ4dBa0VEROoZJWeSW771LXj00TBg7aBBoJkjRESknlFyJrnnvJ9OxA8AACAASURBVPPgmWdg/nwYMAA+/rjqfURERLKEkjPJTaeeCtOnw6efQr9+8P77cUckIiKSFiVnkrsGDAiD1W7aBP37w9y5cUckIiJSJSVnktt694bXXgtjog0cCK+8EndEIiIildJQGtXwrW+F95Yt03s1bhxvvBLp1i1MmH7iiTBkCDz5JIwYUfV+IiIiMVByVg0vvQSffw6rV8P27VWXb9Qo/UQu1atFi/DerJlmJaqxzp3DhOknnwyjRsGDD8KYMXFHJSIi8hVKzqphwYLw7h66Ma1ZU/3Xp5/Ce+/t/LxpU9XnzcvbmajVNNnLz8/sd5TV2rcPGfZpp8H558OKFXDNNXFHJSIiWaSsLFTAFBbGF4OSs91gFrowNWkCe+5Zs2Nt2QJr11Y/yVu5EhYv3vl53br0ztesWc2TvJYtQ61gvdSiBUydCuecA9deGxK08eNVNSkiUg3uIYHZuhW2bQvv5a/EzxUtp1sujmOXlUGPHmE0prgoOYtZo0Zh1qF27Wp2nO3bQ4K2O7V5n3++6+d05g5v3Dj9ZtnKXk2bxpAXFRWFfmeXXgq33ALLl4eZBfL0fIyI1K6ysuxORGpSri6ZhZqsgoLwXv5K/JxquUmT6u9TWFjzipeaUnKWI/LzoVWr8KoJd9iwYfeSvCVLdv28eXPV58vLq52avObNq9lkW1AAEyaECdN//etQFfnQQ/HWY4vUsbKynU04ie+p1lWnTK7sn6pmqLpJTjr/s1ub8vOrn4gUFob/Z63uPrVZrqp9GlqXHCVnsguz0PTZrBl06FCzY23evHtNtsuXw0cf7fy8fn1652vevLpJndFyxG203HIQLf/fT2m57HxaPDmBRq2b1uzCs5j7zldZ2c73xOWarKut42TbsbM5gajJ/rkoL2/nKz8/9Xtl25LLJCYITZpkV8KSvFxQoB4auSKjyZmZDQXuAvKBP7r7L5K2NwYeAY4AlgNnufsiMzsR+AXQCNgC3ODuMzIZq9S+xo3Dq337mh1n27bdb7JdunTXz+7JRzfgovB6AWgDRUVOy5a2S7Nsixbhj3W2JAu7u+6r1y/VkfzDXp0f+epsKyzM3LGzYf9MHdtMyYnkhowlZ2aWD9wDnAgsAWab2RR3X5BQ7CJgpbvvb2ZnA78EzgK+BE5190/N7BBgOtApU7FKdisogNatw6sm3EMtXIXJ3My5rHn0f1nToitrhn6DNVub7thWPj2n2c4fh/LlVOsKCyvfnmpdVdu1T3xx6IdfROpSJmvO+gIL3f1DADObCIwEEpOzkcD4aPlp4LdmZu7+r4Qy84EmZtbY3dPoxSSSmllo+mzeHDp2TFFg7OEwelkYB+2NW+GFF6BLl7oOU0REGri8DB67E/BxwuclfLX2a0cZd98GrAaSn1v8OjA3VWJmZpeaWamZlS5btqzWApcGbMiQkJR9+WWYMH3Bgqr3ERERqUWZTM5qzMwOJjR1XpZqu7vf7+4l7l5SXFxct8FJ7jrmGJg1K3TU6t8f/vGPuCMSEZEGJJPJ2SfA3gmfO0frUpYxswKgFeHBAMysM/AMcL67f5DBOEW+6tBDw3ycrVvD4MHw4otxRyQiIg1EJvuczQYOMLOuhCTsbOCcpDJTgAuAN4EzgBnu7mbWGpgK3Ojur2cwRpGK7bsvvPYanHRSmJOzTx/YY48wOmFF761bq9e4iIjUSMaSM3ffZmZXEZ60zAcecPf5ZnYLUOruU4AJwKNmthBYQUjgAK4C9gd+ZGY/itad5O5fZCpekZQ6dAhNnDfdBP/9LyxcGGrUvvwy9bgUhYVVJ3Dly8XF4VFUERGRBOY5MvBRSUmJl5aWxh2GNBTbtoXRcj//HL74IvV74vKWLamP065d5Qlc4nvT3B0cV0SkoTGzOe5ekmqb/rddZHcUFOxMoqrivnMS08oSuLlzd050mkqzZlUncOXvbdqoeVVEpJ5SciaSaWY7Jz7t1q3q8ps2hYQtVQJX/v7hh/Dmm6F5NdU8PAUFVTevlr8XF2tOURGRLKLkTCTbFBXB174WXlXZvr3q5tUvvoB33w3LFc1G37ZteoncnnuGGjwREckYJWci9Vl+fkic9tij6rLuYSb6qvrJzZsX3levTn2cpk3Te+ChvHk1L6uHUxQRyTpKzkQaCrOdM7kfcEDV5Tdv3rV5NVUz66JFYZDeZcsqbl4tLk7vgYfiYmjUqNYvW0SkvlFyJiKpNW4Me+8dXlUpKwvNq5X1k/viC3j//bC8aVPq47Rpk94DD3vsESZJ1UMPIpKDlJyJSM3l5YWar+JiOPjgysu6w7p1VfeTe+utsLxqVerjNGmSXj+5PfYIQ5aoeVVE6gklZyJSt8ygRYvw2n//qstv2ZK6eTWxlu7//g9mzw7Nq9u3f/UY+fmpm1dTNbcWF4daQxGRmCg5E5Hs1qgRdO4cXlUpK4MVKyrvJ/fFF2G2h88/h40bUx+ndetd+8I1bRqeoq3q1aRJeuWKilSTJyIVUnImIrkjLw/atw+vHj2qLr9uXdX95N57L/SR27QpJHPlyzWdXaVRo9pL9qpbrrBQ/fVEspiSMxFpuJo3D699963efu6wdevORC3VKzGRq+qVquzq1Tsfnkh+VTReXbrMap7s1SRxzM+vWfwiOU7JmYhIdZmFmq9GjcLQJHWtrCwkaLub+KVTds2aistt21az+AsK6q6WMPnVuLFqDSXrKTkTEalv8vJC8tGkSTzn37Yt/YRvd5LEdevC1GQVba9pk3Ljxukle40bh0SysDC8J7+yYb0SzZyk5ExERKqnoGBnk3BdS25SrmnzcarXypW71hImv7Zu3fVzqgGY60peXvYljOmsr8mx8vJyPilVciYiIvVH3E3KqZSVpU7iUiVy2bZ+/frdO0aqIWvqUqYTxk6dYPz4+C4vtjOLiIjkgry8nQljQ+EeErRsTDirKpuqRjS5/H775W5yZmZDgbuAfOCP7v6LpO2NgUeAI4DlwFnuvijadhNwEbAduNrdp2cyVhEREUmT2c7aJql1GRsF0czygXuAYUAPYLSZJQ88dBGw0t33B34D/DLatwdwNnAwMBS4NzqeiIiISE7L5BDVfYGF7v6hu28BJgIjk8qMBB6Olp8GBpuZResnuvtmd/8IWBgdT0RERCSnZTI56wR8nPB5SbQuZRl33wasBtqluS9mdqmZlZpZ6bJly2oxdBEREZF41OvJ3dz9fncvcfeS4uLiuMMRERERqbFMJmefAHsnfO4crUtZxswKgFaEBwPS2VdEREQk55jXdKTlig4ckq3/AIMJidVs4Bx3n59Q5krgUHe/3MzOBka5+zfM7GDgcUI/s47AS8AB7l7hwCpmtgxYnJGL2VV74Ms6OE82asjXDg37+nXtDVdDvv6GfO3QsK+/Lq59H3dP2eyXsWdg3X2bmV0FTCcMpfGAu883s1uAUnefAkwAHjWzhcAKwhOaROWeBBYA24ArK0vMon3qpF3TzErdvaQuzpVtGvK1Q8O+fl17w7x2aNjX35CvHRr29cd97RkdoMTdpwHTktb9KGF5E3BmBfveCtyayfhEREREsk29fiBAREREJNcoOau+++MOIEYN+dqhYV+/rr3hasjX35CvHRr29cd67Rl7IEBEREREqk81ZyIiIiJZRMlZCmb2gJl9YWbvVLDdzOxuM1toZm+Z2eF1HWOmpHHtA81stZnNi14/SlWuPjKzvc3sZTNbYGbzzeyaFGVy+d6nc/05ef/NrMjM/mlm/46u/ccpyjQ2s0nRvf+HmXWp+0gzI83rH2tmyxLu/cVxxJopZpZvZv8ys2dTbMvZew9VXnuu3/dFZvZ2dG2lKbbH8jdf08mn9hDwW+CRCrYPAw6IXkcCv4vec8FDVH7tAK+6+yl1E06d2gZ8x93nmlkLYI6ZveDuCxLK5PK9T+f6ITfv/2ZgkLuvM7NC4DUze87d/55Q5iJgpbvvH43L+EvgrDiCzYB0rh9gkrtfFUN8deEa4F2gZYptuXzvofJrh9y+7wDHu3tFY5rF8jdfNWcpuPsswrhrFRkJPOLB34HWZtahbqLLrDSuPWe5+1J3nxstryX8sUqe0zWX730615+Tovu5LvpYGL2SO+SOBB6Olp8GBpuZ1VGIGZXm9ecsM+sMDAf+WEGRnL33aVx7QxfL33wlZ7snrYnZc9jRUfPHc9FsDjknarboDfwjaVODuPeVXD/k6P2PmnbmAV8AL7h7hffe3bcBq4F2dRtl5qRx/QBfj5p2njazvVNsr6/uBL4LlFWwPZfvfVXXDrl73yH8T8jfzGyOmV2aYnssf/OVnEl1zSVMOXEY8P+AyTHHU+vMrDnwZ+Bad18Tdzx1rYrrz9n77+7b3b0XYS7fvmZ2SNwx1aU0rv+vQBd37wm8wM6apHrNzE4BvnD3OXHHUtfSvPacvO8JjnX3wwnNl1ea2YC4AwIlZ7urwU7M7u5ryps/ohkgCs2sfcxh1Zqov82fgcfc/S8piuT0va/q+nP9/gO4+yrgZWBo0qYd997C3MGtgOV1G13mVXT97r7c3TdHH/8IHFHXsWVIP2CEmS0CJgKDzOxPSWVy9d5Xee05fN8BcPdPovcvgGcIc3oniuVvvpKz3TMFOD96iuMoYLW7L407qLpgZnuV97Uws76Ef0O58EeK6LomAO+6+x0VFMvZe5/O9efq/TezYjNrHS03AU4E3ksqNgW4IFo+A5jhOTJQZDrXn9TPZgShT2K95+43uXtnd+9CmN95hrufl1QsJ+99Oteeq/cdwMyaRQ8/YWbNgJOA5JEKYvmbr6c1UzCzJ4CBQHszWwKMI3SQxd3vI8wXejKwENgAfDOeSGtfGtd+BnCFmW0DNgJn58IfqUg/YAzwdtT3BuBm4GtQ/++9mT0HTHT3ipolEq9/DbASuIJdrz+r7n/0f/wXu/uLNTxUB+BhM8snJJwHA980s1uAPsAbwG3Ao2a2kPDQzNlRDA4c4O4LdyP+c4EL3P2kGsZfU8nX/6S7Pxtdf6m7TwGuNrMRhKd6VwBjY4u2DiRd+wRS3Ptc1YDu+57AM9H/bxYAj7v782Z2OcT7N18zBIhkMTNbl/CxKWHIg+3R58vc/bG6jyp71GJylnzctBOudMtGD1l8BBRGncpFRFJSzZlIFnP35uXLlSUiZlagH3zJFvr3KFIz6nMmUg9ZGKl/iZl9z8w+Ax40szZm9qyF0bxXRsudE/aZadHo3hZG/X7NzG6Pyn5kZsN2s2xXM5tlZmvN7EUzuydFh+rysunE+BMzez063t8SHzgwszFmttjMlpvZ9yv5fo40s8+iZrrydaeb2VvRcl8ze9PMVpnZUjP7rZk1quBYD5nZTxM+3xDt86mZXZhUdriFkdbXmNnHZjY+YfOs6H2Vma0zs6PLv9uE/Y8xs9kWZmGYbWbHpPvdVPN7bmtmD0bXsNLMJidsG2lhtPQ1ZvaBmQ2N1i8ysxMSyo0vv89m1sXM3MwuMrP/A2ZE65+K7sPq6N/IwQn7NzGzX0f3c3X0b6yJmU01s28lXc9bZnZ6qmsVyUVKzkTqr72AtsA+wKWE/54fjD5/jdAn7LeV7H8k8D7QHvgVMMGswoE1Kyv7OPBPwrhP4wn91iqSToznEPp17AE0Aq4HMLMehNG5xwAdo/N1JoVojK71wKCk4z4eLW8Hvh1dz9HAYOB/KombKIahUTwnEkYMPyGpyHrgfKA1YWDPK8zstGhb+SP6rd29ubu/mXTstsBU4O7o2u4ApppZ4nhaKb+bFKr6nh8lNJMfHB3rN1EMfQmzg9wQXcMAYFFF30cKxwEHAUOiz88Rvqc9CMOwJDbD30548u8Ywr/j8rG2HgZ2dEo3s8MI40pNrUYcIvWakjOR+qsMGOfum919Y/TI+5/dfUM0wv+thB/Liix29z+4+3bCD2IHQgfZtMua2dcIHeZ/5O5b3P01wtNNKaUZ44Pu/h933wg8CfSK1p8BPOvus6JH+39I5QNnPgGMBrDwRNbJ0TrcfY67/93dt7n7IuD3KeJI5RtRfO+4+3pCMpp4fTPd/W13L3P3t6LzpXNcCMncf9390SiuJwhPTJ6aUKai72YXlX3PFp6+GwZc7u4r3X2ru78S7XoR8IC7vxBdwyfunvzUamXGu/v6KD7c/QF3Xxvdr/HAYWbWyszygAuBa6JzbHf3N6JyU4BuZnZAdMwxhOmDtlQjDpF6TcmZSP21zN03lX8ws6Zm9vuomWgNoRmtdWLTXpLPyhfcfUO02LyaZTsCKxLWwa6jae8izRg/S1jekBBTx8RjR8lRZcN4PA6MMrPGwChgrrsvjuLoFjX1fRbF8TNCLVpVdokBWJx0fUdamDx+mZmtBi5P87jlx16ctG4xu45GXtF3s4sqvue9CfdsZYpd9wY+SDPeVHZ8NxZmHPhF1DS6hp01cO2jV1Gqc0X/picB50VJ3GhCTZ9Ig6HkTKT+Sn7U+jvAgcCR7t6Snc1omZwDcCnQ1syaJqyrbHqXmsS4NPHY0TkrnELHw4Ttiwm1RIlNmhCaR98jPGXZkjBkSrVjIBpmJMHjhJqfvd29FXBfwnGrejT+U0IzZKKvsXsDXlb2PX9MuGetU+z3MbBfBcdcT2gKLbdXijKJ13gOYV7CEwiDtnZJiOFLYFMl53oYOJfQ3LwhuQlYJNcpORPJHS0IfYtWRf2XxmX6hFFNVCkw3swamdnR7NoMV5sxPg2cYmbHRp33b6Hqv2GPA9cQkpOnkuJYA6wzs+6E8dzS8SQw1sx6RMlhcvwtCLVSm6L+W+ckbFtGaIbdt4JjTyM0551jZgVmdhbQA3g2zdiS40j5PUcDaD4H3Bs9OFBoO6esmUAY322wmeWZWafo+wGYB5wdlS8hNDNXFcNmQu1mU0LtZHkMZcADwB1m1jGqZTs6quUkSsbKgF+jWjNpgJScieSOO4EmhFqJvwPP19F5zyV0ql8O/JTQJLW5grK7HaO7zweuJCRcSwmD5C6pYrfyPl8z3P3LhPXXExKntcAfopjTieG56BpmEAalnJFU5H+AW8xsLfAjQjJXvu8GQt+v1y08JXpU0rGXA6cQar2WEzrIn5IUd7qq+p7HAFsJtYdfANdGMfyT8MDBbwiTe7/Cztq8HxJqulYCP2bXmshUHiHUXH4CLIjiSHQ98DYwmzC46S/Z9TfpEeBQIOWTvyK5TIPQikitMrNJwHvunvGaO8ldZnY+cKm7Hxt3LCJ1TTVnIlIjZtbHzPaLmsGGEvoZTa5qP5GKRE3G/wPcH3csInFQciYiNbUXMBNYRxij6wp3/1esEUm9ZWZDCP3zPqfqplORnKRmTREREZEsopozERERkSyi5ExEREQkixRk6sBm9gDhsfAv3P2QFNsNuIswpcoGYKy7z422XQD8ICr6U3d/uKrztW/f3rt06VJL0YuIiIhkzpw5c7509+JU2zKWnAEPESbafaSC7cMIE+IeQJhU+XfAkQkDJpYQRpueY2ZTKphqZIcuXbpQWlpaS6GLiIiIZI6ZJU/XtkPGmjXdfRZhYMGKjAQe8eDvhHnfOgBDgBfcvXzutxeAoZmKU0RERCSbxNnnrBO7TiC8JFpX0fqvMLNLzazUzEqXLVuWsUBFRERE6kq9fiDA3e939xJ3LykuTtlsKyIiIlKvZLLPWVU+AfZO+Nw5WvcJMDBp/cw6i0pERER2jzuUlYXX9u27vqe7rrrla3vd9u3Qti1ccklsX2OcydkU4Cozm0h4IGC1uy81s+nAz8ysTVTuJOCmuIIUEZEc4Q5bt8KWLem9tm7NroShPpwrVwa2P+ig3EzOzOwJQg1YezNbQngCsxDA3e8DphGG0VhIGErjm9G2FWb2E2B2dKhb3L2yBwtERCQu7uGHOd2EJ87X1q1xf1tflZcXXvn5u76nuy7d8gUFNT9XTc6fqXWZPG6MMpacufvoKrY7cGUF2x4AHshEXCIi9UJZWfVqeTLx2rw5vXKZqi1p3BgaNar6VVQELVumV7Y6r4KC8COdqYQhr153+5YMirNZU0Qk+2zaBCtXwooV4bV6dfpJSm2+tm3LzPUVFKSfnLRqVfsJT3USI7PMfAciWU7JmYjknrKykFStWLEz0UpMuFItl79v3Fj985mlX8vTtCm0bh1PwlNYqNoakXpAyZmIZK+NGytPripKuFatqryprWlTaNMmPJHVti3sv394T1xXvty6dWg2qyzpibl/iojkFiVnIpJZ27eHWqzqJFfly5s2VXzcvLydCVSbNtCuHRxwwK7rkhOt8uXGjevu+kVEqknJmYikZ+PG6jUPli9XVYvVrNmuCVW3bhUnV4nrWrRQE52I5CQlZyINSXktVnWSq/LlzZsrPm5+/q4JVHHxziSrouSqTRvVYomIpKDkTKS+cf9qLVa6tVmrVlV+7ObNd02kDjww/VosPVknIlIrlJyJVGHdOvj88zC6QfkA2InvFS1XuW7bdsrWrsfXrqNsTXj52nVh3br1lK1dT9m6DWG5/H39RsrWb8S3baOMPByjjLxdlt3yKWvahrKme+NNm1HWpBneqille0XLRU0oa9I0vDduSllRU7xxEWWNm+B5+alj3gZln4EvrcH11tN17iHvzMv76rvW5fY6M/0/R0PVqBF07Rrf+ZWcSYO1fj18+iksXRreK1peuzZTEeQDLaNXLXJgffSqhsp+rLLhBzNx3M66jiVbEsVsXCeSiw46CBYsiO/8Ss4k56xfX3XCVVHSVVQEHTtChw7QsycMHQod2m9lr7lTKfryE2z9OvLWr8XWrSVv/Vry1q3Btm0hjzIMj+qwdi5bXh55LZuT16IZ1rIFeS2bYy2ak9cqWm7VkrxWLbBWLchr1ZK8Nq1CudYtsUaFdVpDILI7kpO3bEsea7pOGqaWtfz/zNWl5EzqjcSkK1XCVf6+Zs1X9y0qCglXx44h6RoyJCyXJ2Ll761bJyUq69bByJEwYwbss0/oX9Upsc9VJeNjtWkT+nAp85EcZqZh3kRqm5Izid2GDamTrOTlVElX48Y7k6xDDoGTTvpqwtWxY4qkKx2rV8Pw4fDmm/Doo3DeebVyvSIiIpVRciYZs2FD1QnX0qUhB0pWnnR16BCSrhNP/GpN124nXelYsSJUr82bB5MmwRlnZOAkIiIiX6XkTKqtPOmqql9XRUlXeXJ18ME7k67EhKtDh9AiGFtr4BdfhMDefx+eeQZOOSWmQEREpCFSciY7bNxYdcK1dGnqobIaNdqZWPXoASec8NWEq2PHmJOudHzySQh+8WJ49tmwLCIiUoeUnDUAiUlXZc2MFSVd5YlVjx4weHDqmq62bbM86UrH4sUwaFCoOZs+Hfr3jzsiERFpgJSc1WPlSVdV/bpSJV2FhTuTq+7dQ06SqqYrJ5KudPz3vyHzXLsWXnoJ+vaNOyIREWmglJxloU2b0utIv3LlV/ctLNyZWHXvDscfn7ojfYNJutKxYEFIzLZtg5dfhl694o5IREQaMCVndag86apsjK5PP6066TrwwJ1JV3JtV7t2Srqq5V//CuNvFBTAK6+EtlsREZEYKTmrBZs3V51wLV0aRmdIVp50degA3brBwIEVNy/m5dX5peW2f/wjTAHQokVoyjzggLgjEhERUXKWLne4556dyVZi8lVR0rXXXiGx6tYNjjuu4pouJV0xmDUrDDC7xx47R/8XERHJAkrO0mQG48aFUerLE6z994cBA1LXdCnpymIvvBCmZNpnH3jxRejUKe6IREREdshocmZmQ4G7gHzgj+7+i6Tt+wAPAMXACuA8d18SbfslMDwq+hN3n5TJWNOxcCG0aqWkq17761/DaP/du4ckbY894o5IRERkFxlLM8wsH7gHGAb0AEabWXJv69uBR9y9J3AL8PNo3+HA4UAv4EjgejOLeY74MICqErN67KmnYNSoMPP5yy8rMRMRkayUyVSjL7DQ3T909y3ARGBkUpkewIxo+eWE7T2AWe6+zd3XA28BQzMYq+S6Rx+Fs8+GI48MTZlt28YdkYiISEqZTM46AR8nfF4SrUv0b2BUtHw60MLM2kXrh5pZUzNrDxwP7J3BWCWX3X8/XHBBeBR2+vTQNi0iIpKl4m6kux44zsz+BRwHfAJsd/e/AdOAN4AngDeB7ck7m9mlZlZqZqXLli2rw7Cl3rjrLrjsMhg2LMyV2axZ3BGJiIhUKpPJ2SfsWtvVOVq3g7t/6u6j3L038P1o3aro/VZ37+XuJwIG/Cf5BO5+v7uXuHtJcXFxpq5D6quf/xyuvRa+/nV45hlo0iTuiERERKqUyeRsNnCAmXU1s0bA2cCUxAJm1t7MymO4ifDkJmaWHzVvYmY9gZ7A3zIYq+QSd/jhD+Hmm+Gcc2DixDCDu4iISD2QsaE03H2bmV0FTCcMpfGAu883s1uAUnefAgwEfm5mDswCrox2LwRetTAP0RrCEBvbMhWr5BB3uP56uOMOuPhiuO8+yM+POyoREZG0mbvHHUOtKCkp8dLS0rjDkDiVlcFVV8Hvfgff+hbceafGPhERkaxkZnPcvSTVNv1ySW7Yvh0uuigkZt/9bngQQImZiIjUQ5q+Seq/rVthzBiYNAl+/OPQ3yw0iYuIiNQ7Ss6kftu8Gc46C/73f+FXv4Ibbog7IhERkRpRcib114YNYTqm6dPht7+FK6+seh8REZEsp+RM6qe1a2HECHjlFZgwAS68MO6IREREaoWSM6l/Vq0KI/7Png2PPQajR8cdkYiISK1Rcib1y5dfwkknwTvvwFNPwemnxx2RiIhIrVJyJvXHZ5/BiSfCwoXhAYBhw+KOSEREpNYpOZP6YckSGDwYPvkEpk6FQYPijkhERCQjlJxJ9vvoo5CMrVgRnszs1y/uiERERDJGyZlkt//8JyRmGzbASy9BScqZLkRERHKGkjPJXu+8AyecECYznzkTevaMOyIREZGM0+SDkp3mzoXjjoP8/DCWmRIzERFpIKpM6plzbAAAHsZJREFUzszsVDNTEid15803Q1NmixYwaxZ07x53RCIiInUmnaTrLOC/ZvYrM9OvpGTWzJlhuIzi4pCY7bdf3BGJiIjUqSqTM3c/D+gNfAA8ZGZvmtmlZtYi49FJwzJ9ehi7bJ99QmL2ta/FHZGIiEidS6u50t3XAE8DE4EOwOnAXDP7VgZjk4bkf/83zJXZvXuoPevQIe6IREREYpFOn7MRZvYMMBMoBPq6+zDgMOA7mQ1PGoRJk/5/e3ceXVV1/n/8/RCQyKAWEIrGCn4LojZkIKIFUcDhi1MokxJpFbAKOFV+tYpaZ/m6rHyrpQ79gRaQanGqES2oEEFsaZEwqQhUsPlVcEIQhGIgIc/vj3NIryGBC+bm3Nx8Xmtlceb77GxW8mTvffaGQYMgJwfeeCPo0hQREWmg4plKYxDwoLsviD3o7jvM7PLEhCUNxtSpcPnlcNpp8MorwUsAIiIiDVg83Zp3Am/v2TGzQ82sA4C7FyUkKmkYHn0URowIlmWaPVuJmYiICPElZ88BFTH7u8NjIgfv17+Gq6+GCy+EmTOhWbOoIxIREUkK8SRnjd19156dcPuQxIUkKc0d7r0Xfv5zGDIEXngB0tOjjkpERCRpxJOcbTSz/D07ZtYf+CJxIUnKcodbb4XbboNLL4Wnn4YmTaKOSkREJKnEk5yNBm4xs3+Z2UfATcCoeB5uZv3MbI2ZrTWzcdWcP9bMiszsHTObb2YZMed+ZWYrzWyVmU00M4u3UJKE3GHsWLjvPhg1CqZMgcZa2lVERKSq/f52dPd1wKlm1iLc3x7Pg80sDXgEOBtYDyw2s5nu/n7MZROAJ919mpn1Be4DfmJmPYCewJ4FFf8CnEEwnYfUNxUVMGYMTJoE118fjDdTri0iIlKtuJouzOx84CQgfU8DlrvfvZ/bugNr3f3D8BkzgP5AbHJ2IvB/wu15QGG47UA6wdg2I5hf7bN4YpUkU14OI0fC9Olwyy3BeDMlZiIiIjWKZxLa3xGsr3ktQaI0BDg2jmcfDXwUs78+PBZrBTAw3B4AtDSz1u7+N4Jk7ZPw6zV3XxXHZ0oy2bULCgqCxOzee2H8eCVmIiIi+xHPmLMe7n4p8KW73wX8EOhcS59/A3CGmS0j6LbcAOw2s+8DJwAZBAldXzPrVfXmcI3PYjMr3rhxYy2FJLWitDSY9f/554NuzFtvjToiERGReiGe5Kw0/HeHmR0FlBGsr7k/G4BjYvYzwmOV3P1jdx/o7jnAreGxLQStaH939+3hGLfZBEkhVe6f5O557p53pJb8SR7//ncwf9krr8BjjwUvAoiIiEhc4knOXjazI4AHgKVACfB0HPctBjqZWUczOwQYCsyMvcDM2pjZnhhuBn4fbv+LoEWtsZk1IWhVU7dmffDVV3DuucEamVOnwujRUUckIiJSr+zzhYAwcSoKW7NeMLNXgHR337q/B7t7uZldA7wGpAG/d/eVZnY3UOzuM4HewH1m5sAC4Orw9ueBvsC7BC8HvOruLx9UCaXufPkl9OsHS5fCH/8IF10UdUQiIiL1jrn7vi8wWxZ2Oya1vLw8Ly4ujjqMhmvjRjjnHHj/fXj2WejfP+qIREREkpaZLXH3vOrOxdOtWWRmgzQJrNTok0/gjDNg9epgnUwlZiIiIgctnnnORhHMRVZuZqUE02m4ux+W0MikfvjXv+DMM4ME7dVXgyRNREREDlo8KwS0rItApB5atw769oWtW2HuXDj11KgjEhERqff2m5yZ2enVHXf3BbUfjtQbq1YFLWa7dgVvZubmRh2RiIhISoinW/MXMdvpBMsyLSF4m1IaohUr4OyzoVEjmD8ffvCDqCMSERFJGfF0a14Yu29mxwAPJSwiSW6LF8N//zc0bw5FRdC5thaLEBEREYjvbc2q1hMsrSQNzV/+EnRlHnEELFigxExERCQB4hlz9luCiWAhSOayCVYKkIakqAjy8yEjI9jOyIg6IhERkZQUz5iz2Jldy4E/uvtfExSPJKNZs2DgQOjUKXgrs127qCMSERFJWfEkZ88Dpe6+G8DM0sysmbvvSGxokhT+9CcYOhQyM+H116F166gjEhERSWlxrRAAHBqzfygwNzHhSFJ5+ulgfcy8vKArU4mZiIhIwsWTnKW7+/Y9O+F2s8SFJEnhiSfgxz+GXr2CFrMjjog6IhERkQYhnuTs32ZWOcOomXUDvk5cSBK53/4WfvrTYMqMWbOgRYuoIxIREWkw4hlzdj3wnJl9TLCu5neBixMalUTnV7+Cm26CH/0IZsyApk2jjkhERKRBiWcS2sVm1gU4Pjy0xt3LEhuW1Dl3uOuu4GvoUHjySWjSJOqoREREGpz9dmua2dVAc3d/z93fA1qY2VWJD03qjHvQWnbXXTBiBPzhD0rMREREIhLPmLMr3H3Lnh13/xK4InEhSZ2qqIBrr4UHHoCrroLHH4e0tKijEhERabDiSc7SzMz27JhZGnBI4kKSOrN7N1x5JTzyCPz85/Dww8Fi5iIiIhKZeF4IeBV4xsz+b7g/CpiduJCkTpSVwfDhwVxmt98Od94J/8nBRUREJCLxJGc3AVcCo8P9dwje2JT6ateuYND/iy/CfffBuHFRRyQiIiKh/fZhuXsFsAgoAboDfYFViQ1LEubrr4NpMl58EX7zGyVmIiIiSabGljMz6wwUhF9fAM8AuHufuglNat327ZCfD/Pnw6RJcIXe6xAREUk2++rWXA28BVzg7msBzGxsnUQltW/rVjj/fPjb34I5zH7846gjEhERkWrsq1tzIPAJMM/MJpvZmQQrBMTNzPqZ2RozW2tme/WfmdmxZlZkZu+Y2XwzywiP9zGz5TFfpWb2owP5bImxeTOcdRYsWgTPPKPETEREJInVmJy5e6G7DwW6APMIlnFqa2aPmdk5+3twOOXGI8C5wIlAgZmdWOWyCcCT7t4VuBu4L/zsee6e7e7ZBGPcdgCvH3DpBD7/HHr3hnffhcJCGDw46ohERERkH+J5IeDf7v60u18IZADLCN7g3J/uwFp3/9DddwEzgP5VrjkReCPcnlfNeYDBwGx33xHHZ0qsDRvg9NNh7Vp45ZWgW1NERESS2gHNOOruX7r7JHc/M47LjwY+itlfHx6LtYKg+xRgANDSzFpXuWYo8MfqPsDMrjSzYjMr3rhxYxwhNSAlJUFi9vHH8NprQbemiIiIJL2op4O/ATjDzJYBZwAbgN17TppZeyATeK26m8NEMc/d84488si6iLd++OCDIDHbvBnmzoVevaKOSEREROIUzyS0B2sDcEzMfkZ4rJK7f0zYcmZmLYBBset4AhcBL7p7WQLjTC0rVwatZOXlMG8eZGdHHZGIiIgcgES2nC0GOplZRzM7hKB7cmbsBWbWxsz2xHAz8Psqzyighi5NqcayZcHgfzN4800lZiIiIvVQwpIzdy8HriHoklwFPOvuK83sbjPLDy/rDawxs38A7YDxe+43sw4ELW9vJirGlLJoEfTtC82awYIFcGLVF2NFRESkPjB3jzqGWpGXl+fFxcVRhxGNBQuCNzHbtYOiIjj22KgjEhERkX0wsyXunlfduahfCJBva84c6NcPMjKCJE2JmYiISL2m5Kw+e/lluOAC6NQpGGN21FFRRyQiIiLfkpKz+uq552DgQMjKCt7KbNs26ohERESkFig5q4+mT4ehQ+HUU4N5zFq1ijoiERERqSVKzuqbSZPgssugTx949VU47LCoIxIREZFapOSsPnnoIRg1Cs47L1grs3nzqCMSERGRWpbIFQKkNv3P/8Ctt8KgQfD003DIIVFHJCIiSaCsrIz169dTWloadShSjfT0dDIyMmjSpEnc9yg5S3bucNttMH48DBsGU6dCY1WbiIgE1q9fT8uWLenQoQNmFnU4EsPd2bRpE+vXr6djx45x36duzWTmDjfcECRmP/0pTJumxExERL6htLSU1q1bKzFLQmZG69atD7hVU7/pk1VFBVx9Nfzud3DttcF4s0bKpUVEZG9KzJLXwdSNftsno927YeTIIDG76Sb4zW+UmImISNLZtGkT2dnZZGdn893vfpejjz66cn/Xrl37vLe4uJjrrrtuv5/Ro0eP2gq33lDLWbIpK4Of/ASeeQbuuisYb6a/iEREJAm1bt2a5cuXA3DnnXfSokULbrjhhsrz5eXlNK5hOE5eXh55edUuLfkNCxcurJ1g6xE1xySTnTthyJAgMXvgAbj9diVmIiJSrwwfPpzRo0dzyimncOONN/L222/zwx/+kJycHHr06MGaNWsAmD9/PhdccAEQJHYjR46kd+/eHHfccUycOLHyeS1atKi8vnfv3gwePJguXbowbNgw3B2AWbNm0aVLF7p168Z1111X+dxYJSUl9OrVi9zcXHJzc7+R9N1///1kZmaSlZXFuHHjAFi7di1nnXUWWVlZ5Obmsm7dusR8w6qhlrNksWMHDBgAr78ODz8cjDcTERE5ENdfD2FLVq3Jzg7GPR+A9evXs3DhQtLS0vjqq6946623aNy4MXPnzuWWW27hhRde2Oue1atXM2/ePLZt28bxxx/PmDFj9pp+YtmyZaxcuZKjjjqKnj178te//pW8vDxGjRrFggUL6NixIwUFBdXG1LZtW+bMmUN6ejoffPABBQUFFBcXM3v2bF566SUWLVpEs2bN2Lx5MwDDhg1j3LhxDBgwgNLSUioqKg7oe/BtKDlLBtu2wYUXwoIF8MQTwXgzERGRemrIkCGkpaUBsHXrVi677DI++OADzIyysrJq7zn//PNp2rQpTZs2pW3btnz22WdkZGR845ru3btXHsvOzqakpIQWLVpw3HHHVU5VUVBQwKRJk/Z6fllZGddccw3Lly8nLS2Nf/zjHwDMnTuXESNG0KxZMwBatWrFtm3b2LBhAwMGDACCucrqkpKzqG3ZAueeC4sXw1NPQQ0Zv4iIyH4dYAtXojSPWcHmtttuo0+fPrz44ouUlJTQu3fvau9p2rRp5XZaWhrl5eUHdU1NHnzwQdq1a8eKFSuoqKio84TrQGjMWZS++AL69oUlS+C555SYiYhIytm6dStHH300AFOnTq315x9//PF8+OGHlJSUAPDMM8/UGEf79u1p1KgR06dPZ/fu3QCcffbZTJkyhR07dgCwefNmWrZsSUZGBoWFhQDs3Lmz8nxdUHIWlU8/hd69YdUqeOmlYLyZiIhIirnxxhu5+eabycnJOaCWrngdeuihPProo/Tr149u3brRsmVLDj/88L2uu+qqq5g2bRpZWVmsXr26snWvX79+5Ofnk5eXR3Z2NhMmTABg+vTpTJw4ka5du9KjRw8+/fTTWo+9JrbnTYf6Li8vz4uLi6MOIz4ffQRnngkffwwzZwatZyIiIgdh1apVnHDCCVGHEant27fTokUL3J2rr76aTp06MXbs2KjDqlRdHZnZEnevdi4RtZzVtQ8/hNNPh88+g9deU2ImIiLyLU2ePJns7GxOOukktm7dyqhRo6IO6VvRCwF1ac2aoMXs66+hqAjimHxPRERE9m3s2LFJ1VL2bSk5qyvvvgtnnRVsz5sHXbtGG4+IiIgkpYR2a5pZPzNbY2ZrzWxcNeePNbMiM3vHzOabWUbMue+Z2etmtsrM3jezDomMNaGWLAkG/zduDG++qcRMREREapSw5MzM0oBHgHOBE4ECMzuxymUTgCfdvStwN3BfzLkngQfc/QSgO/B5omJNqIULg3FlLVsGk8x26RJ1RCIiIpLEEtly1h1Y6+4fuvsuYAbQv8o1JwJvhNvz9pwPk7jG7j4HwN23u3vdTTBSW+bPh3POgbZt4a234L/+K+qIREREJMklMjk7GvgoZn99eCzWCmBguD0AaGlmrYHOwBYz+5OZLTOzB8KWuPrj1VeDmf+PPTZoMTvmmKgjEhERqXV9+vThtdde+8axhx56iDFjxtR4T+/evdkz/dV5553Hli1b9rrmzjvvrJxzrCaFhYW8//77lfu33347c+fOPZDwk1LUU2ncAJxhZsuAM4ANwG6CFxV6hedPBo4Dhle92cyuNLNiMyveuHFjnQW9X4WFkJ8fdGG++Sa0bx91RCIiIglRUFDAjBkzvnFsxowZNS5AXtWsWbM44ogjDuqzqyZnd999N2ftefmuHktkcrYBiG0uygiPVXL3j919oLvnALeGx7YQtLItD7tEy4FCILfqB7j7JHfPc/e8I488MlHlODAzZsDgwZCbC2+8AW3aRB2RiIhIwgwePJg///nP7Nq1C4CSkhI+/vhjevXqxZgxY8jLy+Okk07ijjvuqPb+Dh068MUXXwAwfvx4OnfuzGmnncaaNWsqr5k8eTInn3wyWVlZDBo0iB07drBw4UJmzpzJL37xC7Kzs1m3bh3Dhw/n+eefB6CoqIicnBwyMzMZOXIkO3furPy8O+64g9zcXDIzM1m9evVeMZWUlNCrVy9yc3PJzc1l4cKFlefuv/9+MjMzycrKYty44F3HtWvXctZZZ5GVlUVubi7r1q37Vt/TRE6lsRjoZGYdCZKyocAlsReYWRtgs7tXADcDv4+59wgzO9LdNwJ9geSf/n/KFLj8cujVC155JXgJQEREpI5cfz0sX167z8zO3vd66q1ataJ79+7Mnj2b/v37M2PGDC666CLMjPHjx9OqVSt2797NmWeeyTvvvEPXGmYsWLJkCTNmzGD58uWUl5eTm5tLt27dABg4cCBXXHEFAL/85S954oknuPbaa8nPz+eCCy5g8ODB33hWaWkpw4cPp6ioiM6dO3PppZfy2GOPcf311wPQpk0bli5dyqOPPsqECRN4/PHHv3F/27ZtmTNnDunp6XzwwQcUFBRQXFzM7Nmzeemll1i0aBHNmjVj8+bNAAwbNoxx48YxYMAASktLqaioOKjv9R4JazkLW7yuAV4DVgHPuvtKM7vbzPLDy3oDa8zsH0A7YHx4726CLs0iM3sXMGByomKtFY8+CiNHBnOZzZ6txExERBqM2K7N2C7NZ599ltzcXHJycli5cuU3uiCreuuttxgwYADNmjXjsMMOIz8/v/Lce++9R69evcjMzOSpp55i5cqV+4xnzZo1dOzYkc6dOwNw2WWXsWDBgsrzAwcGw927detWuWB6rLKyMq644goyMzMZMmRIZdxz585lxIgRNGvWDAgS023btrFhwwYGhGtkp6enV54/WAmdhNbdZwGzqhy7PWb7eeD5Gu6dA9SPCcH+93/hhhvgwgvh2WchPT3qiEREpAHaVwtXIvXv35+xY8eydOlSduzYQbdu3fjnP//JhAkTWLx4Md/5zncYPnw4paWlB/X84cOHU1hYSFZWFlOnTmX+/PnfKt6mTZsCkJaWVu1i7A8++CDt2rVjxYoVVFRUkF7Hv9ejfiGgfnOHe+4JErMhQ+CFF5SYiYhIg9OiRQv69OnDyJEjK1vNvvrqK5o3b87hhx/OZ599xuzZs/f5jNNPP53CwkK+/vprtm3bxssvv1x5btu2bbRv356ysjKeeuqpyuMtW7Zk27Ztez3r+OOPp6SkhLVr1wIwffp0zjjjjLjLs3XrVtq3b0+jRo2YPn06u3fvBuDss89mypQp7NgRzO61efNmWrZsSUZGBoWFhQDs3Lmz8vzBUnJ2sNzhllvg9tvh0kvh6aehSZOooxIREYlEQUEBK1asqEzOsrKyyMnJoUuXLlxyySX07Nlzn/fn5uZy8cUXk5WVxbnnnsvJJ59cee6ee+7hlFNOoWfPnnSJmcx96NChPPDAA+Tk5HxjEH56ejpTpkxhyJAhZGZm0qhRI0aPHh13Wa666iqmTZtGVlYWq1evpnnz5gD069eP/Px88vLyyM7OrpzqY/r06UycOJGuXbvSo0cPPv3007g/qzrm7t/qAckiLy/P98yZknDuwajLiRNh1KhgvFkj5bkiIlL3Vq1axQknnBB1GLIP1dWRmS1x97zqrldGcaAqKoKEbOLEIEF77DElZiIiIlJrlFUciPJyuOwymDw56NL89a/BLOqoREREJIUk9G3NlLJrF1xySTDo/9574dZbo45IREREUpCSs3iVlcHnnwetZWPHRh2NiIhIJXfH1JOTlA5mbL+Ss3g1bx4sx9RY3zIREUke6enpbNq0idatWytBSzLuzqZNmw54njRlGgdCiZmIiCSZjIwM1q9fz8aNG6MORaqRnp5ORkbGAd2jbENERKQea9KkCR07dow6DKlFeltTREREJIkoORMRERFJIkrORERERJJIyizfZGYbgf9XBx/VBviiDj4nGTXkskPDLr/K3nA15PI35LJDwy5/XZT9WHc/sroTKZOc1RUzK65pLaxU15DLDg27/Cp7wyw7NOzyN+SyQ8Muf9RlV7emiIiISBJRciYiIiKSRJScHbhJUQcQoYZcdmjY5VfZG66GXP6GXHZo2OWPtOwacyYiIiKSRNRyJiIiIpJElJxVw8x+b2afm9l7NZw3M5toZmvN7B0zy63rGBMljrL3NrOtZrY8/Lq9rmNMFDM7xszmmdn7ZrbSzH5WzTWpXPfxlD8l69/M0s3sbTNbEZb9rmquaWpmz4R1v8jMOtR9pIkRZ/mHm9nGmLr/aRSxJoqZpZnZMjN7pZpzKVv3sN+yp3q9l5jZu2HZiqs5H8nPfK2tWb2pwMPAkzWcPxfoFH6dAjwW/psKprLvsgO85e4X1E04daoc+Lm7LzWzlsASM5vj7u/HXJPKdR9P+SE1638n0Nfdt5tZE+AvZjbb3f8ec83lwJfu/n0zGwrcD1wcRbAJEE/5AZ5x92siiK8u/AxYBRxWzblUrnvYd9khtesdoI+71zSnWSQ/89VyVg13XwBs3scl/YEnPfB34Agza1830SVWHGVPWe7+ibsvDbe3EfywOrrKZalc9/GUPyWF9bk93G0SflUdkNsfmBZuPw+caWZWRyEmVJzlT1lmlgGcDzxewyUpW/dxlL2hi+RnvpKzg3M08FHM/noayC+x0A/D7o/ZZnZS1MEkQthtkQMsqnKqQdT9PsoPKVr/YdfOcuBzYI6711j37l4ObAVa122UiRNH+QEGhV07z5vZMXUcYiI9BNwIVNRwPpXrfn9lh9Stdwj+CHndzJaY2ZXVnI/kZ76SMzlQSwmWnMgCfgsURhxPrTOzFsALwPXu/lXU8dS1/ZQ/Zevf3Xe7ezaQAXQ3sx9EHVNdiqP8LwMd3L0rMIf/tCTVa2Z2AfC5uy+JOpa6FmfZU7LeY5zm7rkE3ZdXm9npUQcESs4O1gYg9q+HjPBYynP3r/Z0f7j7LKCJmbWJOKxaE463eQF4yt3/VM0lKV33+yt/qtc/gLtvAeYB/aqcqqx7M2sMHA5sqtvoEq+m8rv7JnffGe4+DnSr69gSpCeQb2YlwAygr5n9oco1qVr3+y17Ctc7AO6+Ifz3c+BFoHuVSyL5ma/k7ODMBC4N3+I4Fdjq7p9EHVRdMLPv7hlrYWbdCf4PpcIPKcJyPQGscvdf13BZytZ9POVP1fo3syPN7Ihw+1DgbGB1lctmApeF24OBNzxFJoqMp/xVxtnkE4xJrPfc/WZ3z3D3DsBQgnr9cZXLUrLu4yl7qtY7gJk1D19+wsyaA+cAVWcqiORnvt7WrIaZ/RHoDbQxs/XAHQQDZHH33wGzgPOAtcAOYEQ0kda+OMo+GBhjZuXA18DQVPghFeoJ/AR4Nxx7A3AL8D1I/bonvvKnav23B6aZWRpBwvmsu79iZncDxe4+kyBxnW5mawlemhkaXbi1Lp7yX2dm+QRv9W4GhkcWbR1oQHW/lwZU7+2AF8O/NxsDT7v7q2Y2GqL9ma8VAkRERESSiLo1RURERJKIkjMRERGRJKLkTERERCSJKDkTERERSSJKzkRERESSiJIzEUlpZrbbzJbHfI2rxWd3MLOq8yKJiHwrmudMRFLd1+GyRCIi9YJazkSkQTKzEjP7lZm9a2Zvm9n3w+MdzOyNcKHnIjP7Xni8nZm9GC76vsLMeoSPSjOzyWa20sxeD2fYx8yuM7P3w+fMiKiYIlIPKTkTkVR3aJVuzYtjzm1190zgYeCh8NhvgWnhQs9PARPD4xOBN8NF33OBleHxTsAj7n4SsAUYFB4fB+SEzxmdqMKJSOrRCgEiktLMbLu7t6jmeAnQ190/DBd8/9TdW5vZF0B7dy8Lj3/i7m3MbCOQEbMINGbWAZjj7p3C/ZuAJu5+r5m9CmwHCoHCPQvGi4jsj1rORKQh8xq2D8TOmO3d/Gcs7/nAIwStbIvNTGN8RSQuSs5EpCG7OObfv4XbC/nPwtbDgLfC7SJgDICZpZnZ4TU91MwaAce4+zzgJuBwYK/WOxGR6ugvORFJdYea2fKY/Vfdfc90Gt8xs3cIWr8KwmPXAlPM7BfARmBEePxnwCQzu5yghWwM8EkNn5kG/CFM4AyY6O5baq1EIpLSNOZMRBqkcMxZnrt/EXUsIiKx1K0pIiIikkTUciYiIiKSRNRyJiIiIpJElJyJiIiIJBElZyIiIiJJRMmZiIiISBJRciYiIiKSRJSciYiIiCSR/w80/XCv3MZzhgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 720x432 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vuVgE_9RfXEh"
      },
      "source": [
        "## Export\n",
        "\n",
        "This will export the model to `classifier.zip`, which can be downloaded."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-yCH30AVfYLq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "62bb0082-ead1-4360-93de-76702b953c70"
      },
      "source": [
        "dataset_name = 'textbook_name_recognition'\n",
        "saved_model_path = './{}_bert'.format(dataset_name.replace('/', '_'))\n",
        "\n",
        "classifier_model.save(saved_model_path, include_optimizer=False)\n",
        "\n",
        "!zip -r /content/classifier.zip /content/textbook_name_recognition_bert\n",
        "!rm -rf /content/textbook_name_recognition_bert"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:absl:Found untraced functions such as restored_function_body, restored_function_body, restored_function_body, restored_function_body, restored_function_body while saving (showing 5 of 310). These functions will not be directly callable after loading.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: ./textbook_name_recognition_bert/assets\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: ./textbook_name_recognition_bert/assets\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "  adding: content/textbook_name_recognition_bert/ (stored 0%)\n",
            "  adding: content/textbook_name_recognition_bert/assets/ (stored 0%)\n",
            "  adding: content/textbook_name_recognition_bert/assets/vocab.txt (deflated 53%)\n",
            "  adding: content/textbook_name_recognition_bert/variables/ (stored 0%)\n",
            "  adding: content/textbook_name_recognition_bert/variables/variables.index (deflated 75%)\n",
            "  adding: content/textbook_name_recognition_bert/variables/variables.data-00000-of-00001 (deflated 7%)\n",
            "  adding: content/textbook_name_recognition_bert/saved_model.pb (deflated 92%)\n",
            "  adding: content/textbook_name_recognition_bert/keras_metadata.pb (deflated 84%)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R9BnSFqehFUB"
      },
      "source": [
        "## Sample usage"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LbezgcvNhH6o",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "60c8d0b7-0ccb-4e4e-d08d-37515c29676a"
      },
      "source": [
        "def print_examples(inputs, results):\n",
        "  for i in range(len(inputs)):\n",
        "    print('input: {}\\n\\tscore: {:.3f}%'.format(inputs[i], results[i][0] * 100))\n",
        "\n",
        "examples = [\n",
        "  'harry potter by jk. rowling',\n",
        "  'Intro to Data Structures, Cormen et. al, 2020',\n",
        "  'You will be required to purchase these textbooks',\n",
        "  'required readings:',\n",
        "  'harry potter, 339 pages',\n",
        "  'What is gender',\n",
        "  'Al-kitaab fii Ta\\'allum al-\\'Arabiyya, with DVDs, a Textbook for Intermediate Arabic, Part two, third Edition Georgetown University Press, Washington',\n",
        "  'intro to gender studies',\n",
        "  'intro to chemistry',\n",
        "  'Embodied psychologies'\n",
        "]\n",
        "\n",
        "results = tf.sigmoid(classifier_model(tf.constant(list(preprocessing.standardize(example) for example in examples))))\n",
        "\n",
        "print_examples(examples, results)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "input: harry potter by jk. rowling\n",
            "\tscore: 99.998%\n",
            "input: Intro to Data Structures, Cormen et. al, 2020\n",
            "\tscore: 98.616%\n",
            "input: You will be required to purchase these textbooks\n",
            "\tscore: 0.001%\n",
            "input: required readings:\n",
            "\tscore: 0.001%\n",
            "input: harry potter, 339 pages\n",
            "\tscore: 99.982%\n",
            "input: What is gender\n",
            "\tscore: 0.002%\n",
            "input: Al-kitaab fii Ta'allum al-'Arabiyya, with DVDs, a Textbook for Intermediate Arabic, Part two, third Edition Georgetown University Press, Washington\n",
            "\tscore: 99.986%\n",
            "input: intro to gender studies\n",
            "\tscore: 0.405%\n",
            "input: intro to chemistry\n",
            "\tscore: 0.036%\n",
            "input: Embodied psychologies\n",
            "\tscore: 0.005%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6b7fZCAdciWd"
      },
      "source": [
        "# Classifying Timetable Segments"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PpGAd3qMghW0"
      },
      "source": [
        "## Test on manually tagged data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z-MfZbVFgg3v",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f20ee189-244c-444a-c19b-3442b1685bd3"
      },
      "source": [
        "# True - BOOK, False - NOT BOOK, None - MERGED INTO NEXT (interpret as BOOK)\n",
        "manual_labels = [True, True, True, True, True, True, False, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, False, True, False, True, True, True, False, True, True, True, True, False, True, True, True, False, False, False, True, True, True, True, True, True, True, True, True, False, True, True, False, True, True, True, True, True, True, True, True, False, True, False, True, False, True, True, False, True, False, True, True, True, True, True, True, True, True, False, True, True, False, True, True, False, True, True, False, True, True, True, True, True, False, True, True, True, False, True, True, False, False, True, True, False, False, False, False, False, False, True, True, True, False, False, False, False, True, True, True, False, True, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, True, True, False, False, False, False, False, True, False, False, False, False, False, True, False, False, True, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, False, False, True, True, False, False, False, False, False, True, False, False, False, False, False, False, False, False, False, False, False, True, True, False, False, False, True, False, True, True, True, False, True, True, True, False, False, True, True, True, False, True, True, True, False, False, True, False, True, False, False, True, False, True, True, True, False, True, True, True, False, False, True, False, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, False, False, False, False, False, False, True, True, True, True, True, False, False, True, True, True, True, True, False, True, True, True, True, True, False, False, True, True, False, False, False, True, True, True, True, True, True, True, False, True, False, False, True, False, True, True, True, True, True, False, True, True, True, False, True, True, False, False, True, False, True, False, False, False, False, False, True, True, True, False, False, False, True, True, True, True, False, False, False, False, True, True, False, False, True, False, True, True, True, True, True, False, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, False, False, False, False, True, True, True, False, False, True, False, False, True, True, True, False, False, False, True, False, False, True, False, True, True, True, True, False, True, True, True, True, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, False, True, False, True, False, False, True, False, False, True, False, True, True, True, True, True, False, True, False, False, True, True, True, True, True, True, True, False, True, False, True, True, True, True, True, False, True, True, True, True, True, True, True, False, False, True, True, True, True, True, True, True, True, False, True, True, True, True, False, True, True, False, True, True, False, True, True, True, True, True, True, True, True, True, False, False, False, True, False, True, False, False, True, True, True, False, False, False, True, True, True, True, True, False, True, False, True, False, True, False, False, True, False, True, False, True, False, True, False, False, False, True, True, False, True, False, True, False, True, False, True, False, True, False, True, False, True, True, False, True, True, True, True, True, True, True, False, True, False, True, True, True, True, True, True, True, False, True, False, True, True, True, True, True, False, True, False, True, False, True, False, True, True, False, False, True, False, True, False, True, True, True, False, False, True, True, True, True, True, True, True, True, True, True, False, True, True, False, False, False, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, True, False, True, True, True, True, True, False, True, True, True, True, True, False, False, True, True, False, True, True, True, False, False, False, True, True, False, True, True, True, True, False, True, True, True, False, True, True, True, False, True, False, False, False, False, True, True, False, True, False, False, True, True, True, True, False, True, True, True, True, True, False, True, True, True, True, True, True]\n",
        "\n",
        "scores = tf.sigmoid(classifier_model(tf.constant(data[:len(manual_labels)])))\n",
        "computed_labels = list(score[0] > 0.5 for score in scores)\n",
        "\n",
        "tp, fp, tn, fn = 0, 0, 0, 0\n",
        "for i in range(len(manual_labels)):\n",
        "  err = False\n",
        "\n",
        "  if computed_labels[i]:\n",
        "    if manual_labels[i]: tp += 1\n",
        "    else:\n",
        "      fp += 1\n",
        "      err = True\n",
        "  else:\n",
        "    if manual_labels[i]:\n",
        "      fn += 1\n",
        "      err = True\n",
        "    else: tn += 1\n",
        "\n",
        "  if err: print('\"{}\"\\n\\tincorrectly classified: {} ({}%)'.format(data[i], 'BOOK' if computed_labels[i] else 'NOT BOOK', scores[i][0] * 100))\n",
        "\n",
        "print('\\nConfusion Matrix:\\n\\tTP: {:<3d}\\tFP: {:<3d}\\n\\tFN: {:<3d}\\tTN: {:<3d}'.format(tp, fp, fn, tn))\n",
        "print('\\nPrecision: {:.3f}\\nRecall: {:.3f}'.format(tp / (tp + fp), tp / (tp + fn)))"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\"Fadiman, Anne\"\n",
            "\tincorrectly classified: BOOK (99.35369110107422%)\n",
            "\"None. Readings available in Canvas.\"\n",
            "\tincorrectly classified: BOOK (99.24520874023438%)\n",
            "\"0 9 8 7 6 5 4 3 2 1 2004\"\n",
            "\tincorrectly classified: BOOK (99.97216796875%)\n",
            "\"Amazon New $154.66  Used $121.96 Rent $ 76.97 91520\"\n",
            "\tincorrectly classified: NOT BOOK (0.7209190726280212%)\n",
            "\"Reading and video materials will be provided via Canvas and 2hr Dana library Loan.\"\n",
            "\tincorrectly classified: NOT BOOK (0.007098568137735128%)\n",
            "\"The older 7th edition is adequate, and you may be able to find a used copy for sale.\"\n",
            "\tincorrectly classified: NOT BOOK (0.008042467758059502%)\n",
            "\"For the 8th edition, only two purchase options are available.\"\n",
            "\tincorrectly classified: NOT BOOK (0.013738580979406834%)\n",
            "\"32 pages\"\n",
            "\tincorrectly classified: BOOK (68.25786590576172%)\n",
            "\"32 pages\"\n",
            "\tincorrectly classified: BOOK (68.25786590576172%)\n",
            "\"Required Zubrick, James W. The Organic Chem Lab Survival Manual, A Students Guide to Techniques, 10th ed., John Wiley and Sons, 2016. ISBN 9781118875780\"\n",
            "\tincorrectly classified: BOOK (99.99523162841797%)\n",
            "\"One complete set of Integrated Chinese Third Edition, Paperback, Traditional, Level 1 Parts 1  2, i.e., 6 booksset.\"\n",
            "\tincorrectly classified: NOT BOOK (18.30304527282715%)\n",
            "\"No texts u2013 articles, etc., on Canvas\"\n",
            "\tincorrectly classified: BOOK (96.40076446533203%)\n",
            "\"Sheri Biesen, Blackout World War II and the Origins of Film Noir\"\n",
            "\tincorrectly classified: NOT BOOK (29.686927795410156%)\n",
            "\"Optional Krugman, Microeconomics any edition\"\n",
            "\tincorrectly classified: NOT BOOK (0.01655852049589157%)\n",
            "\"Optional Krugman, Microeconomics any edition\"\n",
            "\tincorrectly classified: NOT BOOK (0.01655852049589157%)\n",
            "\"Statistical Inference by Casella and Berger\"\n",
            "\tincorrectly classified: NOT BOOK (30.034080505371094%)\n",
            "\"Probability and Statistics by DeGroot and Schervish\"\n",
            "\tincorrectly classified: NOT BOOK (0.2860962450504303%)\n",
            "\"Statistical Inference by Casella and Berger\"\n",
            "\tincorrectly classified: NOT BOOK (30.034080505371094%)\n",
            "\"Probability and Statistics by DeGroot and Schervish\"\n",
            "\tincorrectly classified: NOT BOOK (0.2860962450504303%)\n",
            "\"Mostly Harmless Econometrics\"\n",
            "\tincorrectly classified: NOT BOOK (0.0025296728126704693%)\n",
            "\"Joshua D. Angrist and JornSteffen Pischke\"\n",
            "\tincorrectly classified: NOT BOOK (31.072038650512695%)\n",
            "\"Readings in the course reader are taken from 1 Introduction to Linear Optimization by Dimitris Bertsimas and John N. Tsitsiklis BT, 2 Network Flows Theory, Algorithms and Applications by Ravindra K. Ahuja, Thomas L. Magnanti, and James H. Orlin AMO, 3 Urban Operations Research by Richard C. Larson and Amedeo R. Odoni LO, and 4 Game Theory by Drew Fudenberg and Jean Tirole FT.\"\n",
            "\tincorrectly classified: NOT BOOK (0.1164606511592865%)\n",
            "\"Please be sure you buy the correct anthology Norton publishes several with confusingly similar titles. Doublecheck the ISBN.\"\n",
            "\tincorrectly classified: BOOK (99.95979309082031%)\n",
            "\"Please be sure you buy the correct anthology Norton publishes several with confusingly similar titles. Doublecheck the ISBN.\"\n",
            "\tincorrectly classified: BOOK (99.95979309082031%)\n",
            "\"Engineering Field Theory, u00D6sterberg and Hansen, 2014  will be distributed in Class.\"\n",
            "\tincorrectly classified: NOT BOOK (1.2445451021194458%)\n",
            "\"Sheri Biesen, Blackout World War II and the Origins of Film Noir\"\n",
            "\tincorrectly classified: NOT BOOK (29.686927795410156%)\n",
            "\" McCloud, Understanding Comics ASAP Lots of used copies are available\"\n",
            "\tincorrectly classified: NOT BOOK (0.19046175479888916%)\n",
            "\" Miller, Janson, and Varley, Batman The Dark Knight Returns\"\n",
            "\tincorrectly classified: NOT BOOK (2.4093589782714844%)\n",
            "\" Forsman, The End of the Fng World\"\n",
            "\tincorrectly classified: NOT BOOK (2.4672298431396484%)\n",
            "\"Connect ISBN  0077412745 SmartBook, the continually adaptive and fully mobile reading experience within Connectu2014now with offline access Purchase price $130\"\n",
            "\tincorrectly classified: BOOK (99.20050811767578%)\n",
            "\"Connect ISBN  0077412745 SmartBook, the continually adaptive and fully mobile reading experience within Connectu2014now with offline access Purchase price $130\"\n",
            "\tincorrectly classified: BOOK (99.20050811767578%)\n",
            "\"Connect ISBN  0077412745 SmartBook, the continually adaptive and fully mobile reading experience within Connectu2014now with offline access Purchase price $130\"\n",
            "\tincorrectly classified: BOOK (99.20050811767578%)\n",
            "\"Connect ISBN  0077412745 SmartBook, the continually adaptive and fully mobile reading experience within Connectu2014now with offline access Purchase price $130\"\n",
            "\tincorrectly classified: BOOK (99.20050811767578%)\n",
            "\"Connect ISBN  0077412745 SmartBook, the continually adaptive and fully mobile reading experience within Connectu2014now with offline access Purchase price $130\"\n",
            "\tincorrectly classified: BOOK (99.20050811767578%)\n",
            "\"Connect ISBN  0077412745 SmartBook, the continually adaptive and fully mobile reading experience within Connectu2014now with offline access Purchase price $130\"\n",
            "\tincorrectly classified: BOOK (99.20050811767578%)\n",
            "\"Terrell, Rogers, Kerr, Spielman, Santore\"\n",
            "\tincorrectly classified: NOT BOOK (0.01189225073903799%)\n",
            "\"Seventh Edition ISBN 0073386456\"\n",
            "\tincorrectly classified: BOOK (99.99524688720703%)\n",
            "\"Connect ISBN  0077412745 SmartBook, the continually adaptive and fully mobile reading experience within Connectu2014now with offline access Purchase price $130\"\n",
            "\tincorrectly classified: BOOK (99.20050811767578%)\n",
            "\"Connect ISBN  0077412745 SmartBook, the continually adaptive and fully mobile reading experience within Connectu2014now with offline access Purchase price $130\"\n",
            "\tincorrectly classified: BOOK (99.20050811767578%)\n",
            "\"No book order.\"\n",
            "\tincorrectly classified: BOOK (99.71566009521484%)\n",
            "\"Remote Sensing and Image Interpretation\"\n",
            "\tincorrectly classified: NOT BOOK (0.0017403773963451385%)\n",
            "\"th Edition John Wiley  Sons\"\n",
            "\tincorrectly classified: BOOK (99.98465728759766%)\n",
            "\"No book order for this class.\"\n",
            "\tincorrectly classified: BOOK (99.97496032714844%)\n",
            "\"Title Pale Rider The Spanish Flu of 1918 and How It Changed the World\"\n",
            "\tincorrectly classified: NOT BOOK (38.5550537109375%)\n",
            "\"Title And the Band Played On\"\n",
            "\tincorrectly classified: NOT BOOK (0.01837824657559395%)\n",
            "\"1 U.S. Constitution\"\n",
            "\tincorrectly classified: NOT BOOK (0.7598419189453125%)\n",
            "\"2 Federal Rules of Civil Procedure \"\n",
            "\tincorrectly classified: NOT BOOK (0.04154777154326439%)\n",
            "\"3 Federal Rules of Evidence\"\n",
            "\tincorrectly classified: NOT BOOK (0.10856536775827408%)\n",
            "\"httpswww.amazon.comskGenna Rae McNeil2C Groundwork  Charles Hamilton Houston and the Struggle for Civil Rights.refnb_sb_noss\"\n",
            "\tincorrectly classified: NOT BOOK (2.6145284175872803%)\n",
            "\"httpswww.amazon.comStatevPeytonElizabethBoalsdp1601565976refsr_1_fkmr0_1keywordsState v. Peyton2C Second Edition published by The National Institute for Trial Advocacyqid1555444071sbookssr11fkmr0\"\n",
            "\tincorrectly classified: NOT BOOK (0.11587919294834137%)\n",
            "\n",
            "Confusion Matrix:\n",
            "\tTP: 394\tFP: 21 \n",
            "\tFN: 29 \tTN: 257\n",
            "\n",
            "Precision: 0.949\n",
            "Recall: 0.931\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}