sentiment_analysis_french_political_tweets.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    },
    "colab": {
      "name": "sentiment_analysis_french_political_tweets.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "d5cd36c3ae9e43369065310934386b95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c1ff2e9989d841cabad13da1127b8c4a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5e186898a315464db5b7d8db593fc035",
              "IPY_MODEL_9af87d245d534122b856dcc121fdca1c"
            ]
          }
        },
        "c1ff2e9989d841cabad13da1127b8c4a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5e186898a315464db5b7d8db593fc035": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e3d9d802fe114f7192fdc375213f70b7",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a11907d509bf4da387e9be255f403932"
          }
        },
        "9af87d245d534122b856dcc121fdca1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_550978fce2fd4a2ca35ff7d7514afd4f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:05&lt;00:00, 44.6kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1551bfef00774669a565319e6cecc238"
          }
        },
        "e3d9d802fe114f7192fdc375213f70b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a11907d509bf4da387e9be255f403932": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "550978fce2fd4a2ca35ff7d7514afd4f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1551bfef00774669a565319e6cecc238": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "95632cd5446043ecb96ce56cc077e49e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d401adb8646241c19cf3d061e9a5df46",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_dd1401e9543c4a879042598b02e53d73",
              "IPY_MODEL_16d7c6097286453197f00d4f667f0378"
            ]
          }
        },
        "d401adb8646241c19cf3d061e9a5df46": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dd1401e9543c4a879042598b02e53d73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7ebd21cd8cdf4bf79639f56d6bb39df5",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 361,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 361,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5acfa2f73039425cb9656ffde20196a6"
          }
        },
        "16d7c6097286453197f00d4f667f0378": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_093947dbb8554cedb00d965a41f05105",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 361/361 [00:07&lt;00:00, 49.5B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d9c26b01ac874973a7b1151a54389c6b"
          }
        },
        "7ebd21cd8cdf4bf79639f56d6bb39df5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5acfa2f73039425cb9656ffde20196a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "093947dbb8554cedb00d965a41f05105": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d9c26b01ac874973a7b1151a54389c6b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8de0340680514f20a5c88b26a6c478cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fdb2e4d9dfbb4c87aee72f42d9548820",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7f38e242d9f64c7aa8d3e7e0e575d3c2",
              "IPY_MODEL_d733618480d84cffa5ab19cf2fac3aa7"
            ]
          }
        },
        "fdb2e4d9dfbb4c87aee72f42d9548820": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7f38e242d9f64c7aa8d3e7e0e575d3c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4d75e26d061549369cbcbc0a685e2a3d",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4ea724a0fed94c45970e3992f861e931"
          }
        },
        "d733618480d84cffa5ab19cf2fac3aa7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2a4cb51843c648d39f5a7fbf57c480b4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:06&lt;00:00, 64.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5ba0bcba8cc04c95b1ab9b2ebb30bc62"
          }
        },
        "4d75e26d061549369cbcbc0a685e2a3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4ea724a0fed94c45970e3992f861e931": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2a4cb51843c648d39f5a7fbf57c480b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5ba0bcba8cc04c95b1ab9b2ebb30bc62": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ffWMpZIhzx2G",
        "colab_type": "text"
      },
      "source": [
        "<center> <h1> Sentiment analysis on US political tweets and application to French political tweets during the 2017 French Presidential election </h1> </center>\n",
        "\n",
        "Project done as part of the Machine Learning for Natural Language Processing course, given in the third year of the engineering cycle at ENSAE Paris, provided by Benjamin Muller (INRIA).\n",
        "\n",
        "Authors: Clotilde Miura, Cédric Allain\n",
        "\n",
        "April 12th, 2020"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VYgeFgYPzx2H",
        "colab_type": "text"
      },
      "source": [
        "## Table of contents\n",
        "1. [Introduction](#Introduction)\n",
        "2. [Preprocessing and data exploration](#Preprocessing)\n",
        "    - [Split into train/test](#Split)\n",
        "    - [Tokenisation with Bert Tokenizer](#Tokenisation)\n",
        "3. [Fine tune Bertmodel for sentiment classification](#Fine_tune)\n",
        "    - [Define data loaders and training parameters](#Define_data_loaders)\n",
        "    - [Training loop](#Trainin_loop)\n",
        "    - [Evaluation](#Evaluation)\n",
        "4. [Application to French tweets before the 2017 presidential election](#Application)\n",
        "    - [Scrape the tweets](#Scrape)\n",
        "    - [Translate the tweets into English](#Translate)\n",
        "    - [Apply model to scraped tweets](#Apply_model)\n",
        "    - [Score of popularity](#Score_popularity)\n",
        "\n",
        "    \n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SdlWgFx61YyS",
        "colab_type": "text"
      },
      "source": [
        "<a id='Introduction'></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hNltvmrhzx2I",
        "colab_type": "text"
      },
      "source": [
        "## Introduction\n",
        "\n",
        "- Nowadays, social media yield a growing influence on politics. Indeed, politicians consider Tweeter as a good indicator to assess their popularity. In this context, NLP, and particularly Sentiment Analysis, rises a growing interest. \n",
        "\n",
        "- This project aims at developping a sentiment model to predict the sentiment of a tweet among 3 different classes : negative, positive, and neutral. We used a dataset gathering tweets about the  2015 GOP debate (grand old party, i.e. the Republican Party). We decided to use BERT embeddding and fine tune it for our specific task. \n",
        "\n",
        "- To propose an originial applications, we decided to apply our model on French political tweets (previously translated in English) during the months preceding the 2017 Presidential Election, in order to asses the popularity of each candidate. \n",
        "\n",
        "- We will first preprocess the data, then fine Tune a pre trained embedding of BERT and evaluate it quantitatively and qualitatively. Finally, we will try to apply it to French tweets."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2yPwaNxF1YwB",
        "colab_type": "code",
        "outputId": "fe0e5c57-96d0-4f12-9c7c-b1790d053839",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 683
        }
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a3/78/92cedda05552398352ed9784908b834ee32a0bd071a9b32de287327370b7/transformers-2.8.0-py3-none-any.whl (563kB)\n",
            "\r\u001b[K     |▋                               | 10kB 28.4MB/s eta 0:00:01\r\u001b[K     |█▏                              | 20kB 35.1MB/s eta 0:00:01\r\u001b[K     |█▊                              | 30kB 30.5MB/s eta 0:00:01\r\u001b[K     |██▎                             | 40kB 34.5MB/s eta 0:00:01\r\u001b[K     |███                             | 51kB 30.8MB/s eta 0:00:01\r\u001b[K     |███▌                            | 61kB 33.8MB/s eta 0:00:01\r\u001b[K     |████                            | 71kB 30.8MB/s eta 0:00:01\r\u001b[K     |████▋                           | 81kB 32.3MB/s eta 0:00:01\r\u001b[K     |█████▎                          | 92kB 34.6MB/s eta 0:00:01\r\u001b[K     |█████▉                          | 102kB 35.1MB/s eta 0:00:01\r\u001b[K     |██████▍                         | 112kB 35.1MB/s eta 0:00:01\r\u001b[K     |███████                         | 122kB 35.1MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 133kB 35.1MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 143kB 35.1MB/s eta 0:00:01\r\u001b[K     |████████▊                       | 153kB 35.1MB/s eta 0:00:01\r\u001b[K     |█████████▎                      | 163kB 35.1MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 174kB 35.1MB/s eta 0:00:01\r\u001b[K     |██████████▌                     | 184kB 35.1MB/s eta 0:00:01\r\u001b[K     |███████████                     | 194kB 35.1MB/s eta 0:00:01\r\u001b[K     |███████████▋                    | 204kB 35.1MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 215kB 35.1MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 225kB 35.1MB/s eta 0:00:01\r\u001b[K     |█████████████▍                  | 235kB 35.1MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 245kB 35.1MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 256kB 35.1MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 266kB 35.1MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 276kB 35.1MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 286kB 35.1MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 296kB 35.1MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 307kB 35.1MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 317kB 35.1MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 327kB 35.1MB/s eta 0:00:01\r\u001b[K     |███████████████████▏            | 337kB 35.1MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 348kB 35.1MB/s eta 0:00:01\r\u001b[K     |████████████████████▍           | 358kB 35.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 368kB 35.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████▌          | 378kB 35.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 389kB 35.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 399kB 35.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████▎        | 409kB 35.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 419kB 35.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████▍       | 430kB 35.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 440kB 35.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 450kB 35.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▏     | 460kB 35.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 471kB 35.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▎    | 481kB 35.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 491kB 35.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 501kB 35.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 512kB 35.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 522kB 35.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 532kB 35.1MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 542kB 35.1MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▍| 552kB 35.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 563kB 35.1MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 573kB 35.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.12.38)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.38.0)\n",
            "Collecting tokenizers==0.5.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d1/3f/73c881ea4723e43c1e9acf317cf407fab3a278daab3a69c98dcac511c04f/tokenizers-0.5.2-cp36-cp36m-manylinux1_x86_64.whl (3.7MB)\n",
            "\u001b[K     |████████████████████████████████| 3.7MB 53.3MB/s \n",
            "\u001b[?25hCollecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
            "\u001b[K     |████████████████████████████████| 1.0MB 55.1MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/b4/7a41d630547a4afd58143597d5a49e07bfd4c42914d8335b2a5657efc14b/sacremoses-0.0.38.tar.gz (860kB)\n",
            "\u001b[K     |████████████████████████████████| 870kB 59.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.21.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.3.3)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.9.5)\n",
            "Requirement already satisfied: botocore<1.16.0,>=1.15.38 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (1.15.38)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.1)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.14.1)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.4.5.1)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.38->boto3->transformers) (0.15.2)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.38->boto3->transformers) (2.8.1)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.38-cp36-none-any.whl size=884628 sha256=c059dba48b54e9f31f87ec925b64de22b947d9dbcaf4f1022bf0731f1bddf5fc\n",
            "  Stored in directory: /root/.cache/pip/wheels/6d/ec/1a/21b8912e35e02741306f35f66c785f3afe94de754a0eaf1422\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sentencepiece, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.38 sentencepiece-0.1.85 tokenizers-0.5.2 transformers-2.8.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Biz55931pkf",
        "colab_type": "code",
        "outputId": "b5c8a52a-e973-40fd-eb60-ecd70021fd9f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "# mount your drive to access the data\n",
        "colab = True \n",
        "if colab:\n",
        "  from google.colab import drive\n",
        "  drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cpftI7jAWJCT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# you must change the path to load the data on your computer \n",
        "\n",
        "# path: path to folder where all data are stored\n",
        "# path_model: path to folder where the model is (going to be) saved\n",
        "\n",
        "if colab:\n",
        "  path = './drive/My Drive/NLP_ENSAE/NLP_project/data/'\n",
        "  path_model = './drive/My Drive/NLP_ENSAE/NLP_project/'\n",
        "else: \n",
        "  path = './data/'\n",
        "  path_model = './'\n",
        "\n",
        "# name of the model to be trained (or already saved)\n",
        "model_name = 'sentiment_classifier'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZMgU5Y5tzx2P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import pandas as pd\n",
        "pd.set_option('display.max_columns', 500)\n",
        "pd.set_option(\"display.max_rows\", 500)\n",
        "import numpy as np \n",
        "import math\n",
        "from tqdm import tqdm\n",
        "import pickle\n",
        "\n",
        "from collections import defaultdict, Counter\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns \n",
        "sns.set(color_codes = True)\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.metrics import classification_report\n",
        "\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "import torch\n",
        "from transformers import BertTokenizer, BertModel, BertForMaskedLM\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torch.autograd import Variable\n",
        "from torch import autograd\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rezyAvBy1e_a",
        "colab_type": "text"
      },
      "source": [
        "<a id='Preprocessing'></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-qhfY9NsHmfI",
        "colab_type": "text"
      },
      "source": [
        "## Preprocessing and data exploration\n",
        "\n",
        "- You must change the path to load the data on your environment execution . It can be for example './drive/Shared with me/NLP_project/data' or if you want to load the data in your own drive './drive/My Drive/....'\n",
        "- You can also upload the csv file directly\n",
        "- The data contains 13871 tweets and 20 different variables but we will only use the sentiment and text variables "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KlZ_Ccwszx2T",
        "colab_type": "code",
        "outputId": "818f711e-7258-4aa7-8e49-e815fe7cf1af",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 428
        }
      },
      "source": [
        "file = 'GOP_REL_ONLY.csv'\n",
        "data = pd.read_csv(path+file, encoding='latin-1' )\n",
        "print('shape data: ', data.shape)\n",
        "data.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "shape data:  (13871, 20)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>candidate</th>\n",
              "      <th>candidate:confidence</th>\n",
              "      <th>relevant_yn</th>\n",
              "      <th>relevant_yn:confidence</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>sentiment:confidence</th>\n",
              "      <th>subject_matter</th>\n",
              "      <th>subject_matter:confidence</th>\n",
              "      <th>candidate_gold</th>\n",
              "      <th>name</th>\n",
              "      <th>relevant_yn_gold</th>\n",
              "      <th>retweet_count</th>\n",
              "      <th>sentiment_gold</th>\n",
              "      <th>subject_matter_gold</th>\n",
              "      <th>text</th>\n",
              "      <th>tweet_coord</th>\n",
              "      <th>tweet_created</th>\n",
              "      <th>tweet_id</th>\n",
              "      <th>tweet_location</th>\n",
              "      <th>user_timezone</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>No candidate mentioned</td>\n",
              "      <td>1.0</td>\n",
              "      <td>yes</td>\n",
              "      <td>1.0</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>0.6578</td>\n",
              "      <td>None of the above</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>I_Am_Kenzi</td>\n",
              "      <td>NaN</td>\n",
              "      <td>5</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>RT @NancyLeeGrahn: How did everyone feel about...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8/7/15 9:54</td>\n",
              "      <td>6.296970e+17</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Quito</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Scott Walker</td>\n",
              "      <td>1.0</td>\n",
              "      <td>yes</td>\n",
              "      <td>1.0</td>\n",
              "      <td>Positive</td>\n",
              "      <td>0.6333</td>\n",
              "      <td>None of the above</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>PeacefulQuest</td>\n",
              "      <td>NaN</td>\n",
              "      <td>26</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>RT @ScottWalker: Didn't catch the full #GOPdeb...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8/7/15 9:54</td>\n",
              "      <td>6.296970e+17</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>No candidate mentioned</td>\n",
              "      <td>1.0</td>\n",
              "      <td>yes</td>\n",
              "      <td>1.0</td>\n",
              "      <td>Neutral</td>\n",
              "      <td>0.6629</td>\n",
              "      <td>None of the above</td>\n",
              "      <td>0.6629</td>\n",
              "      <td>NaN</td>\n",
              "      <td>PussssyCroook</td>\n",
              "      <td>NaN</td>\n",
              "      <td>27</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>RT @TJMShow: No mention of Tamir Rice and the ...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8/7/15 9:54</td>\n",
              "      <td>6.296970e+17</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>No candidate mentioned</td>\n",
              "      <td>1.0</td>\n",
              "      <td>yes</td>\n",
              "      <td>1.0</td>\n",
              "      <td>Positive</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>None of the above</td>\n",
              "      <td>0.7039</td>\n",
              "      <td>NaN</td>\n",
              "      <td>MattFromTexas31</td>\n",
              "      <td>NaN</td>\n",
              "      <td>138</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>RT @RobGeorge: That Carly Fiorina is trending ...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8/7/15 9:54</td>\n",
              "      <td>6.296970e+17</td>\n",
              "      <td>Texas</td>\n",
              "      <td>Central Time (US &amp; Canada)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Donald Trump</td>\n",
              "      <td>1.0</td>\n",
              "      <td>yes</td>\n",
              "      <td>1.0</td>\n",
              "      <td>Positive</td>\n",
              "      <td>0.7045</td>\n",
              "      <td>None of the above</td>\n",
              "      <td>1.0000</td>\n",
              "      <td>NaN</td>\n",
              "      <td>sharonDay5</td>\n",
              "      <td>NaN</td>\n",
              "      <td>156</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>RT @DanScavino: #GOPDebate w/ @realDonaldTrump...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>8/7/15 9:54</td>\n",
              "      <td>6.296970e+17</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Arizona</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                candidate  candidate:confidence relevant_yn  \\\n",
              "0  No candidate mentioned                   1.0         yes   \n",
              "1            Scott Walker                   1.0         yes   \n",
              "2  No candidate mentioned                   1.0         yes   \n",
              "3  No candidate mentioned                   1.0         yes   \n",
              "4            Donald Trump                   1.0         yes   \n",
              "\n",
              "   relevant_yn:confidence sentiment  sentiment:confidence     subject_matter  \\\n",
              "0                     1.0   Neutral                0.6578  None of the above   \n",
              "1                     1.0  Positive                0.6333  None of the above   \n",
              "2                     1.0   Neutral                0.6629  None of the above   \n",
              "3                     1.0  Positive                1.0000  None of the above   \n",
              "4                     1.0  Positive                0.7045  None of the above   \n",
              "\n",
              "   subject_matter:confidence candidate_gold             name relevant_yn_gold  \\\n",
              "0                     1.0000            NaN       I_Am_Kenzi              NaN   \n",
              "1                     1.0000            NaN    PeacefulQuest              NaN   \n",
              "2                     0.6629            NaN    PussssyCroook              NaN   \n",
              "3                     0.7039            NaN  MattFromTexas31              NaN   \n",
              "4                     1.0000            NaN       sharonDay5              NaN   \n",
              "\n",
              "   retweet_count sentiment_gold subject_matter_gold  \\\n",
              "0              5            NaN                 NaN   \n",
              "1             26            NaN                 NaN   \n",
              "2             27            NaN                 NaN   \n",
              "3            138            NaN                 NaN   \n",
              "4            156            NaN                 NaN   \n",
              "\n",
              "                                                text tweet_coord  \\\n",
              "0  RT @NancyLeeGrahn: How did everyone feel about...         NaN   \n",
              "1  RT @ScottWalker: Didn't catch the full #GOPdeb...         NaN   \n",
              "2  RT @TJMShow: No mention of Tamir Rice and the ...         NaN   \n",
              "3  RT @RobGeorge: That Carly Fiorina is trending ...         NaN   \n",
              "4  RT @DanScavino: #GOPDebate w/ @realDonaldTrump...         NaN   \n",
              "\n",
              "  tweet_created      tweet_id tweet_location               user_timezone  \n",
              "0   8/7/15 9:54  6.296970e+17            NaN                       Quito  \n",
              "1   8/7/15 9:54  6.296970e+17            NaN                         NaN  \n",
              "2   8/7/15 9:54  6.296970e+17            NaN                         NaN  \n",
              "3   8/7/15 9:54  6.296970e+17          Texas  Central Time (US & Canada)  \n",
              "4   8/7/15 9:54  6.296970e+17            NaN                     Arizona  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ERpoE5NNzx2Z",
        "colab_type": "code",
        "outputId": "167df946-974b-4b3d-86e2-eb84bbb535af",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 476
        }
      },
      "source": [
        "data.info()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 13871 entries, 0 to 13870\n",
            "Data columns (total 20 columns):\n",
            " #   Column                     Non-Null Count  Dtype  \n",
            "---  ------                     --------------  -----  \n",
            " 0   candidate                  13775 non-null  object \n",
            " 1   candidate:confidence       13871 non-null  float64\n",
            " 2   relevant_yn                13871 non-null  object \n",
            " 3   relevant_yn:confidence     13871 non-null  float64\n",
            " 4   sentiment                  13871 non-null  object \n",
            " 5   sentiment:confidence       13871 non-null  float64\n",
            " 6   subject_matter             13545 non-null  object \n",
            " 7   subject_matter:confidence  13871 non-null  float64\n",
            " 8   candidate_gold             28 non-null     object \n",
            " 9   name                       13871 non-null  object \n",
            " 10  relevant_yn_gold           32 non-null     object \n",
            " 11  retweet_count              13871 non-null  int64  \n",
            " 12  sentiment_gold             15 non-null     object \n",
            " 13  subject_matter_gold        18 non-null     object \n",
            " 14  text                       13871 non-null  object \n",
            " 15  tweet_coord                21 non-null     object \n",
            " 16  tweet_created              13871 non-null  object \n",
            " 17  tweet_id                   13871 non-null  float64\n",
            " 18  tweet_location             9959 non-null   object \n",
            " 19  user_timezone              9468 non-null   object \n",
            "dtypes: float64(5), int64(1), object(14)\n",
            "memory usage: 2.1+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V9zSWgtNzx2d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def plot_counts(col, data, group=None,\n",
        "                rotate_xticks=False, palette=\"Blues\", figsize=(7,5)): \n",
        "    \"\"\"For a categorical variable, plot the distribution of categories\n",
        "    add the percentages on the graph countrary to seaborn function\n",
        "    \n",
        "    Parameters\n",
        "    ----------\n",
        "    col: string : name of variable\n",
        "\n",
        "    data: dataframe of data \n",
        "\n",
        "    group: can plot percentages per group\n",
        "\n",
        "\n",
        "    Returns\n",
        "    -------\n",
        "    None\n",
        "    \n",
        "    \"\"\"\n",
        "    fig, ax = plt.subplots(1, figsize=figsize)\n",
        "\n",
        "    sns.countplot(x=col, data=data, hue=group, palette=palette,  ax=ax)\n",
        "\n",
        "    if rotate_xticks == True:\n",
        "        ax.set_xticklabels(ax.get_xticklabels(), rotation=40, ha=\"right\")\n",
        "\n",
        "    for j in range(len(ax.patches)):\n",
        "        p = ax.patches[j]\n",
        "        height = p.get_height()\n",
        "        ax.text(p.get_x() + p.get_width()/2.,\n",
        "                height + 3,\n",
        "                '{:1.2f}'.format(height/len(data)),\n",
        "                ha=\"center\") \n",
        "        \n",
        "    fig.tight_layout()\n",
        "\n",
        "    return None"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m_AYNb2jzx2i",
        "colab_type": "code",
        "outputId": "1cd65f53-6e03-4d9c-ae98-8594a9cfb585",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 365
        }
      },
      "source": [
        "# distribution of the variable sentiment ie the target\n",
        "plot_counts(col='sentiment', data=data)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAewAAAFcCAYAAAAK4I0VAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dfVhUdf7/8dfMIAiCIog43ouuRdvXNLn01zeV8g7XBV1vCpfWbszKzftMRd2FcjMvEFu73TIr+7qV69aagiXmtj+zvmaSmhm7pYTmDYKBxI0IOnN+f/hrNpY0GIbBMz4f19V1Oed9bt7Hjuc15zMz51gMwzAEAACuaNbmbgAAAPw0AhsAABMgsAEAMAECGwAAEyCwAQAwAQIbAAATILABADABv+ZuoDmdOVMpp5OfoQMAmp/ValHbtq0uWb+qA9vpNAhsAIApMCQOAIAJENgAAJgAgQ0AaLSysu+0aNHDGj58kCZMiNe2bVsvOe+XX/5L06ffpxEjBishYaQ2bHjDVXvxxT/pzjsTFRs7UC+99II3WjeNq/ozbACAZ6xcmaYWLVpo8+ZtOnToKy1YMFu9ev1MUVE9a81XWlqqefNmatash3TLLcN04cJ5FRUVueqdO3fRb387S5s2veXtXbjicYUNAGiUqqoq7djxvqZOnaagoCDdcENfDRo0RNnZ79SZ9y9/eU0DB/4fjRz5C/n7+ysoqJW6d+/hqv/iF/G66aabFRR06W9LX60IbABAoxw7dlQ2m01du3ZzTevZs7fy87+uM+8XX3yukJA2mjZtiuLjR2jBgrk6deqUN9s1LQIbANAoVVVVatUquNa04OBgnT1bWWfeoqIibd2apdmz5+mtt7LUsWNHPfroYm+1amoENgCgUQIDA1VZWVFrWmVl5Y8OawcEBGjIkFsVHf1zBQQE6J577tPnnx9QRUVFnXlRG4ENAGiULl26yeFw6Nixb1zTDh/+Sj16RNWZt1evXrVeWyyWJu/PVxDYAIBGCQwMVGzsrVqz5nlVVVXpwIH9+vDDHYqLG11n3tGjx+iDD/6vDh36UhcuXNDatWvUp09fBQdfHFK/cOGCqqur5XQ65XA4VF1dLYfD4e1duiJZDMO4au/NWVxcwa1JAcADysq+0/LlS7Vnz261bt1G06bN1MiRo/TZZ/v08MOz9N57O13zbtz4pl599SWdO3dOffr01bx5CxUZ2UGStGzZI3r33axa6168OFWjRyd4dX+ag9VqUXh48CXrBDaBDeAKE9y6pQIDWjR3G2iEqurzqig716BlfiqwuXEKAFxhAgNaKGbOuuZuA42Qs2qyKtSwwP4pfIYNAIAJENgAAJgAgQ0AgAkQ2AAAmACBDQCACRDYAACYAIENAIAJENgAAJgAgQ0AgAkQ2AAAmACBDQCACXgtsP/xj3/oV7/6lcaOHasxY8Zo27ZtkqT8/HwlJiYqLi5OiYmJOnLkiGsZd2sAAPgarwS2YRhasGCB0tPTtWnTJqWnp2vhwoVyOp1KTU1VUlKSsrOzlZSUpJSUFNdy7tYAAPA1XrvCtlqtKi8vlySVl5erffv2OnPmjHJzcxUfHy9Jio+PV25urkpKSlRcXOxWDQAAX+SVx2taLBatWrVKDz74oIKCglRZWanVq1eroKBAkZGRstlskiSbzab27duroKBAhmG4VQsLC6t3X5d77igAAI0RERHi0fV5JbAvXLigF154Qc8995z69++vTz/9VHPmzFF6ero3Nn9JxcUVcjqNZu0BAP6Tp0/0aB6nT5c3aH6r1XLZC0mvBPY///lPFRUVqX///pKk/v37KzAwUAEBASosLJTD4ZDNZpPD4VBRUZHsdrsMw3CrBgCAL/LKZ9gdOnTQqVOn9PXXX0uS8vLyVFxcrG7duik6OlpZWVmSpKysLEVHRyssLEzh4eFu1QAA8EUWwzC8Mia8efNmvfjii7JYLJKkWbNmafjw4crLy1NycrLKysrUunVrpaWlKSoqSpLcrtUXQ+IArkQRESGKmbOuudtAI+SsmuzxIXGvBfaViMAGcCUisM2vKQKbO50BAGACBDYAACZAYAMAYAIENgAAJkBgAwBgAgQ2AAAmQGADAGACBDYAACZAYAMAYAIENgAAJkBgAwBgAgQ2AAAmQGADAGACBDYAACZAYAMAYAIENgAAJkBgAwBgAgQ2AAAmQGADAGACBDYAACZAYAMAYAIENgAAJkBgAwBgAgQ2AAAm4OeNjRw/flzTp093vS4vL1dFRYU++eQT5efnKzk5WaWlpQoNDVVaWpq6d+8uSW7XAADwNV65wu7cubM2bdrk+m/YsGGKj4+XJKWmpiopKUnZ2dlKSkpSSkqKazl3awAA+BqvD4nX1NQoMzNTEyZMUHFxsXJzc13hHR8fr9zcXJWUlLhdAwDAF3llSPyH3n//fUVGRurnP/+5Dh48qMjISNlsNkmSzWZT+/btVVBQIMMw3KqFhYV5e5cAAGhyXg/st956SxMmTPD2Zn9UeHhwc7cAAPBREREhHl2fVwO7sLBQe/bsUXp6uiTJbrersLBQDodDNptNDodDRUVFstvtMgzDrVpDFBdXyOk0mmJXAcBtnj7Ro3mcPl3eoPmtVstlLyS9+hn2xo0bFRsbq7Zt20qSwsPDFR0draysLElSVlaWoqOjFRYW5nYNAABfZDEMw2uXmHFxcVqyZImGDBnimpaXl6fk5GSVlZWpdevWSktLU1RUVKNq9cUVNoArUUREiGLmrGvuNtAIOasme/wK26uBfaUhsAFciQhs82uKwOZOZwAAmACBDQCACRDYAACYAIENAIAJENgAAJgAgQ0AgAkQ2AAAmACBDQCACRDYAACYAIENAIAJENgAAJgAgQ0AgAkQ2AAAmACBDQCACRDYAACYAIENAIAJENgAAJgAgQ0AgAkQ2AAAmACBDQCACRDYAACYAIENAIAJENgAAJgAgQ0AgAkQ2AAAmIDXAru6ulqpqakaOXKkEhIS9Pvf/16SlJ+fr8TERMXFxSkxMVFHjhxxLeNuDQAAX+O1wF6xYoUCAgKUnZ2tzMxMzZ49W5KUmpqqpKQkZWdnKykpSSkpKa5l3K0BAOBrvBLYlZWVevvttzV79mxZLBZJUrt27VRcXKzc3FzFx8dLkuLj45Wbm6uSkhK3awAA+CI/b2zk2LFjCg0N1TPPPKPdu3erVatWmj17tlq2bKnIyEjZbDZJks1mU/v27VVQUCDDMNyqhYWF1buv8PBgz+8sAACSIiJCPLo+rwS2w+HQsWPHdN1112nhwoX67LPPNG3aND355JPe2PwlFRdXyOk0mrUHAPhPnj7Ro3mcPl3eoPmtVstlLyS9Eth2u11+fn6uIewbbrhBbdu2VcuWLVVYWCiHwyGbzSaHw6GioiLZ7XYZhuFWDQAAX+SVz7DDwsI0cOBAffTRR5IufsO7uLhY3bt3V3R0tLKysiRJWVlZio6OVlhYmMLDw92qAQDgiyyGYXhlTPjYsWNavHixSktL5efnpzlz5ig2NlZ5eXlKTk5WWVmZWrdurbS0NEVFRUmS27X6YkgcwJUoIiJEMXPWNXcbaIScVZM9PiTutcC+EhHYAK5EBLb5NUVgc6czAABMgMAGAMAECGwAAEyAwAYAwAQIbAAATIDABgDABAhsAABMgMAGAMAECGwAAEyAwAYAwAQIbAAATIDABgDABAhsAABMgMAGAMAECGwAAEyAwAYAwAQIbAAATIDABgDABAhsAABMgMAGAMAECGwAAEyAwAYAwAQIbAAATIDABgDABPy8taGhQ4fK399fAQEBkqSHH35YgwcP1v79+5WSkqLq6mp16tRJK1asUHh4uCS5XQMAwNd49Qr7qaee0qZNm7Rp0yYNHjxYTqdT8+fPV0pKirKzsxUTE6OMjAxJcrsGAIAvatYh8YMHDyogIEAxMTGSpEmTJmnr1q2NqgEA4Iu8NiQuXRwGNwxD/fv310MPPaSCggJ17NjRVQ8LC5PT6VRpaanbtdDQ0Hr3Ex4e7JkdAwDgP0REhHh0fV4L7Ndee012u101NTVatmyZli5dqhEjRnhr8z+quLhCTqfRrD0AwH/y9IkezeP06fIGzW+1Wi57Iem1IXG73S5J8vf3V1JSkvbu3Su73a6TJ0+65ikpKZHValVoaKjbNQAAfJFXAvvs2bMqL7/4TsMwDL3zzjuKjo7W9ddfr3PnziknJ0eStH79eo0aNUqS3K4BAOCL6j0k/tJLL+nee++tM/2VV17RPffcc9lli4uLNXPmTDkcDjmdTvXs2VOpqamyWq1KT09XampqrZ9nSXK7BgCAL7IYhlGvD3FvvPFG7d27t870AQMG6JNPPvF4Y97AZ9gArkQRESGKmbOuudtAI+Ssmuzxz7B/8gp7165dki7+9vnjjz/WD/P9+PHjatWqVYMaAgAADfeTgb1kyRJJUnV1tRYvXuyabrFYFBERod/97ndN1x0AAJBUj8B+//33JUkLFixQenp6kzcEAADqqveXzn4Y1k6ns1bNauUZIgAANKV6B/YXX3yhpUuX6ssvv1R1dbWkiz/Rslgs+uc//9lkDQIAgAYEdnJysm699VY9/vjjatmyZVP2BAAA/kO9A/vEiROaO3euLBZLU/YDAAB+RL0/fB4xYoQ+/PDDpuwFAABcQr2vsKurqzVjxgz1799f7dq1q1Xj2+MAADStegd2r1691KtXr6bsBQAAXEK9A3vGjBlN2QcAALiMegf297co/TE33XSTR5oBAAA/rt6B/f0tSr935swZnT9/XpGRkfr73//u8cYAAMC/1Tuwv79F6fccDof+9Kc/8fAPAAC8wO17itpsNk2bNk1r1qzxZD8AAOBHNOom4B999BE3UgEAwAvqPSQeGxtbK5yrqqpUU1Oj1NTUJmkMAAD8W70De8WKFbVeBwYGqkePHgoODvZ4UwAAoLZ6B/aAAQMkXXy05rfffqt27drxWE0AALyk3olbUVGhBQsWqE+fPhoyZIj69OmjhQsXqry8vCn7AwAAakBgP/bYY6qqqlJmZqYOHDigzMxMVVVV6bHHHmvK/gAAgBowJL5z505t375dgYGBkqQePXpo+fLlGjFiRJM1BwAALqr3FXZAQIBKSkpqTTtz5oz8/f093hQAAKit3lfYEydO1JQpU3T33XerY8eOOnnypNauXavbbrutKfsDAABqQGD/9re/VWRkpDIzM1VUVKT27dtr6tSpDQ7sZ555Rk8//bQyMzPVu3dv7d+/XykpKaqurlanTp20YsUKhYeHS5LbNQAAfE29h8SXLVumHj16aO3atXrnnXe0du1a9ezZU8uWLav3xr744gvt379fnTp1knTxJ2Lz589XSkqKsrOzFRMTo4yMjEbVAADwRfUO7KysLF1//fW1pl1//fXKysqq1/I1NTVaunSpHnnkEde0gwcPKiAgQDExMZKkSZMmaevWrY2qAQDgi+od2BaLRU6ns9Y0h8NRZ9qlPPnkkxozZow6d+7smlZQUKCOHTu6XoeFhcnpdKq0tNTtGgAAvqjen2HHxMToySef1Pz582W1WuV0OvX000+7rnIvZ9++fTp48KAefvjhRjXraeHh3FYVANA0IiJCPLq+egf2kiVL9MADD2jQoEHq2LGjCgoKFBERoeeff/4nl92zZ4/y8vI0bNgwSdKpU6d07733avLkyTp58qRrvpKSElmtVoWGhsput7tVa4ji4go5nUaDlgGApubpEz2ax+nTDbsTqNVqueyFZL0Du0OHDtq4caMOHDiggoIC2e129enTp173E7///vt1//33u14PHTpUzz//vHr16qUNGzYoJydHMTExWr9+vUaNGiXp4ufj586da3ANAABfVO/AliSr1aq+ffuqb9++Htm41WpVenq6UlNTa/08qzE1AAB8kcUwjKt2TJghcQBXooiIEMXMWdfcbaARclZN9viQOM/HBADABAhsAABMgMAGAMAECGwAAEyAwAYAwAQIbAAATIDABgDABAhsAABMgMAGAMAECGwAAEyAwAYAwAQIbAAATIDABgDABAhsAABMgMAGAMAE/Jq7ATS9srLvtHz5H7Rnz8dq0yZUDzwwQyNHjqoz3+uv/4/efTdLp06dUmhoqMaNm6ikpDtd9ZkzH1B+fp5qas7Lbu+oqVMf0ODBt3hxTwDg6kVgXwVWrkxTixYttHnzNh069JUWLJitXr1+pqionrXmMwxDv/vdUvXs2UsnTx7X3Lkz1L59pIYPj5MkzZ79sLp37yE/Pz998cVBzZnzoN54429q165dc+wWAFxVGBL3cVVVVdqx431NnTpNQUFBuuGGvho0aIiys9+pM+8dd9yla665Vn5+furatbsGD47V559/5qr36vUz+fldfI9nsUgOxwUVFZ3y2r4AwNWMK2wfd+zYUdlsNnXt2s01rWfP3tq/f+9llzMMQ599tk9jx46vNX3BgjnKyflENTU1GjDgJl177XVN0jcAoDYC28dVVVWpVavgWtOCg4N19mzlZZd7+eXVcjoNjR49ptb09PRVunDhgvbs2a2jR/NltTJIAwDewNnWxwUGBqqysqLWtMrKSgUFtbrkMm+99Rdt3bpFK1askr+/f526n5+fbrrpZn3yyW59+OEOj/cMAKiLwPZxXbp0k8Ph0LFj37imHT78lXr0iPrR+bOyNunPf35Vq1Y9p/btIy+7bofDoRMnjnu0XwDAjyOwfVxgYKBiY2/VmjXPq6qqSgcO7NeHH+5QXNzoOvNu2/auVq9+Tn/847Pq1KlzrdrRo0e0a9dHqq4+pwsXLig7+x199tle9e3b31u7AgBXNYthGEZzN9Fciosr5HQ2bPfbtm0lPz9zvc8pLS3V4sWL9b//+78KDQ3VvHnzlJCQoJycHN13333at2+fJGno0KEqLCysNQyekJCgpUuXKi8vT8nJyTp8+LBsNpu6deumadOmacSIEc21W267cMGpM2cu/xk+0JwiIkIUM2ddc7eBRshZNVmnT5c3aBmr1aLw8OBL1gnsBgZ2RESIcg6fbqKO4A0xvSIa/A8J8CYC2/yaIrC99i3xBx98UMePH5fValVQUJB+//vfKzo6Wvn5+UpOTlZpaalCQ0OVlpam7t27S5LbNQAAfI3XxnbT0tK0efNmvf3225oyZYoWL14sSUpNTVVSUpKys7OVlJSklJQU1zLu1gAA8DVeC+yQkBDXnysqKmSxWFRcXKzc3FzFx8dLkuLj45Wbm6uSkhK3awAA+CKv3jhlyZIl+uijj2QYhtasWaOCggJFRkbKZrNJkmw2m9q3b6+CggIZhuFWLSwsrN79XO6zAvi2iIiQn54JABrB0+cZrwb2smXLJElvv/220tPTNXv2bG9uvg53v3QG8+NLZ7iScZ7xDZ7+0lmz/D7pV7/6lXbv3q0OHTqosLBQDodD0sUbcRQVFclut8tut7tVAwDAF3klsCsrK1VQUOB6/f7776tNmzYKDw9XdHS0srKyJElZWVmKjo5WWFiY2zUAAHyRV36H/e233+rBBx9UVVWVrFar2rRpo4ULF+rnP/+564YcZWVlat26tdLS0hQVdfG2me7W6ovfYV+d+B02rnT8Dtv8uHGKhxHYVycCG1c6Atv8miKwzXWPTQAArlIENgAAJkBgAwBgAgQ2AAAmQGADAGACBDYAACZAYAMAYAIENgAAJkBgAwBgAgQ2AAAmQGADAGACBDaAOsrKvtOiRQ9r+PBBmjAhXtu2bf3R+fbuzdHMmQ8oLi5WEycm/Og8Gza8odtuG6Phwwfpjjsm6ptvjjZl64DP8mvuBgBceVauTFOLFi20efM2HTr0lRYsmK1evX6mqKieteZr2bKlfvnLMRo+PE7r1r1SZz2ZmW9ry5ZNSk9fpe7de+jkyRMKCQnx1m4APoUrbAC1VFVVaceO9zV16jQFBQXphhv6atCgIcrOfqfOvNddd71GjfqlOnbsVKfmdDr1yisvaubMh9SjR5QsFos6deqs1q3beGM3AJ9DYAOo5dixo7LZbOratZtrWs+evZWf/3WD1lNUVKSiokJ9/XWexo//pW67bYxeeukFOZ1OT7cMXBUYEgdQS1VVlVq1qv1M3uDgYJ09W9mg9Zw+XShJ2rPnY7366npVVJRr7twZiohorzFjxnmsX+BqwRU2gFoCAwNVWVlRa1plZaWCglo1aD0BAQGSpKSkOxUSEiK7vaPGjh2vXbs+8livwNWEwAZQS5cu3eRwOHTs2DeuaYcPf6UePaIatJ6uXburRYsWslgsrmk/+COABiKwAdQSGBio2NhbtWbN86qqqtKBA/v14Yc7FBc3us68TqdT1dXVunDhggzDUHV1tc6fPy/p4jfIhw4doddf/x+dPVupoqJCbd68UTffPMjbuwT4BAIbQB3z5iWrpqZaCQkj9MgjSzRv3iJFRfXUZ5/t04gRg13z7d+/V8OG3az582ersPCUhg27WXPnTnfVH3pogQIDAzV27C80bdoUjRgxSr/85djm2CXA9CyGYRjN3URzKS6ukNPZsN2PiAhRzuHTTdQRvCGmV4ROny732vZahwYpoIXNa9uD51Wfd6is9KzXthcREaKYOeu8tj14Xs6qyQ0+z1itFoWHB1+yzrfEgSYW0MKmRX/d3dxtoBGW3zawuVsAGBIHAMAMCGwAAEzAK4F95swZ3XfffYqLi1NCQoJmzJihkpISSdL+/fs1ZswYxcXFacqUKSouLnYt524NAABf45XAtlgsmjp1qrKzs5WZmakuXbooIyNDTqdT8+fPV0pKirKzsxUTE6OMjAxJcrsGAIAv8kpgh4aGauDAf39po2/fvjp58qQOHjyogIAAxcTESJImTZqkrVsvPsbP3RoAAL7I659hO51OvfHGGxo6dKgKCgrUsWNHVy0sLExOp1OlpaVu1wAA8EVe/1nXH/7wBwUFBek3v/mN3nvvPW9vvpbL/d4Nvi0igmcyo2E4ZtBQnj5mvBrYaWlpOnr0qJ5//nlZrVbZ7XadPHnSVS8pKZHValVoaKjbtYZw98YpMD9v3jiFY8Y3cMygoTx94xSvDYk/8cQTOnjwoJ599ln5+/tLkq6//nqdO3dOOTk5kqT169dr1KhRjaoBAOCLvHKFfejQIb3wwgvq3r27Jk2aJEnq3Lmznn32WaWnpys1NVXV1dXq1KmTVqxYIUmyWq1u1QAA8EVeCeyf/exn+vLLL3+0duONNyozM9OjNQAAfA13OgMAwAQIbAAATIDABgDABAhsAABMgMAGAMAECGwAAEyAwAYAwAQIbAAATIDABgDABAhsAABMgMAGAMAECGwAAEyAwAYAwAQIbAAATIDABgDABAhsAABMgMAGAMAECGwAAEyAwAYAwAQIbAAATIDABgDABAhsAABMgMAGAMAECGwAAEzAK4GdlpamoUOH6pprrtFXX33lmp6fn6/ExETFxcUpMTFRR44caXQNAABf5JXAHjZsmF577TV16tSp1vTU1FQlJSUpOztbSUlJSklJaXQNAABf5JXAjomJkd1urzWtuLhYubm5io+PlyTFx8crNzdXJSUlbtcAAPBVfs214YKCAkVGRspms0mSbDab2rdvr4KCAhmG4VYtLCysuXYHAIAm1WyBfSUIDw9u7hbQTCIiQpq7BZgMxwwaytPHTLMFtt1uV2FhoRwOh2w2mxwOh4qKimS322UYhlu1hiourpDTaTRoGf7R+obTp8u9ti2OGd/AMYOGaugxY7VaLnsh2Ww/6woPD1d0dLSysrIkSVlZWYqOjlZYWJjbNQAAfJVXrrAfe+wxbdu2Td9++63uuecehYaGasuWLXrkkUeUnJys5557Tq1bt1ZaWpprGXdrAAD4IothGA0bE/Yh7g6J5xw+3UQdwRtiekV4fXhz0V93e2178Lzltw30+jETM2ed17YHz8tZNdl3hsQBAED9EdgAAJgAgQ0AgAkQ2AAAmACBDQCACRDYAACYAIENAIAJENgAAJgAgQ0AgAkQ2AAAmACBDQCACRDYAACYAIENAIAJENgAAJgAgQ0AgAkQ2AAAmACBDQCACRDYAACYAIENAIAJENgAAJgAgQ0AgAkQ2AAAmACBDQCACRDYAACYAIENAIAJmDqw8/PzlZiYqLi4OCUmJurIkSPN3RIAAE3C1IGdmpqqpKQkZWdnKykpSSkpKc3dEgAATcKvuRtwV3FxsXJzc/XKK69IkuLj4/WHP/xBJSUlCgsLq9c6rFaLW9v29zP1+xzI/f/37goN8vfq9uB53j5m7GGtvLo9eF5Dj5mfmt+0gV1QUKDIyEjZbDZJks1mU/v27VVQUFDvwG7b1r1/EH26h7u1HK4c4eHBXt3ewl/28+r24HnePmYyU8Z7dXvwPE8fM1wqAgBgAqYNbLvdrsLCQjkcDkmSw+FQUVGR7HZ7M3cGAIDnmTaww8PDFR0draysLElSVlaWoqOj6z0cDgCAmVgMwzCauwl35eXlKTk5WWVlZWrdurXS0tIUFRXV3G0BAOBxpg5sAACuFqYdEgcA4GpCYAMAYAIENgAAJkBgAwBgAgS2Dxg6dKji4+PldDprTfvqq6/cXufTTz+tmpoat5bdvXu3xo/nLk1XqqFDh2rUqFEaM2aM4uPjtWXLlgav4/PPP9e8efMkSWVlZXrxxRdr1ZcsWaKcnByP9Ivm0RTnlUvhGKofAttHnD17Vps2bfLY+p555hmdP3/+R2sXLlzw2HbQPJ566ilt3rxZ6enpWrRokUpKShq0/H/9139p5cqVki6ebNesWVOrvmzZMsXExHisXzQPT59XLoVjqH4IbB8xY8YMPfPMM3WuiouKijRr1ixNnDhRCQkJev755121a665RpWVlXVeP/roo5KkSZMmaezYsSorK1NycrKWLFmipKQkTZgwQZI0b948jR8/XgkJCZo+fbq+++47L+wpPOm6665Tq1atdPz4cd11111KSEjQuHHj9MEHH0iSqqqqNGvWLI0ePVpjxozR7NmzJdUeRVm6dKnKy8s1duxYTZo0SZI0efJk/eMf/9DJkyd1880313rzN2vWLG3cuFGStGPHDk2aNEnjx49XYmKi9u/f783dx09w57ySk5OjhIQEJSQk6LHHHtOtt+ZhV4AAAAj1SURBVN7quipPS0vThAkTNGbMGN111106ceKEJI6hejNgerfeeqvx5ZdfGjNnzjTWrl1ba9rdd99tfPLJJ4ZhGEZ1dbXx61//2vjwww8NwzCM3r17GxUVFa71/PD1f9YWLlxojBs3zqisrHRNKy4udv35iSeeMFasWGEYhmF8/PHHxrhx45pob9FY3x8bhmEYu3btMvr162eMHDnS2LBhg2EYhnHo0CFjwIABRnFxsbFt2zZjypQprmVLS0sNw6j9//jYsWPGgAEDam3jN7/5jfH+++8bhmEYd911l7F9+3bDMAyjpKTEGDBggFFZWWkcPXrUuP32243y8nLDMAzjq6++MmJjY5tux9Eg7pxXqqurjcGDBxt79uwxDMMwtm3bZvTu3dt1vP3wnLFhwwZjzpw5hmFwDNWXaZ/WhbrmzJmjO++8UxMnTpQkOZ1OffLJJ7WGOysrK5WXl6ebb765wesfNWqUgoKCXK83bdqkzMxMnT9/XmfPnlX37t0bvQ/wjlmzZikgIEDBwcHKyMjQrFmzXCMnvXr1UnR0tPbv369rr71WeXl5evTRRzVgwADdcsstDd7WuHHjtHHjRg0bNkxZWVkaOnSogoKCtHPnTn3zzTe64447XPNeuHBB3377rdq1a+epXUUjNeS8Eh4erpYtW7qGskeMGKHWrVu75vvggw/0+uuv6+zZsw36aI1j6CIC24dERUUpNjbW9Yxwi8Uii8WiN998Uy1atKgzv81mk/H/b3RXXV39k+v/YVjn5OTojTfe0Pr16xUWFqbMzExt2LDBQ3uCpvbUU0+pd+/ekqSKiopLztelSxdlZWXp448/1gcffKA//vGPyszMbNC2Ro4cqeXLl+vMmTPauHGjFi9e7KoNHjxY6enp7u0EvKIh55V//etfl1zPiRMntHz5cr355pvq0qWL9u7dq4cffrhePXAMXcRn2D5m5syZev3111VZWSmLxaL+/ftr9erVrnpBQYFOnz4tSeratas+//xzSapzEm7VqtVlT+RlZWUKDg5WaGioampq9NZbbzXB3sAbgoODFR0d7fpMMC8vT//617/Ut29fnTp1SjabTcOHD3d9Oa20tLTO8ufOnbvkFVNgYKCGDRumJ554QhUVFa6rr5tvvlk7d+7UoUOHXPMeOHCgifYSjVHf80pUVJSqqqr06aefSpK2b9+usrIySRffGLZo0UIRERFyOp1av369a3mOofrhCtvHdOjQQWPHjtXLL78sScrIyNDy5cuVkJAg6WIQL1u2TBEREVq0aJFSUlIUEhKiUaNG1VrPlClTdOedd6ply5Zat25dne0MHjxYmzdvVlxcnNq2bauYmBhX+MN8MjIylJKSorVr18rPz0/p6ekKCwvTjh07XN8Gdzqduv/++xUZGakjR464lg0NDXV9yahNmza1TsTfGzdunO644w7Xl9YkqXv37lqxYoWWLFmic+fO6fz587rxxhvVp0+fJt9fNExDzisrV67UI488IkkaMGCAwsPDFRISIrvdrlGjRmn06NFq27atYmNjXT/b4hiqHx7+AQDwmIqKCgUHB0uSPv74Yy1atEh///vfZbUyoNtYXGEDADxm27ZtWrt2rQzDkL+/vzIyMghrD+EKGwAAE+BtDwAAJkBgAwBgAgQ2AAAmQGADaLCpU6e6frcNwDv40hmAy3r66ad19OhRZWRkNHcrSk5OVmRkpObOndvcrQBexxU2AAAmQGADPmb16tUaPHiw+vXrp7i4OO3atUtOp1OrV6/W8OHDNXDgQM2ePdt1i9Hjx4/rmmuu0caNG3XLLbdo4MCB+tOf/iTp4sMaXnjhBb377rvq16+fxowZI+niow//+te/SpL+9re/adKkSXr88ccVExOjYcOGae/evfrb3/6m2NhY3XTTTbWGz2tqapSWlqZbbrlF//3f/62UlBSdO3dO0sXHdg4ZMkQvv/yybrrpJg0aNMh129u//OUvyszM1EsvvaR+/fpp2rRpXvs7Ba4EBDbgQ77++mu99tprevPNN7Vv3z699NJL6tSpk9atW6ft27frz3/+s3bu3Kk2bdpo6dKltZb99NNPtXXrVr366qt69tlnlZeXpyFDhuiBBx7QL37xC+3bt0+bN2/+0e0eOHBA11xzjXbv3q34+Hg99NBD+vzzz/Xee+9pxYoVWrp0qevZ6xkZGcrPz9fbb7+tbdu2qaioSM8++6xrXd9++63Ky8v1wQcfaNmyZVq6dKm+++47JSYmKiEhQffee6/27dtX6xnMwNWAwAZ8iM1mU01NjfLy8nT+/Hl17txZXbt21fr16zV37lx16NBB/v7+mjFjhrKzs2s9bGHGjBlq2bKlrr32Wl177bWXffLSf+rcubMmTJggm82m0aNHq6CgQNOnT5e/v78GDRokf39/ffPNNzIMQxs2bNDixYsVGhqq4OBgPfDAA9qyZYtrXX5+fpo+fbpatGih2NhYBQUFKT8/36N/T4AZcWtSwId069ZNixcv1tNPP63Dhw9r0KBBSk5O1smTJzV9+vRat4i0Wq0qLi52vf7h84MDAwN19uzZem83PDzc9eeWLVvWWV9AQIAqKytVUlKiqqoqjR8/3lUzDENOp9P1OjQ0VH5+/z41NbQXwFcR2ICP+f6pRxUVFUpJSVFGRoY6dOigxx9/XP37968z//Hjxy+7PovF4rHe2rZtq5YtW2rLli2KjIxs8PKe7AUwG4bEAR/y9ddfa9euXaqpqZG/v78CAgJktVr161//WqtWrdKJEyckSSUlJdq+fXu91hkeHq4TJ07Uugp2l9Vq1W233abHH3/cdXVfWFionTt31ruXn3qDAfgqAhvwITU1NVq5cqUGDhyoQYMGqaSkRA899JDuvPNODR06VFOmTFG/fv10++2368CBA/Va5/fPSh84cKDGjRvX6B7nz5+vbt266fbbb9eNN96ou+++u96fUU+cOFGHDx9WTEyMHnzwwUb3ApgJN04BAMAEuMIGAMAECGwAAEyAwAYAwAQIbAAATIDABgDABAhsAABMgMAGAMAECGwAAEyAwAYAwAT+H5f9kpjHFaQuAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 504x360 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XfoGbem_JYS6",
        "colab_type": "text"
      },
      "source": [
        "We observe that we are in a situation of imbalanced data.\n",
        "Indeed, there is way more negative tweets (61%) than positive or neutral ones (respectively 16% and 23%)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "huqRcjwgzx2k",
        "colab_type": "code",
        "outputId": "1c8a8883-d9bf-4a2b-cdb9-080ca64d3865",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 355
        }
      },
      "source": [
        "# distribution of sentiment confidence \n",
        "fig, ax = plt.subplots(1, figsize=(7,5))\n",
        "data['sentiment:confidence'].hist(bins=10, alpha=0.8, ax=ax)\n",
        "ax.set_xlabel('sentiment:confidence')\n",
        "ax.set_ylabel('Counts')\n",
        "plt.title('distribution of sentiment:confidence ')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcsAAAFSCAYAAACda/TMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de1hUZeIH8C/DTU1bBBEBzdItJdNER/ECIqCBxkCKF3KxzHXzWqZ5W00gXC0VL6tl6pNrue1mmqjhDdfM1FJLE83ETDQhZwAFLUFkZpj394c/TqDIO8DMMOT38zw+z8y8M+d85+Xgl3Nm5oyDEEKAiIiI7ktV1wGIiIjsHcuSiIhIgmVJREQkwbIkIiKSYFkSERFJsCyJiIgkWJZkNbNmzcKyZcsAAMePH0d4eLjFlj1mzBhs3boVAJCSkoLnn3/eYsv+7LPPMHr0aIstz1wnTpzAM888A39/f+zbt8/m6y9Tfm4fBEII/P3vf0e3bt0wZMgQ6bZafrumB4dTXQegB4NarUZaWpr0fitXrsTly5eRnJxc5f3ef/99i+T65ZdfEBYWhh9++AFOTnd+HaKiohAVFWWR5VfHihUr8Je//AUvvviizdZZ2Xxbam6ra9asWfDy8sKUKVNsut4TJ07gq6++wpdffolGjRoBgFnbKj1YuGdJ9YoQAiaTqa5jWIVWq8Xjjz9e1zEeOFeuXIGvr69SlESVYVmSxZw9exaDBg2Cv78/XnvtNZSUlChjx44dQ58+fZTra9euRVBQEPz9/REeHo4jR47g4MGDWLNmDXbv3g1/f39l727kyJFYtmwZYmNj8fTTTyM7OxsjR47E5s2bleUJIZCUlISuXbsiIiICR44cUcZCQ0Px9ddfK9dXrlyJadOmAQDi4uIAAN26dYO/vz9Onjx5z2Hd7777DjExMejatStiYmLw3XffKWMjR47E8uXLERsbC39/f4wePRoFBQX3naNNmzahf//+6N69O8aNG4fc3FwAQL9+/ZCdnY1x48bB398fer3+nsdWNmcAYDKZsHbtWvTr1w8BAQGYPHkybty4AeDOnnO7du2wdetW9O3bFwEBAXjvvfcAoMr5LpvblJQUxMbGYsGCBVCr1QgLC8N3332HlJQUBAcHo2fPnhUO2er1eixcuBB9+/ZFr169EB8fj9u3b1fYBv71r3+hZ8+eCAwMxJYtWwAAn3zyCVJTU7Fu3Tr4+/tj3Lhxlc6fTqfDpEmT0KNHDwQEBCApKUmZg1WrViEkJAQ9e/bEjBkzcPPmTekcbN68GW+88QbS09Ph7++PFStW3LOtVrVdA8AXX3yB6OhoqNVqxMbG4ty5c8pYaGgo1q1bB41Gg65du97z+H379iE6OhpdunRBv379cPDgQQDAzZs3MXv2bAQGBiIoKAjLli1DaWlppXNCNiKILKCkpET07dtXrF+/Xuj1erF7927x5JNPiqVLlwohhDh69KgICgoSQgiRmZkp+vTpI3JycoQQQmRnZ4vLly8LIYRYsWKFeP311yssOy4uTgQHB4vz588Lg8Eg9Hq9iIuLE5s2bRJCCLFlyxbh5+enrHvnzp2iS5cu4vr160IIIUJCQsRXX32lLK/8OrKzs8UTTzwhDAaDMr5lyxYRGxsrhBDi+vXrQq1Wi61btwqDwSBSU1OFWq0WBQUFSrawsDBx8eJFUVxcLOLi4sTixYsrnaOvv/5adO/eXZw5c0aUlJSIpKQkMWLECGX87pzlVTVnH3zwgRg6dKjQ6XSipKREzJ07V0yZMqXC85szZ44oLi4WGRkZokOHDuLChQtVzvfdc/vpp58Ko9Eoli5dKoKDg0ViYqIoKSkRhw4dEp07dxaFhYVCCCHmz58vxo4dK65fvy5u3rwpxo4dK5KTk5VtwM/PTyxfvlzo9Xpx4MAB0alTJ3Hjxg0hhBAzZ85UtpcyCQkJIiEhQQghhNFoFBqNRsyfP18UFRWJ27dvi2+//VYIIcTmzZtFv379RFZWligsLBQTJ04U06ZNM2sOyv+8y3KWbauy7fqHH34QPXr0EOnp6cJoNIqUlBQREhIiSkpKlJ9pTEyMyMnJEdevXxcRERHiv//9rxBCiFOnTokuXbqIw4cPi9LSUpGTk6NkmjBhgpg7d64oKioS165dEzExMeLjjz+udNsg2+CeJVnEqVOnYDAY8OKLL8LZ2RkRERHo2LFjpfd1dHSEXq9HZmYmDAYDWrZsiUceeaTK5Q8aNAiPP/44nJyc4OzsfM+4u7u7su6BAwfisccew4EDB2r9vA4cOIDWrVvjueeeg5OTEyIjI9GmTRt88cUXyn0GDx6Mxx57DA0aNEBERAQyMjIqXVZqaipiYmLQoUMHuLi4YOrUqUhPT8cvv/wizVHVnG3cuBFTpkxBixYt4OLigkmTJiEtLQ1Go1F5/KRJk9CgQQO0b98e7du3r7D3I9OyZUvExMTA0dERAwcOhE6nw8SJE+Hi4oLAwEC4uLggKysLQghs2rQJs2fPhpubGxo3boyxY8di586dyrKcnJwwceJEODs7Izg4GI0aNcKlS5fuu+7ExEQkJiYCAE6fPo28vDzMmDEDjRo1gqurK9RqtTK3o0aNQqtWrfDQQw9h6tSp2LVrV63nQLZdf/LJJxg+fDiefvppODo6YtCgQXB2dkZ6erpyn5EjR8LLywtubm4ICQlRto9PP/0UMTEx6N27N1QqFby8vNC2bVtcu3YNX375JWbPno1GjRrBw8MDo0aNqjCPZHt8gw9ZRF5eHry8vODg4KDc5uPjU+l9W7dujdmzZ2PlypW4cOECAgMDlTd33I+3t3eV669s3Xl5edV8FvfKy8u753n4+Pgoh08BwNPTU7ncsGFD3Lp1677L6tChg3L9oYcegpubG3Jzc9GyZcsqc1Q1Z1qtFhMnToRK9fvfviqVCvn5+cr1Zs2amZWxMh4eHsrlBg0a3LM8V1dXFBUVoaCgAMXFxRg8eLAyJu56jdnNzU15I1V1s+h0Ovj4+FR4fJm8vDz4+voq1319fWE0Gms9B7LtWqvVYtu2bfjoo4+U2wwGQ4Vt7+7to2xMp9MhODj4nnVqtVoYjUYEBgYqt5lMJunvAFkXy5IswtPTE7m5uRBCKP+xaLVatGrVqtL7azQaaDQaFBYWIj4+HsnJyVi8eHGF/5TKu9/tZe5et06nQ2hoKIA7/0EVFxcr97169arZy23evDm0Wm2F23Q6HYKCgqp83P2WdeXKFeX6rVu3cOPGjSr/SCjvfnPWokULLFiwAF27dr3nMbK9Vtnzr46mTZuiQYMG2Llzp9nPqTpZvL29odPpYDQa7ynMu+dWq9XCyckJHh4eyMnJqXaWMrLt2tvbG+PGjcP48eOrvWxvb29kZWXdc3vZEYKjR49W+ocB1Q0ehiWL6Ny5M5ycnLBhwwYYDAbs3bsX33//faX3vXjxIo4cOQK9Xg8XFxe4uroqe0UeHh64cuVKtd/xWlBQoKx79+7dyMzMVP5qb9++PXbt2gWDwYDvv/++wscC3N3doVKpkJ2dXelyg4OD8fPPPyM1NRVGoxG7du3ChQsX0Ldv32rlA4DIyEikpKQgIyMDer0eS5cuRadOnaR7lUDVc/b8889j+fLlSlkUFBSY/TnNms53ZVQqFYYOHYoFCxYoe3S5ubk4dOiQ2VmqKvdOnTrB09MTS5Yswa1bt1BSUoITJ04AuDO3H374IbKzs1FUVIRly5ZhwIABtS4b2XY9dOhQbNy4EadOnYIQArdu3cKBAwdQWFgoXfaQIUOQkpKCI0eOwGQyITc3F5mZmWjevDl69+6Nt99+G4WFhTCZTMjKysI333xTq+dCtcOyJItwcXHBypUrsXXrVnTv3h27du1C//79K72vXq/HkiVLEBAQgMDAQBQUFGDq1KkAgIiICABAQEAABg0aZPb6O3XqhMuXL6NHjx5Yvnw5VqxYgaZNmwIAXnvtNWRlZaF79+5YuXIlNBqN8riGDRti3LhxeP7556FWqyu81gTc2VtavXo11q9fj4CAALz//vtYvXo13N3dqzU/ANCrVy9MnjwZr7zyCgIDA5GdnW32h9urmrMXXngBoaGhGD16NPz9/TFs2DCcPn3arOXWdL7vZ/r06WjdujWGDRuGLl26YNSoUVW+JlnekCFDcOHCBajVakyYMAEAEB8fj/j4eAB3XrddvXo1Ll++jJCQEPTp0we7d+8GAMTExCAqKgpxcXEICwuDi4sL5s6dW+vnI9uuO3bsiHnz5iEpKQndunXDM888g5SUFLOW3alTJ7z11lvKUYG4uDjlKMaiRYtgMBgwcOBAdOvWDa+++mqFIyJkew5C8MufiYiIqsI9SyIiIgmWJRERkQTLkoiISIJlSUREJMGyJCIikmBZEhERSTzQp4e4fr0IJpN1Pjnj4dEY+fnyDybbg/qUFahfeetTVoB5rak+ZQUevLwqlQOaNn3ovuMPdFmaTMJqZVm2/PqiPmUF6lfe+pQVYF5rqk9ZAeYtj4dhiYiIJFiWREREEixLIiIiCZYlERGRBMuSiIhIgmVJREQkwbIkIiKSYFkSERFJsCyJiIgkHugz+BARUc04uzrBVNchyikqNlh1+SxLIiKqNhOA+DVH6jqGYv743lZdPg/DEhERSbAsiYiIJFiWREREEixLIiIiCZYlERGRBMuSiIhIgmVJREQkwbIkIiKSYFkSERFJsCyJiIgkWJZEREQSLEsiIiIJliUREZGEzb51JDQ0FC4uLnB1dQUATJs2DUFBQUhPT0d8fDxKSkrg6+uLxYsXw8PDAwBqPEZERGRJNt2zXLFiBbZv347t27cjKCgIJpMJ06dPR3x8PNLS0qBWq5GcnAwANR4jIiKytDo9DHvmzBm4urpCrVYDAGJjY7Fnz55ajREREVmaTb/8edq0aRBCoGvXrpg6dSp0Oh18fHyUcXd3d5hMJty4caPGY25ubrZ8SkRE9ACwWVn+5z//gbe3N/R6PebPn4+kpCT079/fVquvlIdHY6su39OziVWXb0n1KStQv/LWp6wA81pTfcoKVJ3318ISODvZ13tErTm/NitLb29vAICLiwtGjBiB8ePH44UXXoBWq1XuU1BQAJVKBTc3N3h7e9dorDry8wthMolaPrPKeXo2wdWrN62ybEurT1mB+pW3PmUFmNea6lNWQJ7X0dUJBqPJhonkajO/KpVDlTtQNvmz4NatW7h5886TEEJg165d8PPzw1NPPYXbt2/j+PHjAICNGzciIiICAGo8RkREZGk22bPMz8/HK6+8gtLSUphMJrRt2xYJCQlQqVRYtGgREhISKnwEBECNx4iIiCzNJmXZqlUrbNu2rdKxLl26IDU11aJjRERElmRfr84SERHZIZYlERGRBMuSiIhIgmVJREQkwbIkIiKSYFkSERFJsCyJiIgkWJZEREQSLEsiIiIJliUREZEEy5KIiEiCZUlERCTBsiQiIpKw2Zc/E5F9cnZ1QtlX+P5aWAJH17r9b0EFwFBirNMMRHdjWRI94EwA4tccAQA4O6lgMJqqfoCVJY3tWafrJ6oMD8MSERFJsCyJiIgkWJZEREQSLEsiIiIJliUREZEEy5KIiEiCZUlERCTBsiQiIpJgWRIREUmwLImIiCRYlkRERBIsSyIiIgmWJRERkQTLkoiISIJlSUREJMGyJCIikmBZEhERSbAsiYiIJFiWREREEixLIiIiCZYlERGRBMuSiIhIgmVJREQkwbIkIiKSsHlZvvPOO2jXrh3Onz8PAEhPT0dUVBTCw8MxevRo5OfnK/et6RgREZEl2bQsf/jhB6Snp8PX1xcAYDKZMH36dMTHxyMtLQ1qtRrJycm1GiMiIrI0m5WlXq9HUlISEhMTldvOnDkDV1dXqNVqAEBsbCz27NlTqzEiIiJLs1lZ/vOf/0RUVBRatmyp3KbT6eDj46Ncd3d3h8lkwo0bN2o8RkREZGlOtljJyZMncebMGUybNs0WqzObh0djqy7f07OJVZdvSfUpK1C/8tp71l8LS+Ds9PvfzeUv1wVHlQru1Zgze5/f8upTVqDqvHdvN/bAmvNrk7L89ttvkZmZibCwMABATk4O/vrXv2LkyJHQarXK/QoKCqBSqeDm5gZvb+8ajVVHfn4hTCZRy2dXOU/PJrh69aZVlm1p9SkrUL/y1oesjq5OMBhNAO4UZdnlulJqMpk9Z/VhfsvUp6yAPG/57cZe1GZ+VSqHKnegbPJnwcsvv4zDhw9j//792L9/P1q0aIF169ZhzJgxuH37No4fPw4A2LhxIyIiIgAATz31VI3GiIiILM0me5b3o1KpsGjRIiQkJKCkpAS+vr5YvHhxrcaIiIgsrU7Kcv/+/crlLl26IDU1tdL71XSMiIjIkuzr1VkiIiI7xLIkIiKSYFkSERFJsCyJiIgkWJZEREQSLEsiIiIJliUREZEEy5KIiEiCZUlERCTBsiQiIpJgWRIREUmwLImIiCRYlkRERBIsSyIiIgmWJRERkQTLkoiISIJlSUREJMGyJCIikmBZEhERSbAsiYiIJFiWREREEixLIiIiCZYlERGRBMuSiIhIgmVJREQkwbIkIiKSYFkSERFJsCyJiIgkWJZEREQSLEsiIiIJliUREZEEy5KIiEiCZUlERCRR47I8evQovvnmG0tmISIisktml2VcXBxOnDgBAFi7di2mTp2K119/HatXr7ZaOCIiIntgdln+9NNP6Ny5MwBg8+bN2LBhAzZt2oSNGzdaLRwREZE9cDL3jiaTCQ4ODsjKyoIQAn/+858BAL/++qvVwhEREdkDs8uya9euSEpKwtWrV9G/f38AQFZWFpo2bWq1cERERPbA7MOwb731Fh5++GG0a9cOr7zyCgDg4sWLeOGFF6wWjoiIyB6YvWd59OhRTJ06tcJtffv2xZ49eyweioiIyJ6YvWc5Z86cSm+Pj4836/ETJkxAVFQUnnvuOYwYMQIZGRkAgEuXLmH48OEIDw/H8OHD8fPPPyuPqekYERGRJUnLMjs7G9nZ2RBCKJfL/n399ddwcXExa0ULFy7EZ599hm3btmH06NGYPXs2ACAhIQEjRoxAWloaRowYUaF8azpGRERkSdLDsP3794eDgwOEEMobe8o0a9ZMef1SpkmTJsrlwsJCODg4ID8/H2fPnsX69esBAJGRkZg3bx4KCgoghKjRmLu7u3nPnIiIyEzSsjx37hyAOycl+Oijj2q1sjlz5uCrr76CEALvv/8+dDodvLy84OjoCABwdHRE8+bNodPpIISo0Vh1ytLDo3Gtno+Mp2cT+Z3sRH3KCtSvvPae9dfCEjg7/X6QqfzluuCoUsG9GnNm7/NbXn3KClSd9+7txh5Yc37NfoNPbYsSAObPnw8A2LZtGxYtWoTJkyfXepm1kZ9fCJNJWGXZnp5NcPXqTass29LqU1agfuWtD1kdXZ1gMJoA3CnKsst1pdRkMnvO6sP8lqlPWQF53vLbjb2ozfyqVA5V7kCZXZbZ2dlYvnw5MjIycOvWrQpjBw4cqFao5557DvHx8WjRogVyc3NRWloKR0dHlJaWIi8vD97e3hBC1GiMiIjI0swuy2nTpqFVq1aYOXMmGjZsWK2VFBUV4bffflPKbP/+/fjTn/4EDw8P+Pn5YceOHYiOjsaOHTvg5+enHEqt6RgREZElmV2WP/30Ez7++GOoVNU/Rl1cXIzJkyejuLgYKpUKf/rTn7B69Wo4ODggMTERs2bNwqpVq/Dwww9j4cKFyuNqOkZERGRJZpdlt27dcPbsWTz11FPVXkmzZs2wadOmSsfatm2LzZs3W3SMiIjIkswuS19fX4wZMwb9+/dHs2bNKozV9Rt1iIiIrMnssiwuLkZISAiMRiNycnKsmYmIiMiumF2Wb731ljVzEBER2a1qfXTkflq1amWRMERERPbI7LIsf9q7Mg4ODgCgnBSdiIjoj8jssiw77V2Zq1ev4p133oFarbZ4KCIiIntS4xP7eXp6Ys6cOVi6dKkl8xAREdmdWp0F9+LFiyguLrZUFiIiIrtk9mHYESNGKK9RAnc+SnLhwgVMnDjRKsGIiIjshdllOXTo0ArXGzZsiPbt2+PRRx+1dCYiIiK7YnZZDho0yJo5iIiI7JbZr1kaDAasWLECYWFh6NixI8LCwrBixQro9Xpr5iMiIqpzZu9ZLl68GKdPn8abb74JHx8faLVarFq1CoWFhZg9e7Y1MxIREdUps8tyz5492L59O5o2bQoAaNOmDZ588klER0ezLImI6A/N7MOw5c/cY87tREREfxRml2VERATGjx+PQ4cOITMzEwcPHsTEiRMRERFhzXxERER1zuzDsNOnT8d7772HpKQk5OXlwcvLC88++yzGjx9vzXxERER1TrpneeLECSxevBguLi6YPHky/ve//+HUqVPYu3cv9Ho9zp49a4ucREREdUZalmvWrEG3bt0qHQsICMDq1astHoqIiMieSMsyIyMDQUFBlY716tULZ86csXgoIiIieyIty8LCQhgMhkrHjEYjioqKLB6KiIjInkjLsk2bNjh8+HClY4cPH0abNm0sHoqIiMieSMty1KhRSEhIwN69e2EymQAAJpMJe/fuRWJiIl566SWrhyQiIqpL0o+OaDQaXLt2DTNnzoTBYICbmxtu3LgBZ2dnvPrqq4iMjLRFTiIiojpj1ucsX3rpJQwdOhQnT57EjRs34ObmBn9/fzRu3Nja+YiIiOqc2SclaNy48X3fFUtERPRHZvbp7oiIiB5ULEsiIiIJliUREZEEy5KIiEiCZUlERCTBsiQiIpIw+6MjRES2oFKpAFfz/mv6tbAEjmbet0ZZABhKjFZbPtUfLEsisivCJBC/5ohZ93V2UsFgNFktS9LYnlZbNtUvPAxLREQkwbIkIiKSYFkSERFJsCyJiIgkWJZEREQSNinL69ev429/+xvCw8Oh0WgwadIkFBQUAADS09MRFRWF8PBwjB49Gvn5+crjajpGRERkSTYpSwcHB4wZMwZpaWlITU1Fq1atkJycDJPJhOnTpyM+Ph5paWlQq9VITk4GgBqPERERWZpNytLNzQ0BAQHK9c6dO0Or1eLMmTNwdXWFWq0GAMTGxmLPnj0AUOMxIiIiS7P5SQlMJhM+/vhjhIaGQqfTwcfHRxlzd3eHyWTCjRs3ajzm5uZmdhYPj8aWeVL34enZxKrLt6T6lBWoX3ntPeuvhSVwdvr97+byl+uEQ/UyWDOvo0oFdwv+/Ox9W7hbVXnv3m7sgTXn1+ZlOW/ePDRq1AhxcXH43//+Z+vVV5CfXwiTSVhl2Z6eTXD16k2rLNvS6lNWoH7lrQ9ZHV2dlLPgWPuMOGYRMDuDtfOWmkwW+/nVh22hPFne8tuNvajN/KpUDlXuQNm0LBcuXIjLly9j9erVUKlU8Pb2hlarVcYLCgqgUqng5uZW4zEiIiJLs9k+9NKlS3HmzBm8++67cHFxAQA89dRTuH37No4fPw4A2LhxIyIiImo1RkREZGk22bP86aefsGbNGjz66KOIjY0FALRs2RLvvvsuFi1ahISEBJSUlMDX1xeLFy8GcOebB2oyRkREZGk2KcvHH38cP/74Y6VjXbp0QWpqqkXHiIiILMm+3spERERkh1iWREREEixLIiIiCZYlERGRBMuSiIhIgmVJREQkwbIkIiKSYFkSERFJsCyJiIgkWJZEREQSLEsiIiIJliUREZEEy5KIiEiCZUlERCTBsiQiIpJgWRIREUmwLImIiCRYlkRERBIsSyIiIgmWJRERkQTLkoiISIJlSUREJMGyJCIikmBZEhERSbAsiYiIJFiWREREEixLIiIiCZYlERGRBMuSiIhIgmVJREQkwbIkIiKSYFkSERFJsCyJiIgkWJZEREQSLEsiIiIJliUREZEEy5KIiEiCZUlERCTBsiQiIpKwSVkuXLgQoaGhaNeuHc6fP6/cfunSJQwfPhzh4eEYPnw4fv7551qPERERWZpNyjIsLAz/+c9/4OvrW+H2hIQEjBgxAmlpaRgxYgTi4+NrPUZERGRpNilLtVoNb2/vCrfl5+fj7NmziIyMBABERkbi7NmzKCgoqPEYERGRNTjV1Yp1Oh28vLzg6OgIAHB0dETz5s2h0+kghKjRmLu7e7UyeHg0tuyTuounZxOrLt+S6lNWoH7ltfesvxaWwNnp97+by1+uEw7Vy2DNvI4qFdwt+POz923hblXlvXu7sQfWnN86K0t7kJ9fCJNJWGXZnp5NcPXqTass29LqU1agfuWtD1kdXZ1gMJoA3Cmesst1RsDsDNbOW2oyWeznVx+2hfJkectvN/aiNvOrUjlUuQNVZ2Xp7e2N3NxclJaWwtHREaWlpcjLy4O3tzeEEDUaIyIisoY624f28PCAn58fduzYAQDYsWMH/Pz84O7uXuOxB52zqxMca/Dv18KSGj2uqn/Org/0QQsi+oOxyf9o//jHP7B3715cu3YNL730Etzc3LBz504kJiZi1qxZWLVqFR5++GEsXLhQeUxNxx5kJgDxa45U+3HWOJSVNLanRZdHRFSXbFKWb7zxBt544417bm/bti02b95c6WNqOkZERGRp9vVWJiIiIjvEsiQiIpJgWRIREUmwLImIiCRYlkRERBIsSyIiIgmWJRERkQTLkoiISILnJCOrUKlUgJVOeVd2er5q5QFgKDFaJQ8R/fGxLMkqhEnU6NR75qjJ6fl4+j0iqg0ehiUiIpJgWRIREUmwLImIiCRYlkRERBIsSyIiIgmWJRERkQTLkoiISIJlSUREJMGyJCIikmBZEhERSbAsiYiIJFiWREREEixLIiIiCZYlERGRBMuSiIhIgmVJREQkwbIkIiKSYFkSERFJsCyJiIgkWJZEREQSLEsiIiIJliUREZEEy5KIiEiCZUlERCTBsiQiIpJgWRIREUmwLImIiCRYlkRERBIsSyIiIol6XZaXLl3C8OHDER4ejuHDh+Pnn3+u60hERPQHVK/LMiEhASNGjEBaWhpGjBiB+Pj4uo5ERER/QE51HaCm8vPzcfbsWaxfvx4AEBkZiXnz5qGgoADu7u5mLUOlcrBmRKsv/571OTjA/eEG1X6ck5MKRqPJLrKYoyZ5VQ4OEH0w6ggAAA+5SURBVDb+eSjrrqP1mqv8z8oa20Jt8shYO6+ltxt73xbuVlVea/6O14SDQ+3mV/ZYByGEqPHS69CZM2cwc+ZM7Ny5U7lt4MCBWLx4MTp06FCHyYiI6I+mXh+GJSIisoV6W5be3t7Izc1FaWkpAKC0tBR5eXnw9vau42RERPRHU2/L0sPDA35+ftixYwcAYMeOHfDz8zP79UoiIiJz1dvXLAEgMzMTs2bNwm+//YaHH34YCxcuRJs2beo6FhER/cHU67IkIiKyhXp7GJaIiMhWWJZEREQSLEsiIiIJliUREZEEy7IWzDmR+7vvvotnn30WGo0GgwcPxqFDh2wfFOZl3bJlCzQaDaKjo6HRaLBhwwbbB/1/1TlJ/sWLF/H0009j4cKFtgt4F3Pyrly5Ej179kR0dDSio6Px5ptv2j4ozJ/bXbt2QaPRIDIyEhqNBteuXbNt0P9nTt4ZM2Yo8xodHY327dvj888/t8us+fn5ePnll6HRaDBgwAAkJibCaDTaPCtgXt6rV69i/PjxSt7t27fbPiiAhQsXIjQ0FO3atcP58+crvU9paSnefPNN9OvXD/3798fmzZstF0BQjY0cOVJs27ZNCCHEtm3bxMiRI++5z8GDB8WtW7eEEEJkZGSIrl27iuLiYpvmFMK8rDdv3hQmk0m53LdvX5GRkWHTnGXMySuEEEajUcTFxYmpU6eKt99+25YRKzAn74oVK+o0Yxlzsp4+fVoMGDBA5OXlCSGE+O2338Tt27dtmrOMudtCmYyMDNG9e3dRUlJii3gVmJP1H//4h7Id6PV6MWTIELFz506b5ixjTt6pU6eKd955RwghRH5+vggODhZardamOYUQ4ttvvxVarVaEhISIH3/8sdL7bN26VYwePVqUlpaK/Px8ERQUJLKzsy2yfu5Z1lDZidwjIyMB3DmR+9mzZ1FQUFDhfkFBQWjYsCEAoF27dhBC4MaNG3aZtXHjxnBwuHMy4du3b8NgMCjX7TEvAKxduxZ9+/bFo48+auOUv6tO3rpmbtYPPvgAo0ePhqenJwCgSZMmcHV1tdu85X366afQaDRwcXGxVUwA5md1cHBAUVERTCYT9Ho9DAYDvLy8bJq1OnnPnTuHoKAgAIC7uzvat2+P3bt32zyvWq2WnqFt165dGDp0KFQqFdzd3dGvXz/s2bPHIutnWdaQTqeDl5cXHB0dAQCOjo5o3rw5dDrdfR+zbds2PPLII2jRooWtYgKoXtbPP/8czz77LEJCQjBmzBi0a9fOplkB8/OeO3cOhw8fxqhRo2yesbzqzO/OnTuh0WgwevRonDx50tZRzc6amZmJ7Oxs/OUvf8GgQYOwatUqiDr4SHZ1f8/0ej1SU1MRExNjy5gAzM86YcIEXLp0CYGBgcq/rl272m3eDh06YNeuXRBCIDs7GydPnoRWq7V5XnPodDr4+Pgo1729vZGTk2ORZbMsbeSbb77BP//5TyxZsqSuo1QpLCwMO3fuRFpaGrZv346LFy/WdaRKGQwGzJ07F2+++abyy27vYmNj8fnnnyM1NRV//etfMWHCBFy/fr2uY1WqtLQUP/74I9avX49///vfOHjwYJ29VlUd+/btg4+PD/z8/Oo6yn3t2bMH7dq1w+HDh3Hw4EEcP37cYns/1jBr1ixcu3YN0dHRmD9/Pnr27FlvfucsiWVZQ9U5kfvJkycxffp0vPvuu3VyOr6anHTex8cHHTt2xIEDB2yU8nfm5L169SqysrLw8ssvIzQ0FB9++CE2bdqEuXPn2mVeAPD09ISzszMAoHfv3vD29sZPP/1kl1l9fHwQEREBFxcXNG7cGGFhYTh9+rRNs1Ynb5ktW7bUyV4lYH7Wjz76CFFRUVCpVGjSpAlCQ0Nx7Ngxu83r7u6O5ORkfPbZZ1i9ejWKiorw5z//2eZ5zeHt7V1hr1en01nsSB7LsobMPZH76dOnMWXKFKxYsaLOvmfT3KyZmZnK5YKCAhw7dgxPPPGETbMC5uX18fHBsWPHsH//fuzfvx8vvvgihg0bhnnz5tllXgDIzc1VLmdkZODKlSt47LHH7DJrZGQkDh8+DCEEDAYDjh49ivbt29s0a3XyAkBOTg5OnDgBjUZj65gAzM/asmVLHDx4EMCdw8ZHjhzB448/brd5r1+/rrxb98iRIzh//rzyOqe9iYiIwObNm2EymVBQUIB9+/YhPDzcMgu3yNuEHlAXLlwQQ4YMEc8884wYMmSIyMzMFEIIMWbMGHH69GkhhBCDBw8WAQEBIioqSvl37tw5u8w6f/58MXDgQBEVFSU0Go3YsGGDzXNWJ295df1OU3PyzpgxQzz77LNCo9GIwYMHiwMHDtht1tLSUrFgwQIREREhBg4cKBYsWCBKS0vtNq8QQqxatUq89tprdZKxjDlZL1++LEaNGiUiIyPFgAEDRGJiojAYDHab98CBA6J///4iPDxcxMbGirNnz9ZJ1nnz5omgoCDh5+cnevXqJQYOHHhPVqPRKOLj40VYWJgICwsTGzdutNj6eSJ1IiIiCR6GJSIikmBZEhERSbAsiYiIJFiWREREEixLIiIiCZYlkQ2MGTMGW7duresYNiOEwN///nd069YNQ4YMwfHjx6v8vNusWbOwbNkyGyYkqh6nug5A9EezcuVKXL58GcnJycpt77//fp1kmTVrFry8vDBlyhSbrvfEiRP46quv8OWXX6JRo0YAgLS0NJtmILIk7lkSkcVduXIFvr6+SlES1XcsS3rgrV27FkFBQfD390d4eDiOHDkCk8mEtWvXol+/fggICMDkyZOVr1b75Zdf0K5dO2zduhV9+/ZFQEAA3nvvPQDAwYMHsWbNGuzevRv+/v6IiooCAIwcOVL5ItqUlBTExsZiwYIFUKvVCAsLw3fffYeUlBQEBwejZ8+eFQ7Z6vV6LFy4EH379kWvXr0QHx+P27dvAwCOHTuGPn364F//+hd69uyJwMBAbNmyBQDwySefIDU1FevWrYO/vz/GjRtX6fPX6XSYNGkSevTogYCAACQlJQEATCYTVq1ahZCQEPTs2RMzZszAzZs3pXOwefNmvPHGG0hPT4e/vz9WrFih5Cxz9uxZDBo0CP7+/njttddQUlJSIdMXX3yB6OhoqNVqxMbG4ty5c8pYaGgo1q1bB41Gg65du97z+H379iE6OhpdunRBv379lFPL3bx5E7Nnz0ZgYCCCgoKwbNky5byoRFIWOxcQUT2UmZkp+vTpI3JycoQQQmRnZ4vLly+LDz74QAwdOlTodDpRUlIi5s6dK6ZMmaLc54knnhBz5swRxcXFIiMjQ3To0EFcuHBBCHHn1Huvv/56hfXExcWJTZs2CSGE2LJli/Dz8xOffvqpMBqNYunSpSI4OFgkJiaKkpIScejQIdG5c2dRWFgohLhzGsKxY8eK69evi5s3b4qxY8eK5ORkIYQQR48eFX5+fmL58uVCr9eLAwcOiE6dOokbN24IIYSYOXOmWLp0aYUsCQkJIiEhQQhx5/RgGo1GzJ8/XxQVFYnbt2+Lb7/9VgghxObNm0W/fv1EVlaWKCwsFBMnThTTpk0zaw62bNkiYmNjlXUePXpUBAUFCSGEKCkpEX379hXr168Xer1e7N69Wzz55JNKzh9++EH06NFDpKenC6PRKFJSUkRISIjyZc4hISEiJiZG5OTkiOvXr4uIiAjx3//+VwghxKlTp0SXLl3E4cOHRWlpqcjJyVEyTZgwQcydO1cUFRWJa9euiZiYGPHxxx/XZLOhBxD3LOmB5ujoCL1ej8zMTBgMBrRs2RKPPPIINm7ciClTpqBFixZwcXHBpEmTkJaWppxQGgAmTZqEBg0aoH379mjfvn2FvR+Zli1bIiYmBo6Ojhg4cCB0Oh0mTpwIFxcXBAYGwsXFBVlZWRBCYNOmTZg9ezbc3NzQuHFjjB07Fjt37lSW5eTkhIkTJ8LZ2RnBwcFo1KgRLl26dN91JyYmIjExEcCdE/3n5eVhxowZaNSoEVxdXaFWqwEAqampGDVqFFq1aoWHHnoIU6dOxa5du2o9B6dOnYLBYMCLL74IZ2dnREREoGPHjsr4J598guHDh+Ppp5+Go6MjBg0aBGdnZ6Snpyv3GTlyJLy8vODm5oaQkBBkZGQAuPPFzzExMejduzdUKhW8vLzQtm1bXLt2DV9++SVmz56NRo0awcPDA6NGjaowj0RV4Rt86IHWunVrzJ49GytXrsSFCxcQGBiIWbNmQavVYuLEiVCpfv97UqVSIT8/X7nerFkz5XLDhg1x69Yts9fr4eGhXG7QoME9y3N1dUVRUREKCgpQXFyMwYMHK2NCCJhMJuW6m5sbnJx+/1WuTpayL8st//gyeXl58PX1Va77+vrCaDTWeg7y8vLg5eUFBwcH5bbyX9ir1Wqxbds2fPTRR8ptBoMBeXl5ynVPT88K6y0b0+l0CA4OvmedWq0WRqMRgYGBym0mk6nKr6kjKo9lSQ88jUYDjUaDwsJCxMfHIzk5GS1atMCCBQsq/Qb7X375pcrllS+B2mratCkaNGiAnTt3wsvLq9qPl2Xx9vaGTqeD0Wi8pzCbN2+OK1euKNe1Wi2cnJzg4eFRq2+f9/T0RG5uLoQQSj6tVotWrVopmcaNG4fx48dXe9ne3t7Iysq65/ayIwRHjx6t9A8DIhkehqUH2sWLF3HkyBHo9Xq4uLjA1dUVKpUKzz//PJYvX66URdl345nDw8MDV65cqbD3V1MqlQpDhw7FggULlD263NxcHDp0yOwsVZV7p06d4OnpiSVLluDWrVsoKSnBiRMnANz5TssPP/wQ2dnZKCoqwrJlyzBgwIBal03nzp3h5OSEDRs2wGAwYO/evfj++++V8aFDh2Ljxo04deoUhBC4desWDhw4gMLCQumyhwwZgpSUFOVNWrm5ucjMzETz5s3Ru3dvvP322ygsLITJZEJWVha++eabWj0XenCwLOmBptfrsWTJEgQEBCAwMBAFBQWYOnUqXnjhBYSGhmL06NHw9/fHsGHDcPr0abOWGRERAQAICAjAoEGDap1x+vTpaN26NYYNG4YuXbpg1KhRVb4mWd6QIUNw4cIFqNVqTJgwAQAQHx+P+Ph4AHdes129ejUuX76MkJAQ9OnTB7t37wYAxMTEICoqCnFxcQgLC4OLiwvmzp1b6+fj4uKClStXYuvWrejevTt27dqF/v37K+MdO3bEvHnzkJSUhG7duuGZZ55BSkqKWcvu1KkT3nrrLeWoQFxcHLRaLQBg0aJFMBgMGDhwILp164ZXX30VV69erfXzoQcDv8+SiIhIgnuWREREEixLIiIiCZYlERGRBMuSiIhIgmVJREQkwbIkIiKSYFkSERFJsCyJiIgkWJZEREQS/wdI6n0TKSmXXQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 504x360 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qKZYQ9ZKzx2n",
        "colab_type": "code",
        "outputId": "3f520658-5229-4a3a-fdd3-03230764921a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        }
      },
      "source": [
        "data['sentiment:confidence'].describe()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "count    13871.000000\n",
              "mean         0.756936\n",
              "std          0.217682\n",
              "min          0.186000\n",
              "25%          0.651700\n",
              "50%          0.681300\n",
              "75%          1.000000\n",
              "max          1.000000\n",
              "Name: sentiment:confidence, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p7-Be1Bjzx2q",
        "colab_type": "text"
      },
      "source": [
        "- We do not know excately how the sentiment confidence was computed \n",
        "- We decide to discard tweets with confidence < 0.6"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "agELT9Fzzx2r",
        "colab_type": "code",
        "outputId": "55e751b0-a649-4fbc-e5c6-5fd9687d54a0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# discard tweets with sentiment confidence <0.6\n",
        "data = data[data['sentiment:confidence']>=0.6]\n",
        "print('new data shape:', data.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "new data shape: (12209, 20)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O_qTsLe7zx2y",
        "colab_type": "code",
        "outputId": "017b21f7-6a5b-4dcb-936f-c3ff81c9c4e3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        }
      },
      "source": [
        "# some examples of tweets\n",
        "print('neutral tweet:', data.loc[0, 'text'])\n",
        "print('positive tweet:', data.loc[1, 'text'])\n",
        "print('negative tweet:', data.loc[2, 'text'])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "neutral tweet: RT @NancyLeeGrahn: How did everyone feel about the Climate Change question last night? Exactly. #GOPDebate\n",
            "positive tweet: RT @ScottWalker: Didn't catch the full #GOPdebate last night. Here are some of Scott's best lines in 90 seconds. #Walker16 http://t.co/ZSfFÛ_\n",
            "negative tweet: RT @TJMShow: No mention of Tamir Rice and the #GOPDebate was held in Cleveland? Wow.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EFnYjx7y1ktE",
        "colab_type": "text"
      },
      "source": [
        "<a id='Split'></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T3nSHJ-DJ6O4",
        "colab_type": "text"
      },
      "source": [
        "#### Split the data in train and test set \n",
        "\n",
        "- We use classicaly 75% of tweets for the train set \n",
        "- We fix the random_state to ensure reproductibility of results "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XQgtxAJwzx26",
        "colab_type": "code",
        "outputId": "9cd8ac75-0333-4b78-aecb-de0407f289e9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "# split in train and test \n",
        "X, y = data.text, data.sentiment\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
        "print('Train shape:', X_train.shape)\n",
        "print('Test shape:', X_test.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train shape: (9156,)\n",
            "Test shape: (3053,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MSEvyY7uzx3D",
        "colab_type": "code",
        "outputId": "745124ab-48cf-478b-de85-79f1ed6f056d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "y_train"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7076     Negative\n",
              "8294      Neutral\n",
              "2338     Negative\n",
              "1027     Negative\n",
              "13858    Positive\n",
              "           ...   \n",
              "5552     Negative\n",
              "3732     Positive\n",
              "11220    Negative\n",
              "12293    Positive\n",
              "3119     Negative\n",
              "Name: sentiment, Length: 9156, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9RtFyt2Yzx3F",
        "colab_type": "code",
        "outputId": "0748f080-6cb0-45d9-e010-7e7f5f13f045",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "y_test"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1823     Negative\n",
              "13442    Positive\n",
              "6815     Negative\n",
              "4579     Negative\n",
              "9472     Negative\n",
              "           ...   \n",
              "6968     Negative\n",
              "7179     Negative\n",
              "9106     Negative\n",
              "2295      Neutral\n",
              "12288    Negative\n",
              "Name: sentiment, Length: 3053, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xQYzMem1zx3J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# encode the labels\n",
        "le = LabelEncoder()\n",
        "le.fit(y_train)\n",
        "y_train = le.transform(y_train)\n",
        "y_test = le.transform(y_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gjnPXzD3zx3L",
        "colab_type": "code",
        "outputId": "338d6f7a-1eb7-49f3-88f4-0922f016bfd3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "y_test"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 2, 0, ..., 0, 1, 0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xN4jqF3-vj_M",
        "colab_type": "text"
      },
      "source": [
        "The encoding is done as follow:\n",
        "- 0 for negative tweets\n",
        "- 1 for neutral ones \n",
        "- 2 for positive ones"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dLh9jhrD1nJP",
        "colab_type": "text"
      },
      "source": [
        "<a id='Tokenisation'></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "12dumyc7zx3P",
        "colab_type": "text"
      },
      "source": [
        "### Tokenisation with Bert Tokenizer\n",
        "\n",
        "- <a href = 'https://huggingface.co/transformers/pretrained_models.html'> list of pretrained models  </a>\n",
        "- We used bert-base-uncased:  12-layer, 768-hidden, 12-heads, 110M parameters.\n",
        "Trained on lower-cased English text.\n",
        "\n",
        "    \n",
        "- Less number of tokens than with word to vec (vocabulary only contains around 30000 tokens)\n",
        "- This is because BERT works at the subwords level. Therefore there are less  tokens\n",
        "- BERTTokenizer does not remove ponctuation\n",
        "- We will add  [CLS] token at the beginning of each sentence (understand sequence as a tweet not a real sentence ending with a point) and [SEP] at the end\n",
        "- We will use some specific functions to preprocess tweets used in a Lab (remove html, url and hashtags)\n",
        "- need to convert to id each tokens \n",
        "- the maximum len of tweets is 167 ==> We will padd all sequences with inferior length with 0 "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ji-Syqr8zx3Q",
        "colab_type": "code",
        "outputId": "089a9a98-bc38-498c-a90e-0f897e4bac6a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 83,
          "referenced_widgets": [
            "d5cd36c3ae9e43369065310934386b95",
            "c1ff2e9989d841cabad13da1127b8c4a",
            "5e186898a315464db5b7d8db593fc035",
            "9af87d245d534122b856dcc121fdca1c",
            "e3d9d802fe114f7192fdc375213f70b7",
            "a11907d509bf4da387e9be255f403932",
            "550978fce2fd4a2ca35ff7d7514afd4f",
            "1551bfef00774669a565319e6cecc238"
          ]
        }
      },
      "source": [
        "# the maximum length BERT can handle is 512\n",
        "tokenizer =  BertTokenizer.from_pretrained('bert-base-uncased') \n",
        "tokens = list(map(lambda t: ['[CLS]'] + tokenizer.tokenize(t)[:510] + ['[SEP]'],\n",
        "                  data.text))\n",
        "maxlen = np.max([len(seq) for seq in tokens])\n",
        "print('max length:', maxlen)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d5cd36c3ae9e43369065310934386b95",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=231508, style=ProgressStyle(description_wid…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "max length: 167\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gHYBRno4zx3S",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MyTokenizer():\n",
        "    \"\"\"Class wich will use BERT tokenizer \n",
        "    \n",
        "    Parameters\n",
        "    ----------\n",
        "    tokenizer : BERT tokenizer from a given pretrained model\n",
        "\n",
        "    maxlen : maxlen of sequences tolerated by BERT.\n",
        "      We choose 167 as it is the maximum len of sequences.\n",
        "\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, maxlen=167, from_pretrained='bert-base-uncased'):\n",
        "        self.tokenizer = BertTokenizer.from_pretrained(from_pretrained) \n",
        "        self.maxlen = maxlen\n",
        "        \n",
        "\n",
        "    def tokenize(self, text):\n",
        "        '''\n",
        "        Parameters\n",
        "        ----------\n",
        "        text: series of tweets \n",
        "        \n",
        "        Returns\n",
        "        -------\n",
        "        tokens: the tokenized text ie array of shape (n, maxlen)\n",
        "          where n is the number of sequences/tweets \n",
        "\n",
        "        '''\n",
        "        tokens = list(map(lambda t: ['[CLS]'] + self.tokenizer.tokenize(t)[:maxlen-2] + ['[SEP]'],\n",
        "                          text))\n",
        "        tokens = [self.remove_html(seq) for seq in tokens]\n",
        "        tokens = [self.remove_hashtags(seq) for seq in tokens]\n",
        "        tokens = [self.remove_url(seq) for seq in tokens]\n",
        "        \n",
        "        # padding with zeros sequences with len <= maxlen\n",
        "        # convert to id tokens\n",
        "        tokens = pad_sequences(list(map(self.tokenizer.convert_tokens_to_ids,\n",
        "                                        tokens)),\n",
        "                               maxlen=self.maxlen,\n",
        "                               truncating=\"post\",\n",
        "                               padding=\"post\",\n",
        "                               dtype='int')\n",
        "        return tokens\n",
        "        \n",
        "        \n",
        "    # special preprocessing for tweets\n",
        "    \n",
        "    @staticmethod\n",
        "    def remove_html(tokens):\n",
        "        tokens = filter(lambda x: x[0]+x[-1] != '<>', tokens)\n",
        "        return list(tokens)\n",
        "    \n",
        "    @staticmethod\n",
        "    def remove_hashtags(tokens):\n",
        "        tokens = map(lambda x: x.replace('#', ''), tokens)\n",
        "        return list(tokens)\n",
        "    \n",
        "    @staticmethod\n",
        "    def remove_url(tokens):\n",
        "        tokens = filter(lambda x: \"http\" not in x, tokens)\n",
        "        return list(tokens)\n",
        "\n",
        "    \n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9N_F_S_Bzx3U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer = MyTokenizer(maxlen=maxlen)\n",
        "train_tokens  = tokenizer.tokenize(X_train)\n",
        "test_tokens = tokenizer.tokenize(X_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GcWAuY7Nzx3W",
        "colab_type": "code",
        "outputId": "10716e98-356d-4f79-ec1f-a3bebc11a143",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "train_tokens.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(9156, 167)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ma-1qKgTzx3Y",
        "colab_type": "code",
        "outputId": "0f30ea3b-a9ca-4f50-8ba0-c24de26894ed",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print('size of the vocabulary:', tokenizer.tokenizer.vocab_size)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "size of the vocabulary: 30522\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mAlvnUZazx3a",
        "colab_type": "text"
      },
      "source": [
        "##### Define torch tensor "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZeVSc_ylzx3b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_tokens_tensor = torch.tensor(train_tokens)\n",
        "test_tokens_tensor = torch.tensor(test_tokens)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EYCT9i4xLUDO",
        "colab_type": "text"
      },
      "source": [
        "#### Compute the attention mask. \n",
        "\n",
        "- 1 if the token id is !=0 and 0 otherwise "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BRVD_Aoq4shq",
        "colab_type": "code",
        "outputId": "cfa629d2-5432-43c4-e509-6188f3611099",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "# attention masks. shape (n, maxlen).\n",
        "train_masks_tensor = torch.tensor((train_tokens_tensor != 0).long())\n",
        "print('train mask shape:', train_masks_tensor.shape)\n",
        "\n",
        "test_masks_tensor = torch.tensor((test_tokens_tensor != 0).long())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "train mask shape: torch.Size([9156, 167])\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:1: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:4: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  after removing the cwd from sys.path.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fIyVCRFFzx3h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y_train_tensor = torch.tensor(y_train)\n",
        "y_test_tensor = torch.tensor(y_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xu5waLMR1qtw",
        "colab_type": "text"
      },
      "source": [
        "<a id='Fine_tune'></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VFbNOgbUzx3n",
        "colab_type": "text"
      },
      "source": [
        "## Fine tune Bertmodel for sentiment classification\n",
        "\n",
        "- For each sentence/tweet, the embedding of the [cls] token is extracted. The BERT embedding is in 768 dimensions.  \n",
        "- Then we add on top of it a Dense layer of shape (768, 3) because we have 3 classes in our problem\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "no5O17d8zx3o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class BertClassifier(nn.Module):\n",
        "    def __init__(self, from_pretrained='bert-base-uncased'):\n",
        "        super(BertClassifier, self).__init__()\n",
        "\n",
        "        self.bert = BertModel.from_pretrained(from_pretrained)\n",
        "        # dimension of the hidden state (768)\n",
        "        self.hidden_dim = self.bert.pooler.dense.out_features \n",
        "\n",
        "        #self.dropout = nn.Dropout(dropout)\n",
        "        self.linear = nn.Linear(self.hidden_dim, 3)\n",
        "        \n",
        "        \n",
        "    def forward(self, tokens, attn_masks=None):\n",
        "        # pooled output is the encoding of CLS token\n",
        "        #_, pooled_output = self.bert(tokens, attention_mask=masks, output_all_encoded_layers=False)\n",
        "\n",
        "        # Feeding the input to BERT model to obtain contextualized representations\n",
        "        cont_reps, _ = self.bert(tokens, attention_mask=attn_masks)\n",
        "\n",
        "        # Obtaining the representation of [CLS] head\n",
        "        cls_rep = cont_reps[:, 0]\n",
        "        logits = self.linear(cls_rep)\n",
        "\n",
        "        return logits # don't use softmax activation as nn.CrossEntropy do it internally\n",
        "    \n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bq0cgnAJ1t9-",
        "colab_type": "text"
      },
      "source": [
        "<a id='Define_data_loaders'></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B37oczJtMXyQ",
        "colab_type": "text"
      },
      "source": [
        "### Define data loaders and training parameters\n",
        "- classical batch size of 4\n",
        "- 5 epochs (sufficient to reach good results) . In the orgininal paper the authors only suggest a small number of epochs (2-4)\n",
        "- set the tensors in cuda mode to use GPU of colab\n",
        "- Criterion: we use the cross entropy of pytorch. It applies softmax to the logits and its formula is defined as follow\n",
        "\n",
        "$$-\\sum_{i=1}^n\\sum_{k=1}^3\\mathbb{1}_{y_i=k}log(p(y_i = k | x_i))$$\n",
        "\n",
        "With n, the number of training samples, $y_i$ the class of observation, $x_i$ the embedded representation of obsevration i, and $p(y_i = k| x_i)$ the probability observation i belongs to class $k$, computed with softmax function\n",
        "\n",
        "- Optimizer : Adam Optimizer, an adaptative method which compute different learning rates per feature and keeps in memory both first and second moment of the gradient of the loss. We use a small learning rate to avoid catastrophic forgetting. Using a too large reference learning rate (because learning rate in Adam is the product of a reference learning rate $\\eta$ and a term with first and second moment of the gradient ) can lead to miss the minimum of the loss."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a8_ypxSazx3r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BATCH_SIZE = 4\n",
        "EPOCHS = 5\n",
        "gpu = True\n",
        "\n",
        "train_dataset = TensorDataset(train_tokens_tensor,\n",
        "                              train_masks_tensor,\n",
        "                              y_train_tensor)\n",
        "#train_sampler = RandomSampler(train_dataset)\n",
        "#train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=BATCH_SIZE)\n",
        "train_dataloader = DataLoader(train_dataset, \n",
        "                              batch_size=BATCH_SIZE,\n",
        "                              num_workers=5)\n",
        "\n",
        "test_dataset = TensorDataset(test_tokens_tensor,\n",
        "                             test_masks_tensor,\n",
        "                             y_test_tensor)\n",
        "#test_sampler = SequentialSampler(test_dataset)\n",
        "#test_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=BATCH_SIZE)\n",
        "test_dataloader = DataLoader(test_dataset,\n",
        "                             batch_size=BATCH_SIZE,\n",
        "                             num_workers=5)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eioKgpiFjof5",
        "colab_type": "code",
        "outputId": "3cd4cb58-7152-4bd3-8b39-4226d14510b7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 115,
          "referenced_widgets": [
            "95632cd5446043ecb96ce56cc077e49e",
            "d401adb8646241c19cf3d061e9a5df46",
            "dd1401e9543c4a879042598b02e53d73",
            "16d7c6097286453197f00d4f667f0378",
            "7ebd21cd8cdf4bf79639f56d6bb39df5",
            "5acfa2f73039425cb9656ffde20196a6",
            "093947dbb8554cedb00d965a41f05105",
            "d9c26b01ac874973a7b1151a54389c6b",
            "8de0340680514f20a5c88b26a6c478cd",
            "fdb2e4d9dfbb4c87aee72f42d9548820",
            "7f38e242d9f64c7aa8d3e7e0e575d3c2",
            "d733618480d84cffa5ab19cf2fac3aa7",
            "4d75e26d061549369cbcbc0a685e2a3d",
            "4ea724a0fed94c45970e3992f861e931",
            "2a4cb51843c648d39f5a7fbf57c480b4",
            "5ba0bcba8cc04c95b1ab9b2ebb30bc62"
          ]
        }
      },
      "source": [
        "sentiment_model = BertClassifier()\n",
        "\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "# adam optimizer \n",
        "optimizer = optim.Adam(sentiment_model.parameters(), lr=1e-5)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "95632cd5446043ecb96ce56cc077e49e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=361, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8de0340680514f20a5c88b26a6c478cd",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=440473133, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PyqEliWJ1xaV",
        "colab_type": "text"
      },
      "source": [
        "<a id='Training_loop'></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WKhLCLhwtamU",
        "colab_type": "text"
      },
      "source": [
        "### Training loop\n",
        "\n",
        "Remark: at the end of each epoch, the accuracy on all the train set is computed\n",
        "\n",
        "WARNING  \n",
        "\n",
        "Training the model is pretty long (about 20 minutes), make sure to download the saved model and to point at direction path_model under the name 'sentiment_classifier'."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F8vJDUs8He0K",
        "colab_type": "code",
        "outputId": "20ded310-11ca-4dba-fc6b-4df8923eebf4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "%%time\n",
        "\n",
        "n_train = train_tokens_tensor.shape[0]\n",
        "\n",
        "def train_model(sentiment_model, gpu=True, EPOCHS=5, verbose=True):\n",
        "  \"\"\"\n",
        "  \n",
        "  \"\"\"\n",
        "\n",
        "  if gpu: \n",
        "    print('GPU mode')\n",
        "    sentiment_model = sentiment_model.to(\"cuda\")\n",
        "    # optimizer has to be in cuda mode also\n",
        "    optimizer = optim.Adam(sentiment_model.parameters(), lr=1e-5) \n",
        "\n",
        "  for ep in tqdm(range(EPOCHS)): # iterate over epochs\n",
        "    list_preds = [] # list of predictions during this epochs\n",
        "    list_labels = [] # list of true labels\n",
        "    train_loss = 0\n",
        "\n",
        "    # tweets is of shape (BATCH_SIZE, maxlen)\n",
        "    for i, (tweets, attn_masks, labels) in enumerate(train_dataloader): \n",
        "        if gpu:\n",
        "          # convert to cuda tensor\n",
        "          tweets, attn_masks, labels = tweets.cuda(), attn_masks.cuda(), labels.cuda() \n",
        "\n",
        "        # Clear gradients\n",
        "        optimizer.zero_grad()  \n",
        "        \n",
        "        #computes the logits with the forward pass\n",
        "        logits = sentiment_model(tweets, attn_masks)\n",
        "\n",
        "        # compute the predictions ie the class wich maximize the logits\n",
        "        preds = torch.argmax(logits, axis=1).cpu().detach().numpy()\n",
        "        #list_preds.append(list(preds))\n",
        "        list_preds += list(preds)\n",
        "        #list_labels.append(list(labels.detach().cpu().numpy())) # add labels to compute at the end of \n",
        "        list_labels += list(labels.detach().cpu().numpy())\n",
        "\n",
        "        # Computing loss\n",
        "        loss = criterion(logits, labels) # premier param = logits of shape (n_batch, n_class=3), 2e param = labels \n",
        "        train_loss+=loss.item()\n",
        "\n",
        "        # Backpropagating the gradients\n",
        "        loss.backward()\n",
        "\n",
        "        # Optimization step\n",
        "        optimizer.step()\n",
        "\n",
        "        # monitoring \n",
        "        if i%100==0 and verbose:\n",
        "          print('Iteration {} of epoch  {} completed'.format(i, ep))\n",
        "\n",
        "    # monitoring \n",
        "    # print the accuracy of all the train set \n",
        "    accuracy = (np.array(list_preds) == np.array(list_labels)).sum()/n_train\n",
        "    print('Epoch:', ep, 'loss:', train_loss, 'accuracy:', accuracy)\n",
        "\n",
        "  return sentiment_model\n",
        "\n",
        "if os.path.isfile(path_model + model_name):\n",
        "  print('Model already exists. Loading of the model.')\n",
        "  sentiment_model = BertClassifier()\n",
        "  sentiment_model.load_state_dict(torch.load(path_model + model_name))\n",
        "  # for unknown reason, when we load a saved model\n",
        "  # we have to specify again the device\n",
        "  sentiment_model.to('cuda')\n",
        "else:\n",
        "  print('Model does not exist yet. Training of the model.')\n",
        "  sentiment_model = train_model(sentiment_model)\n",
        "  torch.save(sentiment_model.state_dict(), path_model + model_name)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "GPU mode\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r  0%|          | 0/5 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Iteration 0 of epoch  0 completed\n",
            "Iteration 100 of epoch  0 completed\n",
            "Iteration 200 of epoch  0 completed\n",
            "Iteration 300 of epoch  0 completed\n",
            "Iteration 400 of epoch  0 completed\n",
            "Iteration 500 of epoch  0 completed\n",
            "Iteration 600 of epoch  0 completed\n",
            "Iteration 700 of epoch  0 completed\n",
            "Iteration 800 of epoch  0 completed\n",
            "Iteration 900 of epoch  0 completed\n",
            "Iteration 1000 of epoch  0 completed\n",
            "Iteration 1100 of epoch  0 completed\n",
            "Iteration 1200 of epoch  0 completed\n",
            "Iteration 1300 of epoch  0 completed\n",
            "Iteration 1400 of epoch  0 completed\n",
            "Iteration 1500 of epoch  0 completed\n",
            "Iteration 1600 of epoch  0 completed\n",
            "Iteration 1700 of epoch  0 completed\n",
            "Iteration 1800 of epoch  0 completed\n",
            "Iteration 1900 of epoch  0 completed\n",
            "Iteration 2000 of epoch  0 completed\n",
            "Iteration 2100 of epoch  0 completed\n",
            "Iteration 2200 of epoch  0 completed\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r 20%|██        | 1/5 [03:55<15:41, 235.43s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch : 0 loss : 1442.841350659728 accuracy : 0.7362385321100917\n",
            "Iteration 0 of epoch  1 completed\n",
            "Iteration 100 of epoch  1 completed\n",
            "Iteration 200 of epoch  1 completed\n",
            "Iteration 300 of epoch  1 completed\n",
            "Iteration 400 of epoch  1 completed\n",
            "Iteration 500 of epoch  1 completed\n",
            "Iteration 600 of epoch  1 completed\n",
            "Iteration 700 of epoch  1 completed\n",
            "Iteration 800 of epoch  1 completed\n",
            "Iteration 900 of epoch  1 completed\n",
            "Iteration 1000 of epoch  1 completed\n",
            "Iteration 1100 of epoch  1 completed\n",
            "Iteration 1200 of epoch  1 completed\n",
            "Iteration 1300 of epoch  1 completed\n",
            "Iteration 1400 of epoch  1 completed\n",
            "Iteration 1500 of epoch  1 completed\n",
            "Iteration 1600 of epoch  1 completed\n",
            "Iteration 1700 of epoch  1 completed\n",
            "Iteration 1800 of epoch  1 completed\n",
            "Iteration 1900 of epoch  1 completed\n",
            "Iteration 2000 of epoch  1 completed\n",
            "Iteration 2100 of epoch  1 completed\n",
            "Iteration 2200 of epoch  1 completed\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r 40%|████      | 2/5 [07:51<11:46, 235.59s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch : 1 loss : 805.0995346605778 accuracy : 0.863914373088685\n",
            "Iteration 0 of epoch  2 completed\n",
            "Iteration 100 of epoch  2 completed\n",
            "Iteration 200 of epoch  2 completed\n",
            "Iteration 300 of epoch  2 completed\n",
            "Iteration 400 of epoch  2 completed\n",
            "Iteration 500 of epoch  2 completed\n",
            "Iteration 600 of epoch  2 completed\n",
            "Iteration 700 of epoch  2 completed\n",
            "Iteration 800 of epoch  2 completed\n",
            "Iteration 900 of epoch  2 completed\n",
            "Iteration 1000 of epoch  2 completed\n",
            "Iteration 1100 of epoch  2 completed\n",
            "Iteration 1200 of epoch  2 completed\n",
            "Iteration 1300 of epoch  2 completed\n",
            "Iteration 1400 of epoch  2 completed\n",
            "Iteration 1500 of epoch  2 completed\n",
            "Iteration 1600 of epoch  2 completed\n",
            "Iteration 1700 of epoch  2 completed\n",
            "Iteration 1800 of epoch  2 completed\n",
            "Iteration 1900 of epoch  2 completed\n",
            "Iteration 2000 of epoch  2 completed\n",
            "Iteration 2100 of epoch  2 completed\n",
            "Iteration 2200 of epoch  2 completed\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r 60%|██████    | 3/5 [11:48<07:51, 235.97s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch : 2 loss : 446.2581831216812 accuracy : 0.9262778505897772\n",
            "Iteration 0 of epoch  3 completed\n",
            "Iteration 100 of epoch  3 completed\n",
            "Iteration 200 of epoch  3 completed\n",
            "Iteration 300 of epoch  3 completed\n",
            "Iteration 400 of epoch  3 completed\n",
            "Iteration 500 of epoch  3 completed\n",
            "Iteration 600 of epoch  3 completed\n",
            "Iteration 700 of epoch  3 completed\n",
            "Iteration 800 of epoch  3 completed\n",
            "Iteration 900 of epoch  3 completed\n",
            "Iteration 1000 of epoch  3 completed\n",
            "Iteration 1100 of epoch  3 completed\n",
            "Iteration 1200 of epoch  3 completed\n",
            "Iteration 1300 of epoch  3 completed\n",
            "Iteration 1400 of epoch  3 completed\n",
            "Iteration 1500 of epoch  3 completed\n",
            "Iteration 1600 of epoch  3 completed\n",
            "Iteration 1700 of epoch  3 completed\n",
            "Iteration 1800 of epoch  3 completed\n",
            "Iteration 1900 of epoch  3 completed\n",
            "Iteration 2000 of epoch  3 completed\n",
            "Iteration 2100 of epoch  3 completed\n",
            "Iteration 2200 of epoch  3 completed\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\r 80%|████████  | 4/5 [15:43<03:55, 235.87s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch : 3 loss : 322.13973075151443 accuracy : 0.9438619484491044\n",
            "Iteration 0 of epoch  4 completed\n",
            "Iteration 100 of epoch  4 completed\n",
            "Iteration 200 of epoch  4 completed\n",
            "Iteration 300 of epoch  4 completed\n",
            "Iteration 400 of epoch  4 completed\n",
            "Iteration 500 of epoch  4 completed\n",
            "Iteration 600 of epoch  4 completed\n",
            "Iteration 700 of epoch  4 completed\n",
            "Iteration 800 of epoch  4 completed\n",
            "Iteration 900 of epoch  4 completed\n",
            "Iteration 1000 of epoch  4 completed\n",
            "Iteration 1100 of epoch  4 completed\n",
            "Iteration 1200 of epoch  4 completed\n",
            "Iteration 1300 of epoch  4 completed\n",
            "Iteration 1400 of epoch  4 completed\n",
            "Iteration 1500 of epoch  4 completed\n",
            "Iteration 1600 of epoch  4 completed\n",
            "Iteration 1700 of epoch  4 completed\n",
            "Iteration 1800 of epoch  4 completed\n",
            "Iteration 1900 of epoch  4 completed\n",
            "Iteration 2000 of epoch  4 completed\n",
            "Iteration 2100 of epoch  4 completed\n",
            "Iteration 2200 of epoch  4 completed\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 5/5 [19:39<00:00, 236.00s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch : 4 loss : 289.1397498100996 accuracy : 0.9493228484054173\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qvSg1DmJqkr9",
        "colab_type": "text"
      },
      "source": [
        "- After 5 epochs, we have an accuracy of 0.95 on all the train set. Which is very good. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FdUeP37N12u_",
        "colab_type": "text"
      },
      "source": [
        "<a id='Evaluation'></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sALEIzVitrzx",
        "colab_type": "text"
      },
      "source": [
        "### Evaluation "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qq2ovp1ktyvl",
        "colab_type": "text"
      },
      "source": [
        "#### Quantitative evaluation on the test set\n",
        "\n",
        "- As the problem is unbalanced, accuracy may not be the best metric. We need to evaluate the performaces on each classes anc chek the f1 score for example which is an harmonic average of the precision and the recall : \n",
        "$$f_1 = \\frac{2 precision \\times recall}{precision + recall }$$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oKbfRWUNtsKl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%time\n",
        "\n",
        "sentiment_model.eval()\n",
        "predictions = []\n",
        "\n",
        "with torch.no_grad():\n",
        "    for step_num, batch_data in enumerate(test_dataloader):\n",
        "        token_ids, masks, labels = tuple(t.to('cuda') for t in batch_data)\n",
        "        # compute the predictions\n",
        "        logits = sentiment_model(token_ids, masks)\n",
        "        preds = torch.argmax(logits, axis=1).cpu().detach().numpy()\n",
        "        predictions += list(preds)\n",
        "\n",
        "predictions = np.array(predictions)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-NaHbSnOt5U9",
        "colab_type": "code",
        "outputId": "023d163e-3c32-4ac7-af92-075fed9036d8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "print(classification_report(y_test, predictions))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.93      0.94      1993\n",
            "           1       0.78      0.87      0.82       612\n",
            "           2       0.91      0.85      0.88       448\n",
            "\n",
            "    accuracy                           0.90      3053\n",
            "   macro avg       0.88      0.88      0.88      3053\n",
            "weighted avg       0.91      0.90      0.91      3053\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YHF5Pd6ct44f",
        "colab_type": "text"
      },
      "source": [
        "- Performances are very good for the negative class. But we remind it's the majority one (61%)\n",
        "- Nonetheless, neutral and positive class also have very good performances \n",
        "- Even if positive class represent a smaller proportion than neutral tweets (16% against 23%), performances are better globally (better for precision and F1 score). It might be because neutral tweets can be more difficult to classify than biased tweets because they are more ambiguous"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_1rA8zFHxq3h",
        "colab_type": "text"
      },
      "source": [
        "#### Qualitative evaluation\n",
        "\n",
        "- Check what kind of tweet are misclassified \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VuOgr8hYxYhv",
        "colab_type": "code",
        "outputId": "8ad3c044-e899-47cb-fcb3-0c6b12e95108",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "df_eval = pd.DataFrame({'tweet': X_test,\n",
        "                        'prediction': predictions,\n",
        "                        'y_true': y_test})\n",
        "df_eval.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tweet</th>\n",
              "      <th>prediction</th>\n",
              "      <th>y_true</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1823</th>\n",
              "      <td>RT @Jocklaflair: \"What about Black liv-...... ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13442</th>\n",
              "      <td>RT @RWSurferGirl: FOX News won't admit who the...</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6815</th>\n",
              "      <td>Does @realDonaldTrump believe he's gonna win r...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4579</th>\n",
              "      <td>RT @cristela9: Trump looks like an Oompa Loomp...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9472</th>\n",
              "      <td>RT @monaeltahawy: TwitterLand: has God spoken ...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   tweet  prediction  y_true\n",
              "1823   RT @Jocklaflair: \"What about Black liv-...... ...           0       0\n",
              "13442  RT @RWSurferGirl: FOX News won't admit who the...           0       2\n",
              "6815   Does @realDonaldTrump believe he's gonna win r...           0       0\n",
              "4579   RT @cristela9: Trump looks like an Oompa Loomp...           0       0\n",
              "9472   RT @monaeltahawy: TwitterLand: has God spoken ...           1       0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 104
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hhvgSnjTzjhP",
        "colab_type": "code",
        "outputId": "fd7bc70f-469a-41cf-fbf4-b18309977de5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# example of negative tweet misclassified in neutral one\n",
        "df_eval.loc[9472, 'tweet']"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'RT @monaeltahawy: TwitterLand: has God spoken to you about the #GOPDebates? What did she say?'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 106
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QX-mPbFV0Jww",
        "colab_type": "text"
      },
      "source": [
        "- even for a human it's not very clear the tweet is negative because of a lack of 'negative' words. A child wouldn't understand for example the tweet is negative"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OW4ZcB44x2Bm",
        "colab_type": "code",
        "outputId": "2b42f2d8-5944-4302-dab9-ee1d90026af6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "# example of positive tweet missclassified in negative one \n",
        "df_eval.loc[13442, 'tweet']"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"RT @RWSurferGirl: FOX News won't admit who the Republican leader is right now I mean @realDonaldTrump only has a double-digit lead _Ùà¼_Ùàü #GOPD\\x89Û_\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 107
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aLYf-Xuv3-9P",
        "colab_type": "text"
      },
      "source": [
        "- Actually , we can't really understand why this tweet has been classified in a positive class in the first place. We would agree more with our classifier"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7m3bzv2C2DJc",
        "colab_type": "text"
      },
      "source": [
        "<a id='Application'></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6TEpuDt44psG",
        "colab_type": "text"
      },
      "source": [
        "## Application to French tweets before the 2017 presidential election\n",
        "\n",
        "- We will need to scrape those tweets \n",
        "- Then translate them into English \n",
        "- Best would have been to use directly a French embedding (e.g CamemBERT) and train the classification model on a corpus of French tweets but we coud not find such a dataset\n",
        "- We use the package GetOldTweets3: no limitation in time or of  number of tweets "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i0LCmt8g2FYu",
        "colab_type": "text"
      },
      "source": [
        "<a id='Scrape'></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e7mpM16VRvGA",
        "colab_type": "text"
      },
      "source": [
        "### Scrape the tweets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pF-OKOMD5W76",
        "colab_type": "code",
        "outputId": "393296ec-4d33-490a-9d38-280672b6bcf4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 190
        }
      },
      "source": [
        "!pip install GetOldTweets3"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting GetOldTweets3\n",
            "  Downloading https://files.pythonhosted.org/packages/ed/f4/a00c2a7c90801abc875325bb5416ce9090ac86d06a00cc887131bd73ba45/GetOldTweets3-0.0.11-py3-none-any.whl\n",
            "Requirement already satisfied: lxml>=3.5.0 in /usr/local/lib/python3.6/dist-packages (from GetOldTweets3) (4.2.6)\n",
            "Collecting pyquery>=1.2.10\n",
            "  Downloading https://files.pythonhosted.org/packages/78/43/95d42e386c61cb639d1a0b94f0c0b9f0b7d6b981ad3c043a836c8b5bc68b/pyquery-1.4.1-py2.py3-none-any.whl\n",
            "Collecting cssselect>0.7.9\n",
            "  Downloading https://files.pythonhosted.org/packages/3b/d4/3b5c17f00cce85b9a1e6f91096e1cc8e8ede2e1be8e96b87ce1ed09e92c5/cssselect-1.1.0-py2.py3-none-any.whl\n",
            "Installing collected packages: cssselect, pyquery, GetOldTweets3\n",
            "Successfully installed GetOldTweets3-0.0.11 cssselect-1.1.0 pyquery-1.4.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "upR64ayW6G_v",
        "colab_type": "text"
      },
      "source": [
        "- First round of presidential election : April the 23 rd \n",
        "- Candidates \n",
        "  - Emmanuel Macron \n",
        "  - Marine Le Pen \n",
        "  - Francois Fillon \n",
        "  - Jean Luc Melenchon\n",
        "  - Benoit Hamon \n",
        "- We scraped the tweets published between the beginning of january and April the 23rd\n",
        "- Need to select the tweets published in France. Problem : the geolocalisation is not always activated in Twitter and so there are a lot of tweets we cannot scrape by specifying the geolocalisation \n",
        "- Cannot use the encoding of emojis because we have error in translation..\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AL2OZn5J4PX_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import GetOldTweets3 as got"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lbhvc3MxCW5I",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def scrape_tweets(candidate, n=1000):\n",
        "  \"\"\"Scrape the tweets for a given candidate one month before the first round of\n",
        "  French Presidential election.\n",
        "\n",
        "  Scrape only in France globally\n",
        "\n",
        "  Parameters\n",
        "  ----------\n",
        "  candidate: sting\n",
        "    name of a candidate\n",
        "    values can be 'Macron', 'Melanchon', 'Fillon', 'Le Pen', 'Hamon'\n",
        "\n",
        "  n: double\n",
        "    number of tweets to scrape\n",
        "    default is 1 000\n",
        "\n",
        "\n",
        "  Returns\n",
        "  -------\n",
        "  tweets: list of scraped tweets\n",
        "\n",
        "  \"\"\"\n",
        "  tweetCriteria = got.manager.TweetCriteria().setQuerySearch(candidate)\\\n",
        "                                            .setSince(\"2017-01-01\")\\\n",
        "                                            .setUntil(\"2017-04-23\")\\\n",
        "                                            .setMaxTweets(n)\\\n",
        "                                            .setNear('France')\\\n",
        "                                            .setWithin('1000km')\n",
        "                                                                \n",
        "  tweets = got.manager.TweetManager.getTweets(tweetCriteria)\n",
        "\n",
        "  return tweets"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5AaiFTymL43w",
        "colab_type": "code",
        "outputId": "3f580704-6275-482f-ed56-f1cbd8b0b818",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "%%time\n",
        "\n",
        "# candidates names\n",
        "candidates = ['Macron', 'Fillon', 'Le Pen', 'Melenchon', 'Hamon']\n",
        "# number of tweets to scrap\n",
        "n_scrape = 1000\n",
        "# dictionary to stock scraped tweets\n",
        "scraped_tweets = {}\n",
        "\n",
        "for candidate in candidates:\n",
        "  # Remove space and lower candidate's name\n",
        "  name = candidate.replace(\" \", \"\").lower()\n",
        "\n",
        "  if os.path.isfile(path+'tweets_'+name):\n",
        "    print(\"Scraped tweets already exists for \" + candidate)\n",
        "    tweets = pickle.load(open(path+'tweets_'+name,\n",
        "                              'rb'))\n",
        "  else:\n",
        "    print(\"Scraping of \" + candidate + \"'s tweets\")\n",
        "    # scrape tweets\n",
        "    tweets = scrape_tweets(candidate=candidate,\n",
        "                          n=n_scrape)\n",
        "    # save french tweets as pickle\n",
        "    pickle.dump(tweets, open(path+'tweets_'+name, 'wb'))\n",
        "\n",
        "  print('number of scraped tweets '+candidate+':', len(tweets))\n",
        "  # declare global variable\n",
        "  globals()['tweets_'+name] = tweets\n",
        "  # append scraped dictionary\n",
        "  scraped_tweets[name] = tweets"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Scraped tweets already exists for Macron\n",
            "number of scraped tweets Macron: 1000\n",
            "Scraped tweets already exists for Fillon\n",
            "number of scraped tweets Fillon: 1000\n",
            "Scraped tweets already exists for Le Pen\n",
            "number of scraped tweets Le Pen: 564\n",
            "Scraped tweets already exists for Melenchon\n",
            "number of scraped tweets Melenchon: 340\n",
            "Scraped tweets already exists for Hamon\n",
            "number of scraped tweets Hamon: 488\n",
            "CPU times: user 23.5 ms, sys: 3.61 ms, total: 27.1 ms\n",
            "Wall time: 777 ms\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Rp5tupxw2vwU",
        "colab_type": "text"
      },
      "source": [
        "<a id='Translate'></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bXp6VpyAR4Ij",
        "colab_type": "text"
      },
      "source": [
        "### Translate the tweets into English\n",
        "\n",
        "Now that we scraped the tweets for the different politicans, we wish to translate them in english in order to apply the model trained previously.\n",
        "\n",
        "To do so, we use the googletrans package, the package of Google Translate: https://pypi.org/project/googletrans/\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5-f8AE07R8uk",
        "colab_type": "code",
        "outputId": "81d042aa-8e63-4ea1-8c8d-027f16e3e26a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 275
        }
      },
      "source": [
        "!pip install googletrans"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting googletrans\n",
            "  Downloading https://files.pythonhosted.org/packages/fd/f0/a22d41d3846d1f46a4f20086141e0428ccc9c6d644aacbfd30990cf46886/googletrans-2.4.0.tar.gz\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from googletrans) (2.21.0)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->googletrans) (2.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->googletrans) (2020.4.5.1)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->googletrans) (3.0.4)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->googletrans) (1.24.3)\n",
            "Building wheels for collected packages: googletrans\n",
            "  Building wheel for googletrans (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for googletrans: filename=googletrans-2.4.0-cp36-none-any.whl size=15777 sha256=e5f0f69eb048b27506e7d3de9ff79fefa2f066597cdde26e7b6372388129f91c\n",
            "  Stored in directory: /root/.cache/pip/wheels/50/d6/e7/a8efd5f2427d5eb258070048718fa56ee5ac57fd6f53505f95\n",
            "Successfully built googletrans\n",
            "Installing collected packages: googletrans\n",
            "Successfully installed googletrans-2.4.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4fD11JIYFzK0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from googletrans import Translator  \n",
        "#from translate import Translator"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dVRfBdVb7uFo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "translator = Translator()\n",
        "\n",
        "def translate_into_english(tweets):\n",
        "  \"\"\"Tranlate a list of tweets into english.\n",
        "\n",
        "  Parameters\n",
        "  ----------\n",
        "  tweets: list\n",
        "    list of tweets to be translated\n",
        "\n",
        "\n",
        "  Returns\n",
        "  -------\n",
        "  tweets_english: list\n",
        "    list of tweets translated into english\n",
        "  \n",
        "  \"\"\"\n",
        "  tweets_english = []\n",
        "\n",
        "  for tweet in tqdm(tweets):\n",
        "    try:\n",
        "      translated_tweet = translator.translate(tweet.text, dest='en').text\n",
        "      tweets_english.append(translated_tweet)\n",
        "    except:\n",
        "      pass\n",
        "\n",
        "  return tweets_english"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FYuFJW3HAVl9",
        "colab_type": "code",
        "outputId": "eb9d5345-284c-4447-d549-2794c5a567d8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "%%time\n",
        "\n",
        "# candidates names\n",
        "candidates = ['macron', 'fillon', 'lepen', 'melenchon', 'hamon']\n",
        "# dictionary to stock scraped tweets\n",
        "scraped_tweets = {}\n",
        "# dictionary to stock english tweets\n",
        "english_tweets = {}\n",
        "\n",
        "for candidate in candidates:\n",
        "\n",
        "  if os.path.isfile(path+'tweets_'+candidate+'_english_v2'):\n",
        "    print(\"English translation already exists for \"+candidate+\"s twwets\")\n",
        "    tweets_english = pickle.load(open(path+'tweets_'+candidate+'_english_v2',\n",
        "                                      'rb'))\n",
        "  else:\n",
        "    # read tweets saved in pickle\n",
        "    tweets = pickle.load(open(path+'tweets_'+candidate, 'rb'))\n",
        "    print('nb scraped tweets '+candidate+':', len(tweets))\n",
        "    # declare global variable\n",
        "    globals()['tweets_'+candidate] = tweets\n",
        "    # append scraped tweets dictionary\n",
        "    scraped_tweets[candidate] = tweets\n",
        "    # translate tweets\n",
        "    tweets_english = translate_into_english(tweets)\n",
        "    # save english tweets as pickle\n",
        "    pickle.dump(tweets_english,\n",
        "                open('tweets_'+candidate+'_english', 'wb'))\n",
        "\n",
        "  print('nb english tweets '+candidate+':', len(tweets_english))\n",
        "  # declare global variable\n",
        "  globals()['tweets_'+candidate+'_english'] = tweets_english\n",
        "  # append english dictionary\n",
        "  english_tweets[candidate] = tweets_english\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "English translation already exists for macrons twwets\n",
            "nb english tweets macron: 643\n",
            "English translation already exists for fillons twwets\n",
            "nb english tweets fillon: 470\n",
            "English translation already exists for lepens twwets\n",
            "nb english tweets lepen: 260\n",
            "English translation already exists for melenchons twwets\n",
            "nb english tweets melenchon: 114\n",
            "English translation already exists for hamons twwets\n",
            "nb english tweets hamon: 208\n",
            "CPU times: user 6.92 ms, sys: 2.31 ms, total: 9.23 ms\n",
            "Wall time: 1.4 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HBLrBoEK2yVI",
        "colab_type": "text"
      },
      "source": [
        "<a id='Apply_model'></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D-WnzciblYLC",
        "colab_type": "text"
      },
      "source": [
        "### Apply model to scraped tweets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RGln4w7wuzMm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_predictions(candidate):\n",
        "  \"\"\"Get vecteurs of predictions for every tweet of a candidate.\n",
        "\n",
        "  Parameters\n",
        "  ----------\n",
        "  candidate: string\n",
        "    name of a candidate\n",
        "\n",
        "  \n",
        "  Returns\n",
        "  -------\n",
        "  predictions: list of integers\n",
        "    list of 0, 1, 2, according of the sentiment predicted for each tweet\n",
        "\n",
        "  \"\"\"\n",
        "  # get list of tweets\n",
        "  tweets_candidate = globals()['tweets_'+candidate+'_english']\n",
        "  # transform the list into a pandas Series\n",
        "  X_candidate = pd.Series(tweets_candidate)\n",
        "  # tokenize tweets\n",
        "  candidate_tokens = tokenizer.tokenize(X_candidate)\n",
        "  # prepare for predictions\n",
        "  candidate_tokens_tensor = torch.tensor(candidate_tokens)\n",
        "  candidate_masks_tensor = torch.tensor((candidate_tokens_tensor != 0).long())\n",
        "  candidate_dataset = TensorDataset(candidate_tokens_tensor,\n",
        "                                    candidate_masks_tensor)\n",
        "  candidate_dataloader = DataLoader(candidate_dataset,\n",
        "                                    batch_size=BATCH_SIZE,\n",
        "                                    num_workers=5)\n",
        "  # compute predictions\n",
        "  predictions = []\n",
        "\n",
        "  with torch.no_grad():\n",
        "      for step_num, batch_data in enumerate(candidate_dataloader):\n",
        "          token_ids, masks = tuple(t.to('cuda') for t in batch_data)\n",
        "          # compute the predictions\n",
        "          logits = sentiment_model(token_ids, masks)\n",
        "          preds = torch.argmax(logits, axis=1).cpu().detach().numpy()\n",
        "          predictions += list(preds)\n",
        "\n",
        "  predictions = np.array(predictions)\n",
        "\n",
        "  return predictions\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-oKqF3SywUwR",
        "colab_type": "code",
        "outputId": "c83db74d-b565-410e-8e2b-39e55948329b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        }
      },
      "source": [
        "%%time\n",
        "\n",
        "candidates = ['macron', 'fillon', 'lepen', 'melenchon', 'hamon']\n",
        "\n",
        "dict_predictions = defaultdict(list)\n",
        "\n",
        "for candidate in tqdm(candidates):\n",
        "  predictions = get_predictions(candidate)\n",
        "  dict_predictions[candidate] = predictions\n",
        "\n",
        "dict_predictions = dict(dict_predictions)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  0%|          | 0/5 [00:00<?, ?it/s]/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:24: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "100%|██████████| 5/5 [00:22<00:00,  4.58s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 15.2 s, sys: 6.65 s, total: 21.9 s\n",
            "Wall time: 22.9 s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dhz3GwgL2cHO",
        "colab_type": "text"
      },
      "source": [
        "As a reminder, the encoding is done as follow:\n",
        "- 0 for negative tweets\n",
        "- 1 for neutral ones \n",
        "- 2 for positive ones"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "70QXGK7Kytcv",
        "colab_type": "code",
        "outputId": "fe614617-d938-4300-928c-06bea9302c02",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "for candidate, predictions in dict_predictions.items():\n",
        "  print(\"%s's predictions:\" %candidate, Counter(predictions))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "macron's predictions: Counter({1: 467, 0: 145, 2: 31})\n",
            "fillon's predictions: Counter({1: 282, 0: 182, 2: 6})\n",
            "lepen's predictions: Counter({1: 183, 0: 71, 2: 6})\n",
            "melenchon's predictions: Counter({1: 82, 0: 30, 2: 2})\n",
            "hamon's predictions: Counter({1: 136, 0: 65, 2: 7})\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mywHC-3q6Ss8",
        "colab_type": "code",
        "outputId": "94f64004-f067-43aa-d689-61cee93928fc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "# creation of a DataFrame\n",
        "\n",
        "labels = ['negative', 'neutral', 'positive']\n",
        "encodings = [0, 1, 2]\n",
        "\n",
        "dict_df = {'candidate': candidates,\n",
        "           'negative': [],\n",
        "           'neutral': [],\n",
        "           'positive': [],\n",
        "           'total': []}\n",
        "          \n",
        "for candidate in candidates:\n",
        "  predictions = dict_predictions[candidate]\n",
        "  count = dict(Counter(predictions))\n",
        "  dict_df['total'].append(len(predictions))\n",
        "  for label, val in zip(labels, encodings):\n",
        "    dict_df[label].append(count[val])\n",
        "\n",
        "df_pred = pd.DataFrame(data = dict_df)\n",
        "df_pred.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>candidate</th>\n",
              "      <th>negative</th>\n",
              "      <th>neutral</th>\n",
              "      <th>positive</th>\n",
              "      <th>total</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>macron</td>\n",
              "      <td>145</td>\n",
              "      <td>467</td>\n",
              "      <td>31</td>\n",
              "      <td>643</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>fillon</td>\n",
              "      <td>182</td>\n",
              "      <td>282</td>\n",
              "      <td>6</td>\n",
              "      <td>470</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>lepen</td>\n",
              "      <td>71</td>\n",
              "      <td>183</td>\n",
              "      <td>6</td>\n",
              "      <td>260</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>melenchon</td>\n",
              "      <td>30</td>\n",
              "      <td>82</td>\n",
              "      <td>2</td>\n",
              "      <td>114</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>hamon</td>\n",
              "      <td>65</td>\n",
              "      <td>136</td>\n",
              "      <td>7</td>\n",
              "      <td>208</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   candidate  negative  neutral  positive  total\n",
              "0     macron       145      467        31    643\n",
              "1     fillon       182      282         6    470\n",
              "2      lepen        71      183         6    260\n",
              "3  melenchon        30       82         2    114\n",
              "4      hamon        65      136         7    208"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kI1f2twJ9ud4",
        "colab_type": "code",
        "outputId": "732383dc-f029-4252-e0fc-d3154fe47a84",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "total_pred = df_pred[labels].sum().sum()\n",
        "print(\"In total, we obtained %d predictions.\" %total_pred)\n",
        "df_pred[labels].sum() / total_pred"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "In total, we obtained 1695 predictions.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "negative    0.290855\n",
              "neutral     0.678466\n",
              "positive    0.030678\n",
              "dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VP8uVooO_O6-",
        "colab_type": "text"
      },
      "source": [
        "- Here we can observe that tweets are rarely predicted as positive: less than 3% of translated tweets are predictited as such.\n",
        "- Most of the time (>67%) tweets are predicted as neutral."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xx44XwSm3JJz",
        "colab_type": "code",
        "outputId": "52f0c5ea-ba65-4479-856f-df803cd36682",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 377
        }
      },
      "source": [
        "# show some example of tweets and their prediction\n",
        "\n",
        "for candidate in candidates:\n",
        "  print(candidate)\n",
        "  predictions = dict_predictions[candidate]\n",
        "  tweets_english = globals()['tweets_'+candidate+'_english']\n",
        "  \n",
        "  for label, val in [('negative', 0), ('neutral', 1), ('positive', 2)]:\n",
        "    i = random.choices(np.argwhere(predictions==0))[0][0]\n",
        "    print('predicted %s:' %label, tweets_english[i])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "macron\n",
            "predicted negative: @Mathieu_Gbt Baroin has no chance in front of Macron.\n",
            "predicted neutral: journalists are never objective. Nobody is. Bcp reviews Macron fall fantasy\n",
            "predicted positive: Macron’s team accuses RT, Sputnik of spreading false information ahead of French presidential poll: Emmanuel… http://dlvr.it/NY2yNL\n",
            "fillon\n",
            "predicted negative: Fillon cries conspiracy! He admits! s excuse! makes the costumes! repays the borrowed money! It's not me ! what credibility !!!\n",
            "predicted neutral: and yet who twice asked to meet Sarkozy Fillon the loser of the UMP LR\n",
            "predicted positive: yes and I will never forgive Fillon for his personal attacks against Sarkozy\n",
            "lepen\n",
            "predicted negative: paris | Paris Bourse: FRANCE 2017-Le Pen poses as a victim of the media and judges: M arine Le Pen ... http://dlvr.it/NTntYG\n",
            "predicted neutral: think of the extreme left Le Pen ...\n",
            "predicted positive: #LaFranceVoteMarine heiress Castle #Lepen #AttrapeTout, the biggest manipulative and thief clan of billionaires\n",
            "melenchon\n",
            "predicted negative: paris | Paris Bourse: FRANCE 2017-Outdistanced Hamon accentuates the offensive against Macron and Mélenchon: Benedict ... http://dlvr.it/NtwW11\n",
            "predicted neutral: we also Mélenchon who paid her permanently with public money and sell it then. Legal. No moral.\n",
            "predicted positive: Mr Jean-Luc Mélenchon for president of France for Human Rights and Equality Justice UNIVERSAL HUMAN VALUES SOCIAL WELFARE\n",
            "hamon\n",
            "predicted negative: And end 14/14 Benoît Hamon By by_kilroyjones Artist #politique #kilroy #kilroyjones # hammon ... https://www.instagram.com/p/BQqDMOoFFRe/\n",
            "predicted neutral: MLP will not be elected, not credible polls, the Left has chosen Hamon, the repulsive right French are no longer the \"calves\"\n",
            "predicted positive: Hamon as meeting other tackle and brings no more\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yjskA5e45VQT",
        "colab_type": "text"
      },
      "source": [
        "When we look in details the tweets and their associated sentiment prediction, it can be seen that predictions are made in an absolute manner, as opposed to a prediction relative to the candidate.\n",
        "\n",
        "This can be illustrated by the following tweet about E. Macron:\n",
        "\"@Mathieu_Gbt Baroin has no chance in front of Macron.\"\n",
        "This tweet is a negative phrase, but is clearly in favour of E. Macron. However, it is categorized as negative because the prediction was made in an absolute and not relative manner by considering the candidate. Indeed, this same tweet can be categorized as positive for E. Macron, but as negative for F. Baroin.\n",
        "\n",
        "Thus, a possible improvement would be to train a template with the text of the tweet, but adding a context variable that would be the subject of the tweet from which the \"sentiment\" of the tweet was determined.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j5Q-lMke228l",
        "colab_type": "text"
      },
      "source": [
        "<a id='Score_popularity'></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gqo9ddce1bTF",
        "colab_type": "text"
      },
      "source": [
        "### Score of popularity\n",
        "\n",
        "Now that we were able to apply our previously trained model on our scraped and translated tweets, we would like to compute a score a popularity for each candidate.\n",
        "\n",
        "We would like the score to take both \"absolute\" results (i.e. the proportion of negative, neutral or positive tweets per candidate) and relative results (i.e. comparing these proportions to the proportions across all predictions).\n",
        "\n",
        "To do this, the following formula is used for a given candidate.\n",
        "\n",
        "$$\\sum_{i} w_i \\times p_i \\times (p_i - \\bar{p}_i)$$\n",
        "\n",
        "where:\n",
        "- $i$ is the label (negative, neutral, positive)\n",
        "- $w_i$ is the weight associated for the label (respectively -1, 0.5 and 2)\n",
        "- $p_i$ and $\\bar{p}_i$ are the proportion of tweets predicted of the label respectively among the candidate's tweets and among all predictions\n",
        "\n",
        "The weight vector [-1, 0.5, 2] (for \"negative\", \"neutral\" and \"positive\" respectively) was chosen for the following reason.\n",
        "Instead of the \"classical\" weight vector [-1, 0, 1] in this kind of problem, we want to emphasize positive tweets, thus doubling their importance, but we also want to give importance to neutral tweets, without giving them the same importance as positive tweets."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xn2FRkEL3kUS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def populatity_score(df_pred):\n",
        "  \"\"\"Compute the score of popularity for each candidate,\n",
        "  from the predicted sentiment of tweets.\n",
        "\n",
        "  Parameters\n",
        "  ----------\n",
        "  df_pred: pandas DataFrame\n",
        "    columns:\n",
        "      - 'candidate' (string), name of the candidate (macron, hamin, etc.)\n",
        "      - 'negative' (resp. 'neutral' and 'positive') (integer),\n",
        "        number of tweets predicted as negative (resp. neutral and positive) \n",
        "        for the candidate\n",
        "      - 'total' (integer), total number of predictions\n",
        "        (negative + neutral + positive) \n",
        "\n",
        "  Returns\n",
        "  -------\n",
        "  pop_score: dictionnary\n",
        "    key: string, candidate name\n",
        "    value: integer, popularity score\n",
        "\n",
        "  \"\"\"\n",
        "  candidates = df_pred['candidate'].tolist()\n",
        "  labels = ['negative', 'neutral', 'positive']\n",
        "  weights = [-1, 0.5, 2]\n",
        "\n",
        "  total_pred = df_pred[labels].sum().sum()\n",
        "  percent_tot = dict(df_pred[labels].sum() / total_pred)\n",
        "\n",
        "  dict_popularity = defaultdict()\n",
        "\n",
        "  for candidate in candidates:\n",
        "    score = 0\n",
        "    for label, w in zip(labels, weights):\n",
        "      n, t = df_pred[[label, 'total']][df_pred.candidate==candidate].iloc[0]\n",
        "      p = n/t\n",
        "      score += w * p * (p-percent_tot[label])\n",
        "    \n",
        "    dict_popularity[candidate] = score\n",
        "\n",
        "  return dict(dict_popularity)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qj4e6gYFUix_",
        "colab_type": "code",
        "outputId": "74f253ed-8c5b-4a9f-bcda-45bd0c86bff7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "dict_popularity = populatity_score(df_pred)\n",
        "dict_popularity"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'fillon': -0.06131823286608419,\n",
              " 'hamon': -0.014612474581522412,\n",
              " 'lepen': 0.01343590179958459,\n",
              " 'macron': 0.03379170063963901,\n",
              " 'melenchon': 0.02151322258448481}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9wIg8KOXl_JZ",
        "colab_type": "text"
      },
      "source": [
        "Now that we have calculated the popularity scores of each candidate, we can look at the ranking that this gives us.\n",
        "We can see that E. Macron is the one with the highest popularity score, followed by J.-L. Mélenchon and M. Le Pen. B. Hamon is further behind and F. Fillon is at the bottom of the ranking.\n",
        "\n",
        "These results are rather close to those found in the presidential election, except for F. Fillon who came third.\n",
        "This very low score for F. Fillon can perhaps be explained by the \"Peneloppe gate\" affair and by the fact his traditional electoral may not use social media ."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m1yIjXavexrJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}