RNN_PT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "RNN_PT.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c194dfd9668b4c7e97a0b598ee810468": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_56a4a9cb54764f7c9d858310441a55aa",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3eaae8eacaf048448ad11007109143e9",
              "IPY_MODEL_bd2c425468d444e9931605adeaeaa78a"
            ]
          }
        },
        "56a4a9cb54764f7c9d858310441a55aa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3eaae8eacaf048448ad11007109143e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b140724336ea4e04af134d48432f74d6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f16d5d027aed4f8f87773bb7951616b2"
          }
        },
        "bd2c425468d444e9931605adeaeaa78a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_42fd164015de419a9c58be0f860c4439",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "9920512it [00:00, 25909449.50it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9ef3b6475ef049b1822ee119866bdc3e"
          }
        },
        "b140724336ea4e04af134d48432f74d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f16d5d027aed4f8f87773bb7951616b2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "42fd164015de419a9c58be0f860c4439": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9ef3b6475ef049b1822ee119866bdc3e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b7c63f595ac24762a3215071b14fd32c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_58e5e45c6ba7449bab80fd696286e072",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_446d95ac8c6b4e1094fbf6703499f412",
              "IPY_MODEL_023cf4ce24da49329796df463123d41f"
            ]
          }
        },
        "58e5e45c6ba7449bab80fd696286e072": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "446d95ac8c6b4e1094fbf6703499f412": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_915c954de9f348278f22b68eec8cee63",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_faef4bfc378447d892f6278cb410d1c5"
          }
        },
        "023cf4ce24da49329796df463123d41f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4a352572ee9b4682b70da3838d5117c5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "32768it [00:00, 230359.16it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2563cdf52b8348bf89bfaa3ef19e8373"
          }
        },
        "915c954de9f348278f22b68eec8cee63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "faef4bfc378447d892f6278cb410d1c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4a352572ee9b4682b70da3838d5117c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2563cdf52b8348bf89bfaa3ef19e8373": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8cdc0cb999bc4b16bdbe0e75e329481a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_26c8ff74123244e29a66b98557519883",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_aecc1a9e4bc4478096278bf04bc4a7eb",
              "IPY_MODEL_d4560bb340f441b89ce84f457bd7b48c"
            ]
          }
        },
        "26c8ff74123244e29a66b98557519883": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "aecc1a9e4bc4478096278bf04bc4a7eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8eec08b3b62642b9856b188a06762742",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f577c19d9e6b4699bbeca1b4ceb27bd4"
          }
        },
        "d4560bb340f441b89ce84f457bd7b48c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2323f246afa04fc5b0b099d8729426d5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "1654784it [00:00, 6775428.01it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cfad60d6e3f7458fbfd33185e5bad018"
          }
        },
        "8eec08b3b62642b9856b188a06762742": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f577c19d9e6b4699bbeca1b4ceb27bd4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2323f246afa04fc5b0b099d8729426d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cfad60d6e3f7458fbfd33185e5bad018": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "146b255c687b41f5ac178e918045d46d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ef6b741c684546b184ca86813c2b02a0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_022f6fad73914b4cbbd65e48de239c2f",
              "IPY_MODEL_b0f8d0bbd29d45b986b4895a447aff7d"
            ]
          }
        },
        "ef6b741c684546b184ca86813c2b02a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "022f6fad73914b4cbbd65e48de239c2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c1d3127d214545aea609ddcd7c19f1fd",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_abd1c8b5ff7f4bf7a751a53bcacf5d5f"
          }
        },
        "b0f8d0bbd29d45b986b4895a447aff7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8f59433e532748f8a33a47e84c31a281",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "8192it [00:00, 67940.87it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ee940fd1588a48668c2a34c8f26c1195"
          }
        },
        "c1d3127d214545aea609ddcd7c19f1fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "abd1c8b5ff7f4bf7a751a53bcacf5d5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8f59433e532748f8a33a47e84c31a281": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ee940fd1588a48668c2a34c8f26c1195": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dair-ai/pytorch_notebooks/blob/master/RNN_PT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BVq5mz9s8CPL",
        "colab_type": "text"
      },
      "source": [
        "## Building RNNs is Fun with PyTorch and Google Colab\n",
        "In this tutorial, I will first teach you how to build a recurrent neural network (RNN) with a single layer, consisting of one single neuron, with PyTorch and Google Colab. I will also show you how to implement a simple RNN-based model for image classification.\n",
        "\n",
        "This work is heavily inspired by Aurélien Géron's book called [\"Hand-On Machine Learning with Scikit-Learn and TensorFlow\"](https://www.oreilly.com/library/view/hands-on-machine-learning/9781491962282/). Although his neural network implementations are purely in TensorFlow, I adopted/reused some notations/variables names and implemented things using PyTorch only. I really enjoyed his book and learned a lot from his explanations. His work inspired this tutorial and I strongly recommend the book.\n",
        "\n",
        "We will be using Google Colab so we need to manually install the PyTorch library first. You can do this by using the following command:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hicY8Xs6_dI4",
        "colab_type": "code",
        "outputId": "af1911cf-b176-4a67-ea43-88e40f6292d9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "!pip3 install torch torchvision"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (1.4.0)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.6/dist-packages (0.5.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from torchvision) (1.12.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torchvision) (1.17.5)\n",
            "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from torchvision) (6.2.2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mXs_oWct-qxY",
        "colab_type": "text"
      },
      "source": [
        "Now we can import the necessary libraries we will use in the tutorial:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BBuAr4-L8CPN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import os\n",
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EMQfRrRl8CPV",
        "colab_type": "text"
      },
      "source": [
        "### RNN with A Single Neuron\n",
        "\n",
        "\n",
        "The idea of this tutorial is to show you the basic operations necessary for building an RNN architecture using PyTorch. This guide assumes you have knowledge of basic RNNs and that you have read the tutorial on [building neural networks from scratch using PyTorch](https://medium.com/dair-ai/a-simple-neural-network-from-scratch-with-pytorch-and-google-colab-c7f3830618e0). I will try to review RNNs wherever possible for those that need a refresher but I will keep it minimal.\n",
        "\n",
        "First, let's build the computation graph for a single-layer RNN. Again, we are not concerned with the math for now, I just want to show you the PyTorch operations needed to build your RNN models.\n",
        "\n",
        "For illustration purposes, this is the architecture we are building:\n",
        "\n",
        "![alt txt](https://docs.google.com/drawings/d/e/2PACX-1vQXBLYvvI1dqAHdLA0hQdsP1PojmCfuSCMK2DXEL0uTvRUqvD1eYK8fsECcNCoekxCbgWJ-k7QF_1s4/pub?w=600&h=400)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kHjE4j2x8CPW",
        "colab_type": "text"
      },
      "source": [
        "And here is the code:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qiY3EllI8CPY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class SingleRNN(nn.Module):\n",
        "    def __init__(self, n_inputs, n_neurons):\n",
        "        super(SingleRNN, self).__init__()\n",
        "        \n",
        "        self.Wx = torch.randn(n_inputs, n_neurons) # 4 X 1\n",
        "        self.Wy = torch.randn(n_neurons, n_neurons) # 1 X 1\n",
        "        \n",
        "        self.b = torch.zeros(1, n_neurons) # 1 X 4\n",
        "        \n",
        "    def forward(self, X0, X1):\n",
        "        self.Y0 = torch.tanh(torch.mm(X0, self.Wx) + self.b) # 4 X 1\n",
        "        \n",
        "        self.Y1 = torch.tanh(torch.mm(self.Y0, self.Wy) +\n",
        "                            torch.mm(X1, self.Wx) + self.b) # 4 X 1\n",
        "        \n",
        "        return self.Y0, self.Y1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cP94VBHE8CPc",
        "colab_type": "text"
      },
      "source": [
        "In the above code, I have implemented a simple one layer, one neuron RNN. I initialized two weight matrices, `Wx` and `Wy` with values from a normal distribution. `Wx` contains connection weights for the inputs of the current time step, while `Wy` contains connection weights for the outputs of the previous time step. We added a bias `b`. The `forward` function computes two outputs -- one for each time step... two in this case. Note that we are using `tanh` as the nonlinearity (activation function).\n",
        "\n",
        "As for the input, we are providing 4 instances, with each instance containing two input sequences.\n",
        "\n",
        "For illustration purposes, this is how the data is being fed into the RNN model: \n",
        "\n",
        "![alt txt](https://docs.google.com/drawings/d/e/2PACX-1vRpQYtOzO1U_3yQLf1885kMaja6MsXtJ8QnlqxrfpTgZmb4WpewJXphGdmotYXDB1VE6zlW6cBY_WqR/pub?w=600&h=600)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ydSxau_eFpwH",
        "colab_type": "text"
      },
      "source": [
        "And this is the code to test the model:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7a7-kIhj8CPe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "N_INPUT = 4\n",
        "N_NEURONS = 1\n",
        "\n",
        "X0_batch = torch.tensor([[0,1,2,0], [3,4,5,0], \n",
        "                         [6,7,8,0], [9,0,1,0]],\n",
        "                        dtype = torch.float) #t=0 => 4 X 4\n",
        "\n",
        "X1_batch = torch.tensor([[9,8,7,0], [0,0,0,0], \n",
        "                         [6,5,4,0], [3,2,1,0]],\n",
        "                        dtype = torch.float) #t=1 => 4 X 4\n",
        "\n",
        "model = SingleRNN(N_INPUT, N_NEURONS)\n",
        "\n",
        "Y0_val, Y1_val = model(X0_batch, X1_batch)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uNaFkJUP8CPj",
        "colab_type": "text"
      },
      "source": [
        "After we have fed the input into the computation graph, we obtain outputs for each timestep (`Y0`, `Y1`), which we can now print as follows:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q-teLNoV8CPl",
        "colab_type": "code",
        "outputId": "5101559c-2dad-4c71-b12b-58fbe854ba46",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "print(Y0_val)\n",
        "print(Y1_val)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[-0.9984],\n",
            "        [-1.0000],\n",
            "        [-1.0000],\n",
            "        [-1.0000]])\n",
            "tensor([[-1.0000],\n",
            "        [-0.2690],\n",
            "        [-1.0000],\n",
            "        [-1.0000]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n6tcX6ii8CP0",
        "colab_type": "text"
      },
      "source": [
        "### Increasing Neurons in RNN Layer\n",
        "Next, I will show you how to generalize the RNN we have just build to let the single layer support an `n` amount of neurons. In terms of the architecture, nothing really changes since we have already parameterized the number of neurons in the computation graph we have built. However, the size of the output changes since we have changed the size of number of units (i.e., neurons) in the RNN layer. \n",
        "\n",
        "Here is an illustration of what we will build:\n",
        "\n",
        "![alt txt](https://docs.google.com/drawings/d/e/2PACX-1vQov6BGg1fXOb7Bg5zenPh7R5j6VsZJh_D6JevQ_sm_fCxmXORxad3qLIFGG1FojzJig0qdcAQoGYoN/pub?w=600&h=404)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t5e2Eh5dGvnB",
        "colab_type": "text"
      },
      "source": [
        "And here is the code:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zcEqoDfP8CP2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class BasicRNN(nn.Module):\n",
        "    def __init__(self, n_inputs, n_neurons):\n",
        "        super(BasicRNN, self).__init__()\n",
        "        \n",
        "        self.Wx = torch.randn(n_inputs, n_neurons) # n_inputs X n_neurons\n",
        "        self.Wy = torch.randn(n_neurons, n_neurons) # n_neurons X n_neurons\n",
        "        \n",
        "        self.b = torch.zeros(1, n_neurons) # 1 X n_neurons\n",
        "    \n",
        "    def forward(self, X0, X1):\n",
        "        self.Y0 = torch.tanh(torch.mm(X0, self.Wx) + self.b) # batch_size X n_neurons\n",
        "        \n",
        "        self.Y1 = torch.tanh(torch.mm(self.Y0, self.Wy) +\n",
        "                            torch.mm(X1, self.Wx) + self.b) # batch_size X n_neurons\n",
        "        \n",
        "        return self.Y0, self.Y1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xlc4vEf88CP5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "N_INPUT = 3 # number of features in input\n",
        "N_NEURONS = 5 # number of units in layer\n",
        "\n",
        "X0_batch = torch.tensor([[0,1,2], [3,4,5], \n",
        "                         [6,7,8], [9,0,1]],\n",
        "                        dtype = torch.float) #t=0 => 4 X 3\n",
        "\n",
        "X1_batch = torch.tensor([[9,8,7], [0,0,0], \n",
        "                         [6,5,4], [3,2,1]],\n",
        "                        dtype = torch.float) #t=1 => 4 X 3\n",
        "\n",
        "model = BasicRNN(N_INPUT, N_NEURONS)\n",
        "\n",
        "Y0_val, Y1_val = model(X0_batch, X1_batch)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c9VNN7jf8CP9",
        "colab_type": "text"
      },
      "source": [
        "Now when we print the outputs produced for each time step, it is of size (`4 X 5`), which represents the batch size and number of neurons, respectively."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r1kiEzop8CP-",
        "colab_type": "code",
        "outputId": "85c311c2-e238-4de8-b81b-926cd41d5542",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "print(Y0_val)\n",
        "print(Y1_val)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[ 0.8637,  0.9878, -0.1758, -0.7462, -0.9552],\n",
            "        [ 0.9995,  1.0000,  0.6667, -0.9998, -0.9814],\n",
            "        [ 1.0000,  1.0000,  0.9454, -1.0000, -0.9924],\n",
            "        [-0.8953,  1.0000,  1.0000, -0.9836,  0.9999]])\n",
            "tensor([[ 1.0000,  1.0000,  0.9154, -1.0000,  0.9993],\n",
            "        [ 0.1450, -0.9337, -0.9938, -0.9316,  0.9942],\n",
            "        [ 0.9997,  1.0000, -0.8217, -1.0000,  0.9984],\n",
            "        [-0.6636,  0.9985,  0.0432, -0.9922,  0.2622]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BRKjsv2t8CQG",
        "colab_type": "text"
      },
      "source": [
        "### PyTorch Built-in RNN Cell"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Rcu2H6D88CQH",
        "colab_type": "text"
      },
      "source": [
        "If you take a closer look at the `BasicRNN` computation graph we have just built, it has a serious flaw. What if we wanted to build an architecture that supports extremely large inputs and outputs. The way it is currently built, it would require us to individually compute the outputs for every time step, increasing the lines of code needed to implement the desired computation graph. Below I will show you how to consolidate and implement this more efficiently and cleanly using the built-in RNNCell module.\n",
        "\n",
        "Let's first try to implement this informally to analyze the role `RNNCell` plays:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dp0Wjh4Z8CQI",
        "colab_type": "code",
        "outputId": "bc101fc6-9060-4240-a82b-b920a4ed3eab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "rnn = nn.RNNCell(3, 5) # n_input X n_neurons\n",
        "\n",
        "X_batch = torch.tensor([[[0,1,2], [3,4,5], \n",
        "                         [6,7,8], [9,0,1]],\n",
        "                        [[9,8,7], [0,0,0], \n",
        "                         [6,5,4], [3,2,1]]\n",
        "                       ], dtype = torch.float) # X0 and X1\n",
        "\n",
        "hx = torch.randn(4, 5) # m X n_neurons\n",
        "output = []\n",
        "\n",
        "# for each time step\n",
        "for i in range(2):\n",
        "    hx = rnn(X_batch[i], hx)\n",
        "    output.append(hx)\n",
        "\n",
        "print(output)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[tensor([[-0.6909,  0.5620,  0.2721, -0.2241,  0.0939],\n",
            "        [-0.5744,  0.4671, -0.4800,  0.9956, -0.7457],\n",
            "        [-0.5976,  0.5328, -0.9615,  1.0000, -0.7355],\n",
            "        [ 0.9859, -0.6745, -0.9990,  0.9984,  0.2917]], grad_fn=<TanhBackward>), tensor([[-0.7276,  0.3742, -0.9864,  1.0000, -0.8790],\n",
            "        [ 0.1334,  0.1078,  0.0161, -0.4180, -0.2246],\n",
            "        [-0.3839,  0.2674, -0.9703,  0.9991, -0.7527],\n",
            "        [ 0.7661,  0.0675, -0.9386,  0.9704, -0.3403]], grad_fn=<TanhBackward>)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cRNUCsXl8CQN",
        "colab_type": "text"
      },
      "source": [
        "With the above code, we have basically implemented the same model that was implemented in `BasicRNN`. `torch.RNNCell(...)` does all the magic of creating and maintaining the necessary weights and biases for us. `torch.RNNCell` accepts a tensor as input and outputs the next hidden state for each element in the batch. Read more about this module [here](https://pytorch.org/docs/stable/nn.html?highlight=rnncell#torch.nn.RNNCell).\n",
        "\n",
        "Now, let's formally build the computation graph using the same information we used above."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Opob45Zj8CQP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class CleanBasicRNN(nn.Module):\n",
        "    def __init__(self, batch_size, n_inputs, n_neurons):\n",
        "        super(CleanBasicRNN, self).__init__()\n",
        "        \n",
        "        self.rnn = nn.RNNCell(n_inputs, n_neurons)\n",
        "        self.hx = torch.randn(batch_size, n_neurons) # initialize hidden state\n",
        "        \n",
        "    def forward(self, X):\n",
        "        output = []\n",
        "\n",
        "        # for each time step\n",
        "        for i in range(2):\n",
        "            self.hx = self.rnn(X[i], self.hx)\n",
        "            output.append(self.hx)\n",
        "        \n",
        "        return output, self.hx"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oL1yBNis8CQa",
        "colab_type": "code",
        "outputId": "7f2ffb2d-e412-4792-a8bb-2687b59d3205",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "FIXED_BATCH_SIZE = 4 # our batch size is fixed for now\n",
        "N_INPUT = 3\n",
        "N_NEURONS = 5\n",
        "\n",
        "X_batch = torch.tensor([[[0,1,2], [3,4,5], \n",
        "                         [6,7,8], [9,0,1]],\n",
        "                        [[9,8,7], [0,0,0], \n",
        "                         [6,5,4], [3,2,1]]\n",
        "                       ], dtype = torch.float) # X0 and X1\n",
        "\n",
        "\n",
        "model = CleanBasicRNN(FIXED_BATCH_SIZE, N_INPUT, N_NEURONS)\n",
        "output_val, states_val = model(X_batch)\n",
        "print(output_val) # contains all output for all timesteps\n",
        "print(states_val) # contain values for final state or final timestep, i.e., t=1"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[tensor([[ 0.4909,  0.6246,  0.2240,  0.4163,  0.9287],\n",
            "        [-0.3733, -0.3016,  0.9961,  1.0000,  0.9759],\n",
            "        [-0.3570, -0.6732,  0.9999,  1.0000,  0.9993],\n",
            "        [-0.9648, -0.5048, -0.5303,  0.9903,  0.3518]], grad_fn=<TanhBackward>), tensor([[-0.6901, -0.8889,  0.9998,  1.0000,  0.9993],\n",
            "        [ 0.6467, -0.8708, -0.6372,  0.1005,  0.5848],\n",
            "        [-0.2918, -0.9729,  0.9811,  1.0000,  0.9881],\n",
            "        [-0.1899, -0.9033,  0.3249,  0.9862,  0.6411]], grad_fn=<TanhBackward>)]\n",
            "tensor([[-0.6901, -0.8889,  0.9998,  1.0000,  0.9993],\n",
            "        [ 0.6467, -0.8708, -0.6372,  0.1005,  0.5848],\n",
            "        [-0.2918, -0.9729,  0.9811,  1.0000,  0.9881],\n",
            "        [-0.1899, -0.9033,  0.3249,  0.9862,  0.6411]], grad_fn=<TanhBackward>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l240cp4g8CQh",
        "colab_type": "text"
      },
      "source": [
        "You can see how the code is much cleaner since we don't need to explicitly operate on the weights as shown in the previous code snippet  --  everything is handled implicitly and eloquently behind the scenes by PyTorch."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n3zn-Ydx8CQi",
        "colab_type": "text"
      },
      "source": [
        "### RNN for Image Classification"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eTJHs-S-8CQk",
        "colab_type": "text"
      },
      "source": [
        "![alt txt](https://docs.google.com/drawings/d/e/2PACX-1vSiMstqkE9hTYmhPD3KMeFRNNKYA2NnrCayahBOEL1TalRqaWF7rH8a7O-nP9c-mKOdZRsWtmAGZfNN/pub?w=969&h=368)\n",
        "\n",
        "Now that you have learned how to build a simple RNN from scratch and using the built-in `RNNCell` module provided in PyTorch, let's do something more sophisticated and special.\n",
        "\n",
        "Let's try to build an image classifier using the MNIST dataset. The MNIST dataset consists of images that contain hand-written numbers from 1–10. Essentially, we want to build a classifier to predict the numbers displayed by a set of images. I know this sounds strange but you will be surprised by how well RNNs perform on this image classification task.\n",
        "\n",
        "In addition, we will also be using the `RNN` module instead of the `RNNCell` module since we want to generalize the computation graph to be able to support an `n` number of layers as well. We will only use one layer in the following computation graph, but you can experiment with the code later on by adding more layers."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4vnhd2vR8CQo",
        "colab_type": "text"
      },
      "source": [
        "#### Importing the dataset \n",
        "Before building the RNN-based computation graph, let's import the MNIST dataset, split it into test and train portions, do a few transformations, and further explore it. You will need the following PyTorch libraries and lines of code to download and import the MNIST dataset to Google Colab."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6FViVH8w8CQr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torchvision\n",
        "import torchvision.transforms as transforms"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yrNdklIv8CQv",
        "colab_type": "code",
        "outputId": "4358c5d4-d002-4318-a653-863d541c1b93",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 383,
          "referenced_widgets": [
            "c194dfd9668b4c7e97a0b598ee810468",
            "56a4a9cb54764f7c9d858310441a55aa",
            "3eaae8eacaf048448ad11007109143e9",
            "bd2c425468d444e9931605adeaeaa78a",
            "b140724336ea4e04af134d48432f74d6",
            "f16d5d027aed4f8f87773bb7951616b2",
            "42fd164015de419a9c58be0f860c4439",
            "9ef3b6475ef049b1822ee119866bdc3e",
            "b7c63f595ac24762a3215071b14fd32c",
            "58e5e45c6ba7449bab80fd696286e072",
            "446d95ac8c6b4e1094fbf6703499f412",
            "023cf4ce24da49329796df463123d41f",
            "915c954de9f348278f22b68eec8cee63",
            "faef4bfc378447d892f6278cb410d1c5",
            "4a352572ee9b4682b70da3838d5117c5",
            "2563cdf52b8348bf89bfaa3ef19e8373",
            "8cdc0cb999bc4b16bdbe0e75e329481a",
            "26c8ff74123244e29a66b98557519883",
            "aecc1a9e4bc4478096278bf04bc4a7eb",
            "d4560bb340f441b89ce84f457bd7b48c",
            "8eec08b3b62642b9856b188a06762742",
            "f577c19d9e6b4699bbeca1b4ceb27bd4",
            "2323f246afa04fc5b0b099d8729426d5",
            "cfad60d6e3f7458fbfd33185e5bad018",
            "146b255c687b41f5ac178e918045d46d",
            "ef6b741c684546b184ca86813c2b02a0",
            "022f6fad73914b4cbbd65e48de239c2f",
            "b0f8d0bbd29d45b986b4895a447aff7d",
            "c1d3127d214545aea609ddcd7c19f1fd",
            "abd1c8b5ff7f4bf7a751a53bcacf5d5f",
            "8f59433e532748f8a33a47e84c31a281",
            "ee940fd1588a48668c2a34c8f26c1195"
          ]
        }
      },
      "source": [
        "BATCH_SIZE = 64\n",
        "\n",
        "# list all transformations\n",
        "transform = transforms.Compose(\n",
        "    [transforms.ToTensor()])\n",
        "\n",
        "# download and load training dataset\n",
        "trainset = torchvision.datasets.MNIST(root='./data', train=True,\n",
        "                                        download=True, transform=transform)\n",
        "trainloader = torch.utils.data.DataLoader(trainset, batch_size=BATCH_SIZE,\n",
        "                                          shuffle=True, num_workers=2)\n",
        "\n",
        "# download and load testing dataset\n",
        "testset = torchvision.datasets.MNIST(root='./data', train=False,\n",
        "                                       download=True, transform=transform)\n",
        "testloader = torch.utils.data.DataLoader(testset, batch_size=BATCH_SIZE,\n",
        "                                         shuffle=False, num_workers=2)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to ./data/MNIST/raw/train-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c194dfd9668b4c7e97a0b598ee810468",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting ./data/MNIST/raw/train-images-idx3-ubyte.gz to ./data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to ./data/MNIST/raw/train-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b7c63f595ac24762a3215071b14fd32c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting ./data/MNIST/raw/train-labels-idx1-ubyte.gz to ./data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to ./data/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8cdc0cb999bc4b16bdbe0e75e329481a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting ./data/MNIST/raw/t10k-images-idx3-ubyte.gz to ./data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to ./data/MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "146b255c687b41f5ac178e918045d46d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting ./data/MNIST/raw/t10k-labels-idx1-ubyte.gz to ./data/MNIST/raw\n",
            "Processing...\n",
            "Done!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ixg3XnDM8CQy",
        "colab_type": "text"
      },
      "source": [
        "The code above loads and prepares the dataset to be fed into the computation graph we will build later on. Take a few minutes to play around with the code and understand what is happening. Notice that we needed to provide a batch size. This is because `trainloader` and `testloader` are iterators which will make it easier when we are iterating on the dataset and training our RNN model with minibatches."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2f0fVaIV8CQ3",
        "colab_type": "text"
      },
      "source": [
        "#### Exloring the dataset\n",
        "Here is a few lines of code to explore the dataset. I won't cover much of what's going on here, but you can take some time and look at it by yourself."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TEKzVTOY8CQ4",
        "colab_type": "code",
        "outputId": "50ff5622-a7e5-4e27-9a2c-873e5d7256f0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "# functions to show an image\n",
        "def imshow(img):\n",
        "    #img = img / 2 + 0.5     # unnormalize\n",
        "    npimg = img.numpy()\n",
        "    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n",
        "\n",
        "# get some random training images\n",
        "dataiter = iter(trainloader)\n",
        "images, labels = dataiter.next()\n",
        "\n",
        "# show images\n",
        "imshow(torchvision.utils.make_grid(images))"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAD8CAYAAAB3lxGOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOxdeXhM1xv+JquEkI0ITUXUvu9bkbYh\ntqJItFpb6YZSFLHHUjsttVVQWpSofacEiVCJ2JcEQUIiq+zrnPv+/og5v5nMhGTm3qSR+z7P+2Ry\n753vu8s575ztfp8CAMmQIaPswqikT0CGDBklC1kEZMgo45BFQIaMMg5ZBGTIKOOQRUCGjDIOWQRk\nyCjjkEwEFApFd4VCEapQKB4qFAovqfzIkCHDMCikWCegUCiMiSiMiLoS0TMiCiKizwDcFd2ZDBky\nDIJULYE2RPQQQDiAHCLaRUR9JfIlQ4YMA2Aikd3qRBSp9v8zImpb0MEKhUJetihDhvSIB1A5/0ap\nROCNUCgUXxPR1yXlX4aMMoinujZKJQLPichJ7f93Xm3jALCRiDYSyS0BGTJKElKNCQQRUW2FQlFT\noVCYEdGnRHRIIl8yZMgwAJK0BAAoFQrFWCI6SUTGRLQFwB0pfMmQIcMwSLZOAMAxAHUA1ALwk1R+\n3kZYWFjQrFmzKCkpiVq0aFHSp/PWwMLCgvbt20eMMbp06RI1atSopE/pvwEAJU4igsz/s23bthAE\nAYIgYNCgQSV+PvrQ3d0d9+/fR4UKFXTut7Kywq5duyAIAqpUqSL5+VStWhULFiwAYwzx8fEYOHCg\nwTadnZ2xaNEiIK8Qa7FTp04YMGCAaNdgZ2eHH374AQB4+VDxwoULGDNmzJtsBOusfyUtAPqKQI0a\nNRAZGclvSHZ2NmbMmAFzc/MSrwCGsiREwNbWFmvWrEFCQgL69OljkC1ra2tERkaCMVZgBXdwcABj\n7LXHiEU3NzcIggDGGNLS0tCoUSODbb777rs4e/YsGGMQBAEtWrTQ2D9r1iykpKTg7t27olyDjY0N\nAgICwBhDbGwsYmJikJmZye+hIAjIycnBiRMnYGFhUZCdt0MEFAoFmjZtitDQUH4DEhIS+OfvvvtO\nssLUsmVLeHt74/bt2xAEAadOnZLET8+ePYtFBMzNzbFgwQIcOHAA9+/f5z4DAgIMsmtiYoKjR48W\nSgSOHDkCExMTSa7PysoKP/zwA54/fw5BELB37160bNnSYLsmJiYYPnw4GGPIyMhATk4Ohg4dyvdX\nqlQJ8fHxYIzhyZMnBvuzs7PjAsAY49vbtGmDPn36oE+fPnj8+DHfb2dnV5Ctt0MEBg0axC92+/bt\n6N+/Pxo1aoTBgweDMYbExESsWLEC5cuXN+jGN2zYEEuXLsXSpUuxbNkyREVFITMzk/+iqJqVUhTe\nP//8E4IgIDExUdRfSQsLC4wYMQLffvstjhw5gsTERK1mpRgiQETYtGlTgSJgYWGB06dPgzGGjh07\nSnIPiQjnz5/nz2rw4MEwMjIy2Gb9+vVx8eJFPHr0CLGxsahSpQpGjBiB27dv82OaNm3K/YohAmvX\nruX21EVAnTdu3CgbItCvXz88evQIjDFcv35dY5+xsTH27dvHb8Ty5cv1uuG9e/fG9evX8eLFC42m\nlqopuWfPHqSmpoIxhq1bt0pSeP/66y8IgoCjR4+KZrNChQq4deuWVoVPTk6Gj48PevbsCV9fX+Tm\n5uKrr74y2J+3tzf8/f1Rrlw5rX3t27dHdnY2GGNo27atqPfOysoKK1aswLVr15CVlYXDhw+jSZMm\notmfN28eGGN4+vQp+vbtC6K8X+SsrCw0bdoURIQJEyaIJgK9e/dGdHQ0GGPYuHEjEhMTtY6pU6cO\nL6MbN26EqalpQfZKtwj06NGD9zPT09PRqVMnjf2mpqY4efIkv/k9e/bU66Z37NgR27Ztw9atWzm3\nbduGrl27olGjRqhWrRpevnwJxhi8vb1FLcAqqirorFmzRLM5Z84cblfVr9y9ezesrKxAlPfrFRMT\ng8uXLxvsq3nz5ti5cyd27Nihc/+oUaPAGMPNmzcLHDjUl9999x0vA6NHjxbVdpMmTZCWloZ79+5p\ntXCysrJw4cIFlCtXDmfOnOHncOzYMb39WVpa4saNGwCAW7dugSivwqsf061bN0RHRwMAHjx4gHff\nffd1Nku3CFy4cAGMMeTm5uocuKpQoQK/8f7+/jAzMxO1AKj4xx9/QBAEREVFSWKfiCQZD5g/fz6e\nPn2KCRMmoH379lr7V61aBUEQsH37doN92dvbIyUlBRkZGXj33Xe1fplUFXXVqlWi37uJEydCEAT4\n+/uLbvvatWu4fv261iAgEeHixYtgjGHAgAH8R4Ixhg8++EBvfx4eHmCMITs7G59++qnW/p49eyIn\nJweMMTx48ADvvffem2yWThEwMjKCt7c3cnNzkZubi/nz52sdY2xsjB49eoAxBqVSqXcroDA8dOiQ\nZAVYRZUIWFtbS+ZDnTVr1kR6ejqys7PRu3dvUWzu3LmTV4Tz589j+vTpWLFiBcaMGYOEhAQkJSWh\nVq1aol5H27ZtERERgaysLLi6uopq29vbG0qlEl5eXjr3q0RA1c1hjCEwMNCgcYjFixeDMYY7d+5o\ntZhUA56MMYSHh7+pBaBi6RQBb29vMMbw8uVLuLm5aRXeAwcOaAya6FJMsXjixAkIgoCTJ0/yZrQU\nFAQBjx49gqWlpWQ+VGzYsCEyMjLw4sWLwvySFJrOzs58hkAXo6OjRb+W4OBgrW6ah4cH1q9fDxcX\nF73tXrx4EU+fPi3w/vz2229gjOHs2bPIyMjg1yjGAKuXlxcEQUBWVhZ27NiBcuXK8S5deHh4Ua+r\ndIrAnj17wBjDyZMnNdYA1KlTh0/DFIcIVKpUiU81TZo0SRIfRITGjRtDEAT89ttvkvlQZ2BgIARB\nwIEDBzBy5EjMnj0bf//9N27cuME5b948ve23adMG48ePx/jx4/H7778DyFvX8eLFC1GvY8iQIRAE\nAQkJCXBycgIRaQyABgcH672GJCYmBj///DOI8tYHfPnllzhy5Ai3DQCTJk2Cs7MzVqxYwbcvWbJE\nlGv7+++/uc1Dhw4BALZs2VLYX391lk4RWLRoEa/gV65cgb+/P/z9/fno/T///AMfHx/JRWDz5s1g\njOHSpUuvm4IxmJ9++ikEQUCbNm0k80GUN5DatGlTnVOEKkZGRmLz5s2oVKmSaH6fPHnCR7HFvJ7f\nf/8dgiAgPj4e69atQ1RUlMaPw/Lly/Vumqtmhvbu3YuHDx9q/fD4+fnxY1ULiBhjqFmzpijX1rZt\nW973Z4xh4sSJ+o55lU4RcHFx0Rj1V6dqhaBqSubZs2eoWLGiJJXm2bNnYIyJstz0ddy/f79kImBs\nbIx27dphxYoVePTokc6Kf+vWLWzevBmNGzeWZEwiLi4OjDFMmzZNVLsqEdBVTq5cuQIbGxu9beuy\nGRUVhWXLlqFdu3Y6r48xhm+++UaUa6tfvz6Sk5O53Ro1auhrq3SKAFHeyjYHBwd4e3tj6dKlcHBw\ngIODA9+vEgExFma8riA8ePBAMvsqBgQE4MqVKzA2Nhbd9vz587UqfUZGBk6dOoWvv/4aNWvWfN0c\ns2j3UQoRcHR0xKlTp7QqqxjLhI8cOcLtPXjwAAcPHkTXrl11Hiu2CLi5ueHff//VuKYvv/xSX3ul\nVwTexL1794IxhrFjx0pScO3s7CAIgmjK/joGBASIMlefn+XKlUNGRoaWCNSvX1/ya1KnVCKgonr/\n+dKlSwa1AFRUKBSwtbVF69at3zhYK6YIuLm54eXLlxAEAenp6ZgyZQrS09NlEdDF6OhoZGRkiPJi\niC4uXrwYSUlJhjTDCk2pREChUKBTp044deoUFi5ciNjYWGRmZkp+PfmpqiCGjNbroouLC7p3784r\nYXx8vMFLx/WhWCJQv359LgABAQFo3bo17Ozs8PLlS4wcOVJfu2+nCHTp0gUZGRlITk6W7MEKgiDp\nugB1SiUCKlpaWuL8+fPFOgOhTlUFEXP6s1u3bkhKSuJjAsePH+czBMVNMUTAyMiIT69eu3YNlSpV\ngpGREebOnQvGmCEDtTpFoMQCjYoFZ2dnMjc3p9zcXEnsu7i4EBHR5s2bJbFf3OjQoQN16tSJbty4\nQWPGjCnp0xEFJiYmZGVlRRcvXqTg4GCaPXs2paamlvRp6Q1bW1vq3r07ERHFxsaSq6srtWjRgmbO\nnElERMnJyeI6LOlWgFjdAZX6xsbGomHDhqIp+4EDByAIgtaa7dLIjz76CNnZ2aKvqS8KGWPYuXOn\nKG/0/Rd5+fJlURYL1a9fH+PHj+etNl9fX8yePdvQgdu3sztARJg2bRoYYzhx4oSob6VZWFjgxo0b\nEAShxAuXoTQ2NkZQUBB8fHxK/FxklhjfXhGQkuvXr38rRECmTCpABCTJRVhUyHkHZMgoFlwF0Cr/\nRjk1uQwZZRyyCMiQUcYhi4AMGWUcsgjIkFHGIYuADBllHLIIFBKbNm0ixpgWAwMDydLSUhKfAIgx\nRi9evKCmTZtK4qO4sGvXLn7PTp48SWZmZiV9SjJeQRaBQmLEiBE6ty9fvpwyMjIk8SkIAgEge3t7\n6tSpkyQ+igP29vZUq1Yt/r+joyMZGclFT18AoOrVq4tmr9Q9ifr169Pz58/p+fPndOvWLRo2bFix\n+L1+/Tpdv36dkpKSNLaPHDmSrK2tJfG5d+9eIiLKycnR8isGypUrR4mJiVxsOnfuLLqPKlWq0Jdf\nfskTq969e5c++ugjysrKEt2XCqNHj6bnz59TUlISTZw4UTI/JYFq1aqRIAg0a9Ys8YyW9GrBoqwY\nHDJkCI+woh5FZuLEiZIHw1CxWbNm+OqrrzTCSrdu3VoSX6royffu3cOrBVWi0dLSEps2bUJ0dDSW\nLVuGnJwcdO7cWfRrcHNzK5YYkB06dMCIESMQHR2N3NxcDZ87d+5Ehw4diqV8SE0nJycwxvDXX3/p\n8/3Sv2z47t27YIxh165daNKkCdq0aYOrV69CEAT06NGjWB9GkyZNcOzYMTDGsH//ftHz6XXp0oWL\ngFKpFDVgioWFBQ4fPqwR1jwzM1N0EbC1tcXBgwd5ZYyJiRH9OdjY2GD37t0a+Sh1MSEhAe+8845e\nPszMzHhA0c2bN2P16tVYvXo1hg4dijp16nDOmTMHq1evxr59+zBt2jTRhZvo/yKwfv16fb5fukWg\nZcuWyMzMxJUrV7QCfW7YsAEhISGi3/A30draGk+fPgVjDN26dRPNbuvWrbF//34uAg8fPhQt2KdC\nocCkSZMgCAJWr14Nov9n7Z0wYYJo11CpUiUcPnyYV8KnT5/C3d1d1PtvY2PDo0rponqSTsYY+vfv\nr1fGIxMTEy1bhaEUrdMxY8aAMYbq1avr8/3SLQLdunWDIAjo16+fzv1ff/216De8MAwLCwNjTNQo\nNkePHoVSqQRjDMnJyaJemyrteWhoKM+d0L9/fwiCgKVLl4rmZ+jQoRoV4tKlS6Lf+969e7+2Etap\nUwdbt27V2Pbrr7/q5UuVFHT9+vVwdnaGs7Mz5s+fD3d3d7i7u/NtHTp0AGN5mbKkEIHdu3cDQNkV\ngVu3bhXY7G7YsCHCw8OL9b3/+vXr80gyYorAo0ePuAgcPnxY1HM+evQoMjIyePJMIsLq1ashCAJG\njBghmp+///6bVzxdSTLMzMzg4eGB9u3bo0GDBnp1p7p3764Rijs/O3XqBHNzc41tfmrhwYtCW1tb\nXLx4EdnZ2a8dX1CFyF+wYIHo5U09RXmZFYGCmvzm5uZYu3Yt4uPjC4wCKwUvXrwIpVKJtWvXitb/\nU09rDUDUVF0NGjRAVlYW1qxZo7F97dq1ouc+VIVo19UKcHV1xYkTJzQq54cffqiXn+nTpxcoAhER\nEejbt68oIkBEqFy5MqZOnYqVK1cWeIyvry8YY6IGtlGxffv2/MehTIoAEeHMmTMaQUNUvy7vv/8+\nD8oohgiYm5vD0dER7u7uWLVqlUZ+OcYYDhw4wAv5qVOnRIuXV7lyZdy+fRtKpVKSNF3Lli3Tukd+\nfn48Ou/06dNF86VKIc8Y49l7iPJyORaUH4AxhiFDhhTZl4mJCVxcXLB9+3bOVq1awc7ODsePHxdN\nBN7EqVOnGjJy/0b+J0WAiJ4Q0S0iuq5yQES2RHSaiB68+msjlghMnz5dI/1StWrVuBBERUUhKytL\nKxlEUejo6IiZM2cWmOwkP+/cuSNaNqLKlSsjJCSEDwZKET69RYsWyM7Ohr+/PypWrIimTZviypUr\nGDZsGARBEHV2QL0loPqVt7S0BGP/n94NCwtD06ZNNbJM6SMCBfGdd97RemZSiUDlypURHh6OtLQ0\nyaaM27dvz1uIYoqAGIuFPgDQDP8PVuBFRGcA1CaiM6/+FwU///wz1a9fn0JCQqhhw4ZkbW1NAwYM\nIGdnZypfvjz5+/vT5cuX9bJtbm5O33zzDc2dO5fc3NwoKSmJcnJy6Pbt27Ru3TpaunQpnTt3TuM7\n9erVox9//JFMTU0NvrY+ffpQ06ZNycjIiLZs2UJr1qwx2GZ+hISE0NixY6l169aUlJRE169fJ3d3\nd3JyciKFQiG6v/ywsbHR+D8yMpJcXFyof//+kvirVq2aJHZ1YciQIVSjRg2Kjo6moKAgyfwAIEEQ\nxDdqYEvAPt+2UCJyfPXZkYhCxWoJEOU11Xfv3o2OHTvybaoUznv27NFLYW1tbXHu3Dkwlpf9+Kuv\nvkKzZs3QuXNnnsRy3rx5iIiIAGMMN2/exM2bN8FYXlxDfRNdqli5cmX8+++/vKkn9lRaftatWxeT\nJ0/mKdVUA4NitgRUuRtVXSY3Nze+TVd3ICgoCDt27IC9vb1o5xAeHq7lR6pZpF27doExhuHDh0v2\n3P6r3YHHRBRCRFeJ6OtX25LU9ivU/xdDBHTxt99+gyAIaNKkiV7fX7NmDRhjCA4O1jkF2a1bNzDG\nkJqailGjRsHGxga2trYIDQ1FdHQ0bG1tDTr/Hj168EVBjDFRK0JhOHv2bNFFQJVSXhd1iYC+A4MF\nsVGjRlo+rly5gqpVq0pyDy9duoQDBw5Ikj5Oxf+qCFR/9bcKEd0gos6Ur9IT0csCvvs1EQW/ot43\npmXLlrxQ6SsCqvn4iRMnamzv2bMnlixZgtzcXAQHB2uNntetWxdDhw41OHGnj4+PhghIVYgKomoN\nhpgi8O677+LcuXNIT0/XqowAwBhDaGgofH190a9fP1SuXFk0340aNcLz58+1/Iqd9Sh/Gdq0aZOk\nz+k/KQL5KrU3Ef1IEncH1Fm+fHlcunQJgiAgJCQEFhYWej/ArKwsLFmyBC1btsSCBQsQFhaGrKws\nXoDETM+dn6oFR4wxXL9+XdKCpIsqERBznYCK48aN42nkVUxJSYGvr6+oWYjUuWrVKi0BuHXrlqQt\nLMYYPDw8JH9W/6nFQkRUnois1D4HElF3IlpGRF6vtnsR0VKpRKBq1aq8efnJJ5/ofWOvXbumUUDV\nC8/GjRslS2mlUCiwcOFCZGZm8pbAL7/8InlByk+VCIi5YlCdDg4OGkk5AgMDDR5H0cWGDRsiNDRU\nawHRnTt39K00hWZUVBSftZKSqm7Of0UEXCivC3CDiO4Q0YxX2+0ob1bgARH9Q0S2UonAgAED+Isd\n6qnKi8r8vxxhYWFYsGABPD09JXkJREWFQsErv4qzZs2SvCDl5/Dhw0V/dyA/7ezsEBwcDMYYXF1d\nRbdfvnx5REZG6hyDmDt3ruT38MKFC8XyrFQrIP8TIiAm9b0hqof8uhVcMt/MU6dOISMjA87OziV+\nLoawRYsWSExM5OXi7t27xea7uESAiEQXgVIXVEQXVME3ZOiHx48fU1BQEEVFRZX0qRiEkJAQ6tq1\nK/31119ERDRgwIBi821ubk7m5ubF4svY2JieP38umj05A5EMGSIgISGB2rRpQ48ePSrpU3kd5AxE\nMmRIhZ07d/7XBaBAyC0BGTLKDuSWgAwZMrQhi4AMGWUcsgjIkFHGIYuADBkiYcWKFfRfGGMrKt5a\nEejQoQP5+/uLZm/BggWkVCpp8ODBotmU8fbA19eXJk6cSJGRkZL7cnR0pKioKPr444/FMVjSqwUN\nWTFYEJ2cnHD79m1ERESIZjMnJ4dT7PMtbtarVw+pqam4d+9esfizsrLCxYsXcfz48RK/dqmogqen\np+S+GjRoAEEQ4OXlVdTvvn3LhnXRyMgIM2bMQFxcHOrVqyea3eIUAXd3dwDAqVOnJPOxY8cOMCZd\n9iR1Dhw4kL8bIbWvkqIKxeFr5cqVAKBPaLu3d9mwOmrXrk3z58+nFStW0P3790v6dPTC559/ToIg\nUKdOnahLly6S+AgJCSEios6dO5OxsbEkPlS4c+eOpPaLiiZNmtDw4cPpyJEjNHnyZIPtOTk5ERHR\nypUrDbZVGHzyyScEgJRKpTgGS7oVIGZLoFy5crh16xb+/fdf0dVX/U2/oKAgSZVe9TpzbGysqPkM\n1Fm5cmVERUVBEASDIyO9icuWLftPtARat26No0ePIjc3l0dYPnHihMF2IyIiRO16vomCIODGjRso\nV65cUb/7dnUHGjVqpPX+9vLlyxEbG6t3hKHXUb07cOXKFckesIuLC38L7sWLF5IWJlVAFqlFoFmz\nZnjy5AmUSiV+/PFHSX3lp7GxMYYMGYKDBw8iISGBV/7c3Fz4+fkZHBVqxYoVACBZzAldFAQBixcv\n1ue7b48INGrUCKdPn4aRkRHf1q5dO2RkZKB///6S3Hh1ESgoFZqhbNmyJU+6WhwiMG7cOAAQNY9i\nQWzWrBmUSiVWrVoluS8iwowZMzB79mwEBATwii8IAjIyMuDv7y9KfgonJyeoUBzXRJSXY0GpVOqb\nZfntEAFnZ2dERUVpDJrZ29vj7t27+OOPPyRLUS71wKC9vT3S0tLAGOP586QWAUdHRwiCgOzsbNSs\nWVNSXxUqVEBMTAzCwsIkbXmYmJjgxx9/5BGnVHz8+DHGjRsnaiBQFSIiIuDp6QlPT0+0b99e0vs4\nbdo03LlzR9/vl34RMDU1xZkzZxAbG6sRL27KlClQKpWSjnSr+rTNmzcX3Xb37t25ACQkJKBt27Zg\njCE9PR1t2rQBUV7uvubNm2PZsmXYvHmzKH7t7Ox4kNb33ntP0sJLRPjpp5+gVCpRrVo1Seybm5tj\n3LhxvOL3798ff/31F8aOHSt6FGBfX18uAuotAtX4AAD4+vqK3k3YtGlT2RUBKysr3L9/Hz4+Phrb\nd+7cifv376NKlSqSFmBVK0AsEahQoQK2bt2qFY1XPTKOipmZmUhISMCNGzcwd+5cg0KpqdPMzIyL\nwJYtWyS9f0TSisC8efO0mv1mZmai+2nfvj2v8CtWrOAVXtexnp6eAIDAwEBRn5ebm5u+Nkq3CMyd\nOxf379/XCE3drVs35ObmYvLkyZIXYDFFwNHRUSNN1+u4evVqSZuYACAIAjZs2CD5PVy4cCEYY5KI\nwL179zREQBAE/PnnnwYP/OVnYGCg1i9+QSKgur+v218Ufv/99xAEQa8Mzq+oUwRMqJRg5MiRpFQq\nafny5XTgwAG6cuUKLVy4kP766y9atmyZpL6DgoLIyEi8JRVWVlbk6OjI/z99+jTFxMRQXFwc+fr6\nUsOGDWnTpk1ERBQQEECXLl0SzXd+qApCfHy8ZD7y+5ICgwcPpvr169OTJ0/4tr1799KcOXNoypQp\nlJuba7APT09Pat++PRH9f20AEdGgQYN0Hu/r60tEJNrz69u3LxGReOsDVCjpVkBhWwJubm6IjY3l\n0YVVar9792588skn+OSTTyQL+XzlyhVRWwLGxsZo1qwZpk+fDmtray1lV40JMMYkX4aq6g7MmTNH\nUj9E0o8J5GdcXBxSU1NRp04dUezpgq+vr9Zx7du3560EMdcP3L17F4IgGGKjdHcHiAg2NjZo06YN\nvvnmGzx//hwXL15EeHg4BEHA06dP0bJlS0kKk9gi8CaWhAhcvnzZkGZmoVjcIhAbG4v09HRRlo+r\n+vfqUPX1nZyc4OnpCV9fX40uwooVK0S7lsaNGyM1NRWhoaGG2Cn9IqBitWrVkJWVVSwFiejtFoFH\njx5xX6qZCKkotgj07NkTGRkZ2LBhA4YOHcq3Ozg4wMfHh08NiuFLfQbgTfD19RV1HMfIyAjbt28H\nAKxfv94QW2+HCJQvXx779u3D/fv3JS2w6gwODpZ0ijA/i1MEgP93rUqbCFy5coVXvLS0NMTExCAm\nJgZJSUkQBAFLlizR5yWbAqmrNQDkNfl9fX0le1bW1ta8xdajRw9DbJXugUEiIiMjI1q9ejX169eP\nmjZtWmx+9+7dS02aNCk2f8UJVUGIi4ujp0+flvTpFAlTp06lf/75h4yMjMjS0pIsLS35Pj8/P5o6\ndaqo/lQDfJcuXaIOHTqIarswyMrKouPHj4tut1SJQPfu3WnEiBG0fv36Yn0zbdGiRTR37txi81ec\n2LFjBw0ePJiSkpIoJiamWHy6u7vT77//brAdPz8/atGiBdnZ2VH37t2JiOjp06dkZGREa9asMdh+\nfkRGRpJCoRDd7ptQtWpVIiL66aefpHFQ0l2BonYHygKLsztARAgKCiqW2YEuXbqgV69eJX5/yzDf\njjGBssBy5cphxowZ8PPzQ4sWLUr8fGS+NdQpAnLyERkyyg7k5CMyZMjQhiwCMmSUccgiIENGGYcs\nAjJklHHIIvAfQ6tWrWjBggUlfRoyyhBkEfiPQfWqqgwZxYaSXiMgrxP4Pz08PMAYQ6tWrUr8XKSg\nq6srOnfuXCy+LC0tsXPnTixcuBCPHj3CqVOnDA7f3qJFC6xZs0Yj6MvSpUvRsGHDEr+3haR+i4WI\naAsRxRLRbbVttkR0mogevPpr82q7gohWE9FDIrpJRC2kFAEfHx9s3LgR9evXl+SmmZqaomPHjli7\ndi2A/79os2bNGnTs2FF0f5MmTQJjTLJcA4Wlg4MDxo4di3PnziE5OVk0u66urjh58qTk51+9enU8\ne/ZMK9LQtGnTDLLr5eWlkX9CxYiICHTv3r1En1khqbcIdCaiFqQpAkuJyOvVZy8iWvLqc08iOk55\nYtCOiP6VSgSGDRvGH64Ub2gWT/kAACAASURBVBS+8847PFWXKnJt/tBf3t7eovoMCQkBY0z0oJhF\nYa9evRAYGMivUcxXtp2dnXHr1i3Jr+Hjjz/mZePRo0eIj4+HIAiYNWuWQXYbNGiAhw8fIiEhgTMr\nKwtKpRLR0dH46KOPJL2u2rVrQxAEPH/+HHXr1tXHhv7LhonImTRFIJSIHF99diSi0FeffyOiz3Qd\nJ5YImJqa4tdff+UP+dKlS/j9999Fuclt27ZFz5494ebmhj179mgE/Xz8+LGWCPj7+4v2gD/88EOE\nh4eDMSZ5JSmIR48e1Qp8KuY7BdWrVy8WEXBwcEBQUBC8vb1RsWJFbN68WRQR0MXz58/zFkFCQoJk\n1/T+++/jyZMnGuIcGRlZ1EQ7oopAktpnhep/IjpCRO+r7TtDRK3EFAEvLy8uANu2bYONjQ1q166N\ndu3aceqRnomv1x83bhwmTJiA8+fPY8iQIVi9ejVCQkJw4sQJ7NmzR6OCiBX6m4hw4cIFMMaKNU4C\nUd6v84ULF/Dvv/9qCcDLly9FfXehuERAnb1790Z0dLRkIlClShVMnjwZSqUSOTk5mDJliug+vL29\nkZ2dzZ/LkiVL+A/G8ePHi9J9lEYEXv3/sqgiQERfE1HwK77xAsqVKwdvb28eTuzRo0ewtLTE4sWL\nERERodH3O378OObNm1ekG12uXDmePOLly5eIjo7GyJEj+X4HBwcMGjQIPj4+/GGEhYWhUqVKolSO\nBw8egDGGTZs2FVsFMTY2xsOHDzUq/q1bt3Ds2DHcunULjDEcPHhQNH+zZ8/G7du3i+36GjRogJiY\nGN6V++yzzyTzpWoNLFiwQFS7derUQVRUFH8+qqQ76gOURRiYLN3dgXbt2mlUdDMzM2zcuJE/4ISE\nBHzzzTecffr00eumd+7cGYIgICAgQCtwab9+/TB16lS8fPmSBzzVMyecBg8cOMAfqFSDnPlZr149\nHDp0iN+/0NBQ+Pr6wsbGBr169QJjDNnZ2Rg8eLBoPnfv3l2sLYEXL17w8pKYmIi2bdtK5ksqEVAf\nn9m4cSP/1f8viMAy0hwYXPrqcy/SHBi8Ukj7b7wAlQhkZmZi5syZMDY25g84f0ISQ7h48WIwxjB8\n+HCNATpra2u0aNECjDHcu3ePDxYuWrTIIH/NmzfXiJxc0HEWFhZwd3cXZdBwxIgRvOUhCAJOnjwJ\nS0tLvn/Xrl1IT0/H8+fPRQ0KOnny5GITgQMHDmjNDjx58gQVK1YU1U/Xrl1x8OBBSUSgb9++vIt2\n4sQJVKhQge9TF4GePXsW1qbeswN/EVE0EeUS0TMiGklEdpTX1H9ARP8Qka3a+MBaInpERLeoEOMB\nhRUBPz8/CIKA0aNH822CIODXX3/VSEwqxkNVVY5evXqhbdu2GD9+PH777TdERERoNJ3FEIFOnTpx\ne7t27dLab2JigtGjR/OBSUP6nCYmJmjQoAG2b9/OfY4ZM0Zr7n7cuHF4+PAhEhMT8dVXX4l2b11d\nXYtNBG7dugVBEHDv3j0EBATg+fPnEARB1BiRw4cPR2JiIheApKQk1KhRQzT7vr6+PCellZWVxr5/\n/vmHDxAWYaagdAcVuXDhAgRBQEREBK5du4Zr165BEARMnTpV9AUoAwYM4M3kuLg41K1bF1u2bEGt\nWrW0ZgcMGRMwNzfHqVOn+CBc/ofZrFkznVmJ9MkbWLduXURGRvJfxaSkJJ0h2q9cucK7OmIOfBIR\n+vfvX6xjAur09vaGIAhYunSpaDYPHTqk8VzS0tJE7XLMnz9f5+Ix9VmC6dOnF8Vm6RaBIUOGaDXv\nVMzMzMTp06fh5eUlSnN5wIABAKDhY+/evahUqRLc3Nxw7NgxXlHUBw+Lytq1a/OHeeTIEY19FhYW\nOqcl//zzT42me2G5fv16buPevXuwsrLSytUXHByM3NxcfozYi5aKe0xAnWvXroUgCJg/f75oNq2t\nrbFr1y6NhUPR0dGiLST7+uuvwRjDhAkTQERwcXHRWrFYpkTA3t4eY8eOxYMHD7B161ZERkYCAGJj\nYxEVFcUr66RJkwxKojFz5kykpaXxlkBmZqZWBW3ZsiWysrKQkZGBo0eP6u3rdSIwevRoLQEIDAzU\n6BcWlh4eHnwcIyYmBgsXLtTYX69ePfz8888a01BDhgwRvSLu3r0b9+7dE2VGpSh0dHREcnKy6M11\nojwh+PTTTzWE4OnTp2jWrJkotp8+fcrzJ+j6AZwxY0ZRbJZuEdBFVYbWChUqYOLEicjNzYUgCHBx\ncdHLnoeHBzIzM3l///79+xg4cKDOY/39/fHkyRMkJyfrnf5MXQSSkpIwduxYNG7cGJ6ennjx4oWW\nCCxfvrzIPmrVqoX4+HhuY9CgQRr769ati99++43vP3PmDHr27KnVBxWDu3fvLnDsQ0qqugIbN26U\nzEd0dLSGEBRhsO61nD59ulam6hUrVpTNloAu5k/TPGvWLL1FwM7OjgtAQkIChg0b9tpm95EjRyAI\nAtLS0tCgQQO9zt/BwUEjAxBjjFfY/MuUo6KiULt27SLZd3Z2xurVq7mN7du3a4yOm5iYIDIyEowx\nPHv2DD///LOk7y2IKQIVK1bEyJEj35hnsEmTJsjIyIAgCOjWrZvo11SxYkX06tULmZmZGiJw9epV\nUX3Y2NigadOmsLGxgampKRISEmQRINIUAWNjYxw8eFBvEbC3t8fJkycxZcoUdOjQ4Y3Hf/nll7yi\n6rsmgShvuk61+iv/zIP6ykRbW9si2/bz89OwOWzYMLRp0warVq3C6dOn+YzLvn379BayovD7778H\nAIwZM8ZgW3fu3IEgCEhISCjQXpUqVRAWFsankcV+J6N///7w8/PT+VJRcHCwZPfR1dWVj90UscXx\ndorAP//8A6K8kXZVP0kfEShXrhzP//7TTz+98fiAgAAA4OmuDHmotWvXxoIFCxAdHc0rbHZ2NhYs\nWID27dvrPcZRp04draXA+blu3bqirj83qPAyxtClSxeDbaky9KrEUtWNUVV0U1NTPoMUFRUl+lhA\n9+7dkZ6erlMAxOwO6KKnpyd/fh988EFRvvt2ikBaWhpmz57Np7YMWeDSt29fMJb3stCCBQtgb2+v\nNYJORHB3d+fLl/38/FC9evViqUT6cNWqVcjJydGo+NnZ2YiIiMCaNWuK9Vy2bt2Ku3fvwtTU1GBb\njRo1wqlTp7QGym7cuIGQkBDcvn2bb5NiufCgQYO0ugBiDwwWxM2bN4MxhosXL+osn6/h2ycCly5d\n0ioEhmRtrVq1Km7cuKHRHL906RI8PT3Rt29feHp6Yv/+/RqDh+pjEv9VDh8+HGPGjOEUcylwUSj2\nFKGFhQXGjRuHlJQUnSPnt2/fxhdffCHqYjJ1fv/997h3756WCKxbt07S+6gSgW+//bao3337RIAo\nr2kUEhKC3NxcvP/++6Ld6E8++QQ+Pj54/Pgx7t27Bx8fH/j4+ODTTz/lYwdSPmiZMgtieno6cnNz\n4ejoWNTvyhmIZMh4G3DixAmKiYmhYcOGFfWrOjMQySIgQ0bZgZyGTIYMGdqQRUCGjDIOWQRkyCjj\nkEVAhowyDlkEZMgo45BFQIaMMg5ZBGTIKOMolSJQvnx56tevH509e5YAUHJyMs2aNYsqV64sqd+q\nVauSi4uLFq2srET35ezsTOfOnSPGGDHGqGHDhuTk5CS6n+nTp5MgCASABEGgSZMmUUREBK1cuZIq\nVKggur+3BcOGDePPhjFGSUlJZG9vL5m/Hj160NmzZyk8PJxWrlypwU2bNhFjjM6cOaOf8ZJeMlyU\nZcOdO3fGjh078PLlS+Tk5OD777/nfPz4MX7++WfRl2hWqFABH330ET766CNcvHhR5wsjS5cuhY2N\njWg+V69ejYSEBA0fjDFER0cXOZ/C61i9enUe3/D+/fs4deoUXr58yV80evz4sV7prurWrYtp06bB\n2dm50N+xsrLSGfPQEFasWBHNmjVDWFgYACA1NdWgNxgrVaqEYcOGYc+ePVovZeWPA2hpaYlKlSqh\nffv2mDVrlkHRrogIH330EWrWrKlzX/fu3fkboW+wU3rfHXBwcMDWrVuRkpKCmJgYzJkzR+uGXL58\nWdTkmUR5r6POmTOnwNdF1VnEVzoLpJmZGRhjWvZV26Kjo0XxU6VKFcTHx+PMmTOoVasWiPJeyKlV\nqxYGDhzIX0H+9ddfi2x7yZIlYIzh/PnzhTq+Z8+eOH/+PGJiYgy6ppo1a6JSpUro3bs39u/fz18G\nU6fq1fOisnnz5pgxY4aGrczMTBw7dkwrYpGFhQW2bt2Ka9eu4ebNm2CMifojkd/X/fv3ERERgapV\nq77p+NIpAtbW1ggODkZmZia8vLzg4OCgdUyvXr2QkpKCL774QrSbO3r0aCxdurRQAiCWCFSqVAln\nz56VXAQqVqzIMw8VFEbsl19+4W9RFsV2rVq18Pz5cx4O+03Zeu3t7REUFMSvb+zYsXpd086dOxEX\nF4dnz55pVXz12In6iIC7u7tGiK99+/ahVatWBQZieffddzX8h4WFSRaxafny5UhNTS1sMtTSKQLd\nunXD9evX0bRpU537u3fvjrS0NFy/fh2VK1c2+KZ++umn2LlzJ2JjYzUq4ebNmzFo0CA8ffpUMhGo\nUqWKRoVXcfv27dixY4doIqAKc/66CmFiYoJhw4ahXr16MDc3L7Tta9eu8cL/9OnTN4b/+uuvvzQq\njL6tgTNnzmhV/ps3b2LUqFGYNWsW3+br61tk2ykpKRoC8LqmfadOnTSEaPfu3XrHvHwTly1bVugg\nOK9YOkWAiArsCzVr1gwpKSlIS0vTN1WzBvv27YvIyEidlVzVAmncuDHatGmDESNGiC4CAwcO5M3M\nESNGwNbWFrdu3UKtWrV4kpL4+HiDsui4uLggNTUVubm56Nq1a4HHmZiYoFOnTjh8+HChC3H79u2R\nnJzMK8B333332mbwrFmzkJSUxI/PyMgoMLDrmzhgwADcvHkTiYmJWLRoEX8eNjY2PGhrYGCgXj8U\n6sKSk5ODJUuWwNXVVes4Y2NjnhSEMYaffvpJsjTzU6ZMQXZ2dqG7XK9YekVAFxs1aoSsrCyEh4fj\nnXfeMeiGGhkZoVu3bkhKStKq/FeuXIG9vb3Wd0xNTfH111+LJgJOTk64c+cOlEolRowYobV/zZo1\nUCqViI+PN6jF07t3bzDGEBsbW+Ax1apVw8mTJ4vUryfKS22uCrgSFBT02riItra2OHjwoEYFu3fv\nnkFhwKysrFClShWNbarBT8YYpk6dqpfdmJgYrVZGQECAlq/Dhw+DMYbc3Fxs2LBB9JBmKg4dOhSZ\nmZkIDAwsavj20i8C5cuXx/LlyxEWFsYjyGZkZODvv/9GmzZt9L6pLi4uWpU/NDT0jf3hzz77jB//\n5ZdfGvRgzc3NeRck/77OnTvzeHaGdgdq1KiBlJQUJCYmav1KW1hY4KeffkJmZiZSUlIwYcKEIv2S\nqVeS8PBwNGrUCFZWVqhWrRrq168POzs7EOWNfagqjIrPnj0TPeKRra0tLl++DMbyMknpm2a9YsWK\n2LdvH1JTUzXOOT09HZ6enrCyssLWrVuRnp6OiIgIDBgwQNTryM/MzExcvHhRn8HG0i8CqmjCaWlp\n2Lx5M9q0acPz9GVkZODy5csYOHAgypUrV6Sbo/6LrmJhREVdBHRV3qJy27ZtGnbMzMywcOFCPm2n\nVCqxdetWg/3ExcWBMYZevXrxbQ0bNsSBAwegVCoREBBQ5PDmRJoi8Pz5cxw5cgRnzpzhyU+Dg4Px\n+eefw9/fX+uXVcwU6CoOGTKE2/fy8jLY3ueff44LFy7w1o6KKqFhLC+RrdjXoc7BgwcjMDBQ39mG\n0i8ChaGHhwdycnLg7e1d6O8IgqBRmY8ePVqoppyYIlCpUiXe7wegVUl27dolWkBTW1tbPHnyBIIg\nYNOmTTx0d7t27QyyO3HiRHh5eeHs2bPw8vLCr7/+im3btiE4OFjretT54sULUcKQ56fK/qxZs0BE\naN26tV6h23Vx1KhROq8lJiYGly5dwrBhw0QNQHv79m0+XmSAnbIhAkSE+Ph4zJkzp9DHAyhyS6Bu\n3boICgrix0+bNk3v83Vzc8OtW7e0pgNVzf+ZM2eKHtG4V69evOCePXvWYAEoLLOzs7UCgvbr109U\nH87OzhqVNCwsDJcvX0Zubi4++eQTUXx07NjxtcLGWF4SGzGyOltYWPABx4CAAENslQ0RaNGiBdLT\n0zFx4sRCf8fd3V1LBB48ePDaJle7du0Mmh0wMjJClSpVMG7cOK349ar+5qFDh0RfRaeiqtURHR1d\nrLkB1WcPGGNYtWqVwavpiPKSwRw9ehSJiYlafXcVHz16hHr16hnsq3z58nzgNDc3F71798aaNWuQ\nnJyMrKwsDZ/79u3TO9qxo6MjOnTooDHjIAgCVq5cWeQu7yu+/SJgbm6OoKAgREVFFWkEvUaNGoiP\nj9cSgrS0NKSmpmrx3Xff1aq4RRWBXr16aVT6q1evYvHixXyaUMp8fTNmzOBprFJTU9GoUSPJfKmz\nRYsW/HoZY/j555+LtAbhddS1ToCxvAVLFy5cwJgxY0QRGyLC06dPuQAsWLBAY9/777+vIXRpaWl6\nzRI4Ojpi+/btfKHT9evXMWvWLN7CSUpK0iefwtstAubm5jh27BgAaGXdLQxbtWqFBw8eFGp1oLOz\ns0GLhdzd3ZGdnc2/u2/fPo0VZVKJgIODA3bv3o3c3FxkZ2ejZ8+eYKzIWWz0pioDkYozZ84UzbYu\nEQgLC9M53WooVSIQFRWlc796+nB9Fic5OTnB19cXQF6Gq1OnTmnkxVy4cCGUSiUEQYC/v39RUqGX\nfhF4Xb940aJFAIBjx47p/evywQcfYPv27QgPD9dZ+c+ePYvt27ejcuXKOH78uN4i4OHhwQvJ3Llz\ntfYDEF0EjI2NceLECTDGkJKSgo8//hhEVKwisHDhQn7dGzduFHUp7WeffYapU6fy5c6Mab/UIxZf\nJwLdunXDkydPwBhDSEjIG5dN52eTJk0QERHBx4UOHDigMx197969cf/+fQiCgPT0dJw9exYrV658\nk/3SLQIeHh4IDQ3VSgOuyrybk5NjkACoc9CgQZg3b54W1fuThowJeHh48O9dvnwZnp6e8PDw4JSi\nJVCnTh1eOVS5B52cnIqtO+Ds7Izff/+dd32kWkjz8ccf8+uU6lp0iYCzszMGDhyInJwcpKWlwdfX\nV69UZFFRUfz8C7O4qX///rh16xYYY7hx48abUryVbhFo0qQJBEHAhQsXYGdnB2NjYwwfPhwvXrxA\nVlYWli1bJmla7fysUqUKNmzYoJcI9OnTBykpKTpnA5RKJbZt22bQ0mBdrFOnDnJzczF69GgoFAqY\nmpri3LlzRVl3rjdNTU0xdOhQXriPHTsmSf5GU1NTBAQESC4CixYtQnZ2NnJycvDgwQM8ePCAL03O\nzs7GokWL9LJbsWJFPqvw448/Fvp7JiYmcHd3L3B5vRpLtwgQEcaMGYOMjAxERkYiODgYgiDgxIkT\nRekTiUr1dQJFbVK3bNkSV69e1SkCo0ePFv1c7e3tcezYMdja2sLJyQlr164FY6wwBcdgWlpa4u+/\n/+ZdkYMHD0oyIzFlyhQuAElJSZJeU69evbB69Wru78aNG0hNTTVoSXf58uURGBgoZZZo/USAiLYQ\nUSwR3Vbb5k1Ez4no+iv2VNs3jYgeElEoEbmLKQIyZb6OXbt25Ytq/Pz8Svx8/oPUWwQ6E1EL0haB\nH3Uc24CIbhCRORHVJKJHRGQsi4BMmf8J6hSBN8YYBHCBiBLfdNwr9CWiXQCyATymvBZBm0J+V4YM\nGSUAQwKNjlUoFDcVCsUWhUJh82pbdSKKVDvm2attMmTI+I9CXxFYT0S1iKgZEUUT0YqiGlAoFF8r\nFIpghUIRrOc5yJAhQwToJQIAYgAwAAIR+dD/m/zPiUg9LvY7r7bpsrERQCvoSJUsQ4aM4oNeIqBQ\nKBzV/v2EiG6/+nyIiD5VKBTmCoWiJhHVJqIrhp2iDBkypITJmw5QKBR/EZErEdkrFIpnRDSHiFwV\nCkUzyhtxfEJE3xARAbijUCh8ieguESmJaAwAJs2py5AhQwwoXk3RlexJKBQlfxIyZLz9uKqr+10q\n05DpQpcuXWj//v0aqaH8/PyKxXd8fDw9fPiwWHyJjc2bN5NSqSRBEEgQBAoLC6Pq1d+OCR0fHx9i\njJFSqeRkjFGtWrVK+tT0Qo0aNejw4cMEgDZs2EDr1q3jXLlyJTVt2pSqVatWdMMlvWTY0MVClStX\nxoEDBzTSZ6k4efJk0RZaWFpa4osvvsDevXtx8+ZNDebm5kIQBFH8mJmZAcgLL3bo0CGMGzdOtPfg\n1eni4oILFy7w6D4JCQn8s1jRd0qSqte98y/J9vPzK9YgKmLRysoKsbGxb4xm9PDhw9fZKf3vDuji\n3r17tW7Ev//+i2fPnuH7778X5QG0atWKR3UpiPq8N56fbm5u8PPz0yq4y5cvF60wVahQAd7e3khJ\nSYEgCNi/fz8GDRqENWvWQBAE+Pj4wMjICB07dkS/fv10hlsvLD/44AP4+/tDEAQAwLVr17Bjxw58\n9dVXkr3mS0RQKBQIDQ3VKQKMMZ52TWyamZlh6NChOHjwIAIDAxEYGIg1a9aAKO9V7qLEvczPH3/8\nkZfvgIAAhIaGajAuLg5paWnYsWPH6+y8fSKwZcsWjfRQUVFR6N+/P0xMTNC4cWODH2r58uUxZ84c\nvHz5UqPCJyYm8vDgqm2GvJNfvnx5HlVYqVRi+PDhGD58OCZMmIDHjx9j3759ohXU7du383O+du0a\nf/X60KFDSE9Px4cffoh3330X+/btgyAIWLp0qd6+Ro4cWaBo5uTkoHPnzqJXxC5dumDPnj280oeE\nhMDe3h5Hjx6VVAS++OILPH78GCkpKVi2bBk8PDywbt067NmzB0SECRMm4I8//tDbvpeX12vfjqxZ\nsyaaN2/+JjtvlwiUK1dO49f/+fPnor8Xv3XrVl5ok5KSMHLkSPTr14/n73v69CkEQcC///5r0C/m\n8OHDeaHdtGmTxj5ra+vCJJosFKtXr47k5GQIgoDNmzfj1YAsiAgbNmzAyJEjUatWLezZsweCIODl\ny5fw8PDQ25+6COzduxejRo3SEAJDczXooiqoiCqJi+q5qIvA77//LqrPjz76CNnZ2bh8+TKqVavG\nt1euXBkffvghJk6ciJycHDg5Oento1atWjykmQHn+naJwLx58zREQI94a69l3bp1kZqayn+18r8i\n2rt3b94SeF06rzexUaNGiIuL44VWrJh7utiuXTskJCQgNzdXZ3N8zpw5ePbsGQRBQHx8vMFRgLdt\n28YrvCqIiLoI5M/gYwjLly+PDh064NGjR1wE1Pf36dOHi8Dhw4dF8WlkZIQpU6ZAqVRi7ty5GuHM\nzczMULt2bT7ucvfuXYN8qVoCWVlZhth5e0SgZ8+eGvnhV69erXdE14LYu3dvXljzP0AzMzMEBgZC\nEAQ8e/aswMy+haEqMMnp06dhYWGhsa9JkyYYPnw4zp8/L9pg3ZUrV3ieBVXcAgsLCxw+fJhfb3x8\nvN7ZetSp3hJQRW5W/R8WFvamKDhFYo8ePTT6/Tdv3tTYb21tzTM+iyUCdnZ2EAQBcXFxGttr1aoF\nX19fCIKAzMxMLFmyxOCchKroQZs3bzbEztshAjVq1NAIKe3n5yfJr6e6CHz33Xca+2bPng1BEJCc\nnGxwIlRVM7Vbt25a+yZPnswL9pEjR0S5rk6dOmmMcXh4eODcuXMQBAGMMSxevBjW1tai+Bo+fDj3\n07ZtW9SrV4/7yX9PDWVYWBgvE9evX9fZPVu1ahUAiHYv58+fD0EQ4OnpCSLCe++9h9GjR+PZs2dI\nSEjAL7/8gk6dOoni6/Hjx2DM4ExKb4cIqEdzef78Od5//31RC5OKzZo1Q3p6OgRBwBdffMG3z5w5\nk2/fsGGDwX6OHj2Ks2fPagmZpaUlvLy8RBcBIkKHDh1w+/Ztjaa5GM3//DQ2Nsbu3bv5r2VaWhoE\nQUBubq5B/WN1qmYCMjMzoVQqcfPmzQLHZ1TjBWK0BBwcHBAVFQVBEGBhYYFz584hOTkZ2dnZOH36\nNDp06CDafXR1deXRqevVq4cmTZpgzpw5uHPnDn788ceipD4v/SLg7u6uMQ6gHoZZCs6bNw+CIGDy\n5MnYvn070tPTuRrrmQtOi6qWQFBQEFxdXTFt2jSkpqYaFNL8dTQyMkK7du0QHx+vIQJPnjzRCuIq\nFt977z3MmTOHTxcyxtC3b19RbA8cOFAjW1NBobnUBwbF6g5cv36dT3/qm/H4TaxcubLGDJiKCQkJ\n8PHx4clOdu/eXRh7pV8Ezp07p3EjpLjp6lSJQFJSEi+848aNE9WHk5MT7t+/r/WAW7Vqha+++gqM\n5WXOEeOX08bGBsuXL9eo/P3798fZs2chCAKWLFki+T1V74YYasva2pqvq2CMITQ0tMBjjx07BsaY\nKBW2QoUKOHToEE+pJkay04Lo6empUTbi4+Ph5ubGF5AtWbIEjOXlQCyEvbdTBNzc3CS5+eXLl+dZ\nkAVBQEREhKhNPHXWqFEDY8eO5VRt37x5M5RKJc6dOyeKn3v37mkIwKFDh0BEmD59Om+mSxEFWJ1i\nikDt2rU1WgHu7u46j/vuu+94dOfw8HA4Ojrq7dPDwwN3796FUqlEVlYWBEEocm6BolB9Fszf319r\nVufu3btlRwRq1KiB0NBQjbDVycnJyMzMNCgZqC527dqVT5WpqE+qbkMppgh0796dr3pUKpXYvHkz\nH52vVauWqJXzdRTTz/Lly7kIFNTE79u3r0Z49/r16+vt77PPPkNcXByio6Mxfvx4tGzZEoIg4OLF\ni5Ldr+7duyM8PBz9+/eHmZkZ325mZqaRdWnGjBmFsVe6RaB3794Frpe+evWqKDfc1NQU69ev5+8C\npKSkIDExUbT3AopKb+XqtQAAGxxJREFUMUWgffv2vAL+8ssvGvuMjY2xfv16CIKAnTt3SnpNYoqA\nqp9fkAgMHDgQfn5+/BhDErp07twZ6enpCAgI4IvSBg8eDEEQDFoOrC9VKeQYY7h48WJhE5TqF2i0\nLKF27dr0zTffkLGxMcXExND7779P3377LRERGRmV7luVnZ1NRERbt26l6dOna+xjjFFkZF5oyHLl\nyhX7uekLhULB6ezsTDVq1OD7hg4dSrt37yZXV1dSKBS0ZcsW+vTTT/X2NWPGDLKwsKDjx49TdHQ0\n1a5dm1xcXIiIyN7e3uBryY8uXbqQmZmZzn02Nja0Y8cOIiKKiYmh0aNHU1ZWlv7OSroVYGhLICUl\nRbSkk+PHj+fN5d69e4Mob2BGEARRF7YUlmK2BKpXr474+HjExMSgZ8+eqF69Op9dqVq1Kh4/fgxB\nECTPSCRVS0CpVCIyMhKhoaEICwvjU4aMMZw5c8bgFO+bNm0q8D2I/Iu8DKWrqytycnIwbNgwrX1G\nRkY8ozRjDJ9//nlRbL+d3YETJ06IdvN79eoFQRAQGxsLorzBwRMnTkAQBINXfOnDjh07ijow+OGH\nHyI8PJx3d+7du4d169YhPDycL48We61Afqoqjj6Zo/Oze/fuOlPKq785uHHjRlGmkjt37ozTp0/j\n/Pnz2LlzJ18IZGFhofEOhhisWLEi0tPTcf36dXTu3Bn169fHpEmTsHz5cjx48ADZ2dk4fvw4OnXq\nVNRyWbpFoFatWli1apWGAMybN0/UgSwLCwu+mi49PR2ZmZkQBAEvXryQtGIURNXotxgVRp0dO3bE\njh07tF6P1pUhWWyqv1Akhr3OnTsXKALqMy2ljYcOHSpwDKx9+/b62i3dIlAWaWRkhEmTJvE04m8D\nQ0JCuBCI8X6CzCJRFgGZJc/GjRtzEfDx8Snx8ylj1CkCcqBRGTLKDt7uQKMyZMjQD7IIyJBRxiGL\ngAwZZRyyCMiQUcYhi4AMGWUcsgjIkFHGUWpFYPz48RopxxhjlJaWRocOHSIrKyvR/W3fvp0EQaCh\nQ4eKblsXTExMqF27dpSYmMhThAmCQJcvX6b58+cbZNvY2Jh8fX0pPDxcpLP976B9+/b022+/8fsF\ngARBoCNHjtCoUaPI2tpaUv/Lli0jAFSlShVJ/Vy9epUeP35MVatWNdxYSS8U0mex0Pjx4xEdHY2Y\nmBgcPHgQ33zzDQ4fPsyXVe7du1e08F8q5ubmgjGGrVu3Sr6oo1WrVhrRf1XLmFWfly1bZpD9qlWr\n8iW2Ur4Y1bx5c8yaNQtxcXEAgNDQUMyaNQsjR46UxJ8qCvTr0nTdu3cPH374oST+jY2N8fDhQ9HD\nqedngwYNkJWVpc9r32/PikFVtBX1l4eMjIzg7e2NnJwcxMXFoWHDhqLd9JYtWyIrKwuDBg3Cp59+\nKtnDJSKYm5vjxYsX/EWmEydO4LvvvoOrqysyMjJ4nANDfIwdO5aLQJs2bSS7loiICJ0VURXUZNSo\nUaL6s7a2fq0AqGioiKqYP5ORqakpBEFAaGiopPEvp02bxkPDFfG7pV8ETE1NUbNmTS4Cul6s+fXX\nXxEZGSlaJCALCwvcuHFDtBde3kRnZ2f+y1+nTh2+vW3btlwE/Pz8DPIRExPDI/O+ePFC5yurYnDP\nnj2vrYy5ubn4+eefRfNXWBFQ5VswhN27d0dKSgouXLjAE5yqROD06dOSlpG7d+8iLCxMn1D7pV8E\natWqhaysLAQHBxcYaPTbb78FY0y0PHfNmzeHIAhwcHCQ9MGqOHPmTPzzzz9wdXXl2ypVqoQzZ85A\nEAQcPnwYzZo109t+69atwRhDdHQ0GjZsCKVSieTkZCxatEj0aylXrhzs7Ozg6emJOXPmoH///oiM\njNSokI8ePRLNX6VKlXhEaMYY3N3dERQUpCUC7733nsG+Zs2axZPPqDIPqXJVSC0CBkSAejtE4E3R\nhlUiIEbcwWrVquHJkydgjGllOBoyZAiioqJEC2iiYpUqVWBnZ6exrUePHjwl2t69e/WObWBhYYGj\nR48CAM+lcPPmTQB5qdDDw8PRqFEjgzIqvYmqWIeMMSQmJhqUwk0XFy9ezO2rx+ATUwQaNGiAmJgY\nCIKAbdu28e1ffvklBEHA8ePHJbt/CoUCCQkJaNq0qT7fLxsi8Oeff4IxhokTJxp8w318fAr0FRIS\ngrS0NNGToOannZ0doqKiAAAvXrwwqK85ceJE/r69ekKVjRs3agTnuHXrlqhjKn369MEXX3yBDz/8\nEAEBAfyeGpqfTxddXFzw7NkznZU/LS0NY8aMMXgwVD292pgxY/j248ePQxCkSbSqopOTkyGBTUu/\nCGzZsuWNIpCcnAzGmMH9ZqK8is4Yw/r167X2LV26VHIRsLe35zkBBEEweBDv2rVrUCqViI+P1wq7\n/cEHH+DKlStcCBISEgwKzU2Uly770qVLPEFGSkqKxvPLzMxEYGAgatasKep9W7hwoZYAxMbGihI6\nzdzcHHv37oUgCIiJiUHjxo35vuIQgTp16uCHH37Q9/ulXwT+/vtv/lD//PNPrf1ff/01n8ozNEyW\nnZ0dEhISIAjaWYcbN26M9PR0REVFSfaw7e3t+TiAIAgIDAxExYoVDbKpEoGC+v9WVlbYu3evztaC\nPhw5cmShBurE6lKpckVkZmZqDUCKFTVp9OjR/Jn88ccfGvtUIqAaKJSC+/bt08qQXQTqJwJE5ERE\nfkR0l4juENH4V9ttieg0ET149dfm1XYFEa0moodEdJOIWoghAu3ateMPddeuXVp99CpVqmg0+wwZ\nPCMi/PDDD2CMaVX0hg0b8mxEYhUsKysr9OrVCx07dgRRXiJPVd6+s2fPonnz5qL4adiwIbp06aJz\nn6mpKdq2bYuYmBjRBuwcHBzQoUMH1K1bF3379uXjAP/++y9/VhcuXDB4Tr1z5846U3Wp+Pvvv4ty\n/1TlQn39xp07d3DhwgWerUrKloCNjY2h4e/1FgFHelWRiciKiMKIqAERLSUir1fbvYhoyavPPYno\nOOWJQTsi+ldsEbh48SJPw0REMDEx4ckmGWP466+/DL7hw4YNA2MMGRkZmDJlCt8+ZMgQLgIF5b0r\nLC0tLTFq1CjcvXsXgiAgLS2NjzUIgoDIyEiDm+SFpZubm0Y2nwYNGohq//vvvwdjDNu3b0fFihXx\nwQcf4IMPPjB41qVixYqIi4vTqvjp6emIjY3l/4u1vkNdBOLj4zlPnjzJtxvagiqIU6ZMwbNnzwyx\nIU53gIgOElFXIgolIkc1oQh99fk3IvpM7Xh+nFgiwBjj2ViqVq2KCxcu4OLFi2CMISgoSLTVgurx\n8PLzyZMnevtp0aIFtm3bhtDQUAiCgJcvXyIyMlLLh/qgk5Rs3rw5nj17xkVAjHl0dXbu3JmP1aim\n08Sil5eXlgDcuXMHzZs3x7hx4/i2oKAgUUKDOzs7w93dHe7u7hpRhm1tbSULQU70/zUIgYGBhtgx\nXASIyJmIIoioIhElqW1XqP4noiNE9L7avjNE1MpQETh16hR/oJcvX+YtAQcHB1y9epXvW7dunWg3\n3snJCZs2bUJSUpJGIRMEAZs2bdLL5tChQzFz5kxeYA4cOIAaNWpg8ODBCAsL0xCBlJQU/P3334ZE\nl30jGzZsyBcPqSIbq7eyxOD+/ftfO5irL2vWrInw8HBuW6lUwtvbm1dCY2NjjdkI9cVXYlNqEejS\npQsEQTC0RWOYCBBRBSK6SkT9X/2flG//y6KIABF9TUTBr/jGC1izZg1/mPHx8WjRogVWrlyJs2fP\nalTQGjVqiP4AXFxcMHz4cO4rPDxc7yzB6hW8a9euqFatGqZPn67xboAq14GKqamp+OGHH/RZIfZG\nPnz4kFegxYsXiy4AlSpVwqNHj/jsgJi2ly9frvHs9+3bp3WMehJbKUOqSy0CqiQ4Bqao1ykCJlQI\nKBQKUyLaS0Q7AOx7tTlGoVA4AohWKBSORBT7avtzyhtMVOGdV9s0AGAjEW18ZR9vOgcvLy+qX78+\nubq6ko2NDR05coSsrKzI0tKSiIgEQaB58+bRs2fPCnNJRUJ4eDiFh4eTvb09denShcLCwnjaLn2g\nUCjI39+fZsyYQQ0aNCB7e3tSKBR05MgRWrhwIYWEhNCUKVPo/fffp9jYWEpOTqbk5GTKzc0V8aqI\n6tSpQzVr1qT09HRau3YtTZs2TVT7RHnpwJydnYmI6IcffhDdvjrmzZvHP5uamtKiRYuobdu2kvp8\nK1CIFoCCiP4gol/ybV9GmgODS1997kWaA4NXxBgYJMrLEKRrAOjly5fIyMiQTOVV/PHHHyEIAubM\nmaO3DV3jC3FxcVi5cqXWYJy5ublkmY9atmyJU6dOITo6WrJ3Bzr9r72rj42qytvPCdINLGBfXFPt\nuiLVrQRq6IehtLtgjUkVaAA/+oJoRUO7/uHKO6U18u4anZgYlMSmSl5eP/JuA/gGlLyCG1MTsfRD\navhYtaWl1G21DGxTWiiFflvm3uf9Y+Zeb6cz7XTm3t4Zep7kyczce+ec3/3dc5577rnnnN+KFXpf\nwJkzZ0zv5PRtCQwPD/PgwYM8ePAgv/rqqzFlRXv7YgUXLlxoaUsgNjaWqqryvffeCyedkFsCfwCQ\nB6BBCFHn3fYXAG8C+EQIsQWAC8C/e/eVw/OGoBXAIIDngsgjKJw5cwbDw8OjgmY2Nzfj5MmTZmUx\nIUhi586dIf//5MmTWLZsGQCgt7cXq1atQn19PQYHB8ccqwURNRspKSn47LPPcNtttyEzM9My/yUn\nJ2POnDkAgISEBMTFxaGjo8O09N1u96jfM2fOxGOPPTbmuOvXr6O0tBQnTpwwLW9faMFJh4aGtBub\nqRgYGADg8anpmOzbASuISahZdnY2i4uLeejQITY3N/OZZ56xTN19WVxcHHbn1owZMxgTE8OYmBhb\ngpwCv8wivHDhgul9AEYah+8eOnTI9PTnzZvH1157LeCUZW2koFWv7IzU+gS6uros6bsRQvD1119n\nf39/OHM7on/EoGR0cc+ePVQUhfX19abGjJQMmTICkYTENIeMQCQhITEWUgQkJKY5pAhISExzSBGQ\nkJjmiGoRqKysBElkZWXZbYqERNQiqkVAq/xSBCQkQkfUiYDT6QRJOJ1Ou02RkLgxYPdAoXAGC5E0\nZS1BI2fPnj1qyml5ebm+3JY23Vajoijs6+tjdXU177jjDksGeGzatIkk+dBDD5mediCY7VPJiOGN\nNWJQg5lOysnJ4alTp1hRUUHAs4T01atX6Xa7eezYMWZnZzM9PZ1vvPEG09PTuXz5cj7//PP8+uuv\nw53i6Zfz58/nyMgIVVW1JPzZeDBDCNLS0pidnT1mGG1KSgpbW1sZHx9vd6UwhXFxcfrkoRMnTvDZ\nZ5+13aYAvHFEoLKy0pI7Vm1tLTs7O/XoRS6XiwMDA6yrq7PlohmXtjZrncFAPtR+myEEs2bNYm1t\nLYeGhnju3Dnef//9o/Z9+umnPHDggKmz7dLS0lhYWMjS0lJ2d3ezsLDQ73GJiYksLCxkYWEhU1NT\nw8ozKSmJn3zyCdvb29nX18fOzk729/fzypUrIS9goi1btn//fp0FBQVcv349t27dyvXr14/aV19f\nT1VVOTw8zNWrV0+U/o0hAk6nUy+kZle62tpavvPOO/rv/Px8btiwwfR8gmFsbCwbGhqoqiqbmppM\nnWzkz4dZWVl0Op3MysoaJQTGSEjB8qOPPmJXVxcffvjhMUuwLV26lIqi8OWXXw7rHHJyclhTU0OX\ny0WXy8UrV66MeVzz97/c3Fx9fxhLd9PhcLC9vV2fqKRNA3/zzTd59uzZkBe3CbScXTBsbGycKP0b\nQwSMMLPS7dq1i4qi8IknnjA13VC5aNEiqqpKt9ttehRd413f336jSDidzkmlHR8fz56eHr711lt+\n92siEG7kocHBwTH9M74isHv3bmZkZIzi8ePH6Xa7OTAwELLAl5aWcmRkhIqi8Msvv9QjRs2dO1df\nySjUFYd3797NqqoqndeuXQtY6fv6+vTIVFr8ygnSj34RCPcOFYgFBQX8+eef6Xa7dRFITExkbm4u\nc3NzWVBQwPPnzzM/P5/79u1jTk6OqZXSl/Pnz+epU6eoqiobGhpMTdv3Tj/RMZN9JNAiOPuLCwF4\nRGBoaCjsprixsl++fJkul4tbtmxhdXU129ra2NPT47cjV/sezlJjNTU1VBSFVVVVo1po77//vt4y\nMGvZ8VtvvZXx8fF+uWDBglGxKY4dOzZRetEtAsEU3lBZWFioF46zZ8+ytraWP/30U8ACNDg4yNra\nWt51112m2qFxzZo1VFWV165dMz0QqhHjVfBwfN3U1MTe3l6/z8XPPfecKUFburu72draSofD4VdQ\nHnjgATocDjocDjqdzlHXsLW1NeSo1StWrCBJqqrK4uJiffuDDz6oV8aOjg5LyoUvk5KS9DyvX78e\nTKj36BaBcJqoE9HYEvBX6Ts7O+lyudjW1jaqGdrc3MyEhATTL25ZWZklrQCjDyeq3MEIRSAuXryY\nFy9eZGtrKzMyMkYtkbZz505LIzf549y5c/VHIEVRwgpH9tRTT1FVVXZ3dzM+Pp433XQTMzMzefHi\nRb0V8OGHH07JeW3YsGFUDIQg/hO9IhCoeZqVlaV3aPlysg7Nz8/njz/+yMuXL+t3EI3Gu8bGjRtH\n3VmamppMX+FYa22kpaWZmq4R4/nI6O9QBfeFF17Ql2c/evQoS0pKmJeXx88//3yMCOTk5DAvL0+n\n2ZXlgw8+0K/X4cOH9ZgVoVILPtPe3s4jR47oK1BrIvDqq6+afg6+jImJ4bfffquLQJDXKXpFwF8r\nIFhYeSE0MXC73ayvrzclzVmzZlFV1YAda+HYGmzFNnYcmtH3MnPmTMbFxTEuLo6KorCiooJFRUUs\nKiriggULGBcXx5tvvtm0oDFG7t69W79Ge/fuNe21ZEpKCjMzM5mZmcl58+bpbwpeeuklS8ucxu3b\nt+sCUFNTE+z/bgwR0FoCRlRWVuotAN/33ZN17rJly4IO8GDsS2hrazPl4q5du5aXLl0yfSCNEeNV\n7Mk8MoRCRVGmbDBNcnIyL1y4MO4rQ7OoVUgrA8VoXLJkCQcHB/U8t27dGux/o1cEfAuxVvGDwWSc\nm5iYyHPnzgX9/jgxMZFdXV2misDevXt5+PBh0wtOsD4JVixCZUtLSzgLZU6K2mtfK6If+VJRFDY3\nN3P27NmWn5exFdDS0jKZ/0a3CPi+HQgGk+3U2rFjx6QHkbhcLrrdbvb09IR9cTMyMuh2u7lixQpT\nC00wd3df/1o1f+D777+3vJJo1Dp7e3t7+eSTT1qal6IoUxY70hitat26dZP5b3SLgL/CPB5CuYt9\n8803AcNZ+WNaWpoeEvvq1athX9wXX3yRqqqOCnRpBifqD/D1q5UTiI4ePWp5JdGoPQZs27bN8rwU\nReHatWun5Lw0AaisrJxscJroFwGNgVoF2rDXUJ1bW1urF5x9+/aN24mUnJzMsrIy0/oEYmJi2NLS\nEm78eb/016cSyI9WCsCSJUtYVFQ0JRUlNjZWHxkYxJj6sJiTkzNlImAcGxDCeYUeizDSUFVVBSGE\n6ek2NjbinnvuwS233IJNmzZh0aJFGBkZAeCJH+gVLADAnXfeifj4eH3brl27wsr70Ucfxd13343+\n/v6w0pkIWVlZo85DQ1VVFaqrqy1dpyE1NRUVFRWWpW/EK6+8AgD47rvvUF5ebmle9913H/r7+1FX\nVzfxwWFi8+bNAIDTp0/jyJEj5iRqdysglJaAlXzkkUfY1tbG7u7uCcemK4rC8+fPc9u2bWGtJzBj\nxgy+++67VFWV5eXllpzXeB2pZg++CsS8vDxL4vT58umnn9Y7A5cvX25pXgkJCezv72dJSYnl56VF\nJh4YGODjjz8eSho3zuPAVDA1NZUOh4MlJSUBRcDhcHDOnDlh52Wcj/72229bcj5ZWVljhGCqKr9G\nKwYC+ePHH3+sX6P09HRL88rOzqaiKLz33nstzScpKUmfVfrFF1+Emo4UAUl7WVZWNiX5aCKwf/9+\ny/NauXIlFUWxtIWzdOlSXrp0Sb9RhJGWFAHJ6cE1a9awoqLC8g7BqeKOHTt0ATh+/Hg4aclYhBIS\n0xwyFqGEhMRYSBGQkJjmiJRxApcBDHg/owm/gbTZakSbvUDk2rzA38aI6BMAACHEP/w9r0QypM3W\nI9rsBaLPZvk4ICExzSFFQEJimiOSROADuw0IAdJm6xFt9gJRZnPE9AlISEjYg0hqCUhISNgA20VA\nCPGIEOIHIUSrEGK73fYEghDinBCiQQhRJ4T4h3fbfCHEESFEi/fz32y28W9CiC4hRKNhm18bhQfv\nev1+WgiRGkE2O4UQ7V5f1wkhVhv2/afX5h+EEA/bZPPvhBCVQogmIcQZIcR/eLdHtK8DwuY5AzMA\n/AggAUAMgHoAi+2eyxDA1nMAfuOzbSeA7d7v2wG8ZbONKwGkAmicyEYAqwF8AUAAWA7gRATZ7ARQ\n7OfYxd4y8isAC71lZ4YNNt8OINX7fS6Af3pti2hfB6LdLYFlAFpJ/kRyBMABAOtstmkyWAdgj/f7\nHgDrbbQFJGsAXPHZHMjGdQD20oPjAGKFELdPjaW/IIDNgbAOwAGSP5NsA9AKTxmaUpDsIPmd93sf\ngLMAfosI93Ug2C0CvwVwwfD7X95tkQgC+FII8a0Q4k/ebXEkO7zfLwKIs8e0cRHIxkj3/Z+9Tee/\nGR6zIs5mIcRdAFIAnECU+tpuEYgm/JFkKoBVAF4QQqw07qSn3RfRr1qiwUYv/hvA3QCSAXQAeNte\nc/xDCDEHwP8BcJDsNe6LIl/bLgLtAH5n+H2Hd1vEgWS797MLwCF4mqGdWrPO+9lln4UBEcjGiPU9\nyU6SCkkVwIf4pckfMTYLIWbCIwD/S/JT7+ao8zVgvwicAvB7IcRCIUQMgI0A/m6zTWMghPi1EGKu\n9h1ANoBGeGzd7D1sM4DP7LFwXASy8e8AnvH2XC8HcM3QlLUVPs/Lj8Lja8Bj80YhxK+EEAsB/B7A\nSRvsEwD+B8BZkiWGXVHnawD2vh0w9Jz+E56e3r/abU8AGxPg6ZWuB3BGsxPALQAqALQA+ArAfJvt\n3A9P8/k6PM+dWwLZCE9P9X95/d4A4P4Isnmf16bT8FSg2w3H/9Vr8w8AVtlk8x/haeqfBlDn5epI\n93UgyhGDEhLTHHY/DkhISNgMKQISEtMcUgQkJKY5pAhISExzSBGQkJjmkCIgITHNIUVAQmKaQ4qA\nhMQ0x/8DXgW7Npq3afwAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y7NloDy08CQ9",
        "colab_type": "text"
      },
      "source": [
        "#### Model\n",
        "Let's construct the computation graph. Below are the parameters:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dqjWHTo08CQ-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# parameters \n",
        "N_STEPS = 28\n",
        "N_INPUTS = 28\n",
        "N_NEURONS = 150\n",
        "N_OUTPUTS = 10\n",
        "N_EPHOCS = 10"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gpFu75e88CRB",
        "colab_type": "text"
      },
      "source": [
        "And finally, here is a figure of the RNN-based classification model we are building:\n",
        "\n",
        "![alt txt](https://docs.google.com/drawings/d/e/2PACX-1vQWhELhewvq_bHgqwf4vwDb5B9DN9-jAxeTF9Y73zr-OsW6OXC-ngxAfojivXyZEhjzLXceTZU2Ncz3/pub?w=550&h=600)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HukX7ZjP8CRC",
        "colab_type": "text"
      },
      "source": [
        "And here is the code for the model:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "14jNZxeg8CRD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class ImageRNN(nn.Module):\n",
        "    def __init__(self, batch_size, n_steps, n_inputs, n_neurons, n_outputs):\n",
        "        super(ImageRNN, self).__init__()\n",
        "        \n",
        "        self.n_neurons = n_neurons\n",
        "        self.batch_size = batch_size\n",
        "        self.n_steps = n_steps\n",
        "        self.n_inputs = n_inputs\n",
        "        self.n_outputs = n_outputs\n",
        "        \n",
        "        self.basic_rnn = nn.RNN(self.n_inputs, self.n_neurons) \n",
        "        \n",
        "        self.FC = nn.Linear(self.n_neurons, self.n_outputs)\n",
        "        \n",
        "    def init_hidden(self,):\n",
        "        # (num_layers, batch_size, n_neurons)\n",
        "        return (torch.zeros(1, self.batch_size, self.n_neurons))\n",
        "        \n",
        "    def forward(self, X):\n",
        "        # transforms X to dimensions: n_steps X batch_size X n_inputs\n",
        "        X = X.permute(1, 0, 2) \n",
        "        \n",
        "        self.batch_size = X.size(1)\n",
        "        self.hidden = self.init_hidden()\n",
        "        \n",
        "        # lstm_out => n_steps, batch_size, n_neurons (hidden states for each time step)\n",
        "        # self.hidden => 1, batch_size, n_neurons (final state from each lstm_out)\n",
        "        lstm_out, self.hidden = self.basic_rnn(X, self.hidden)      \n",
        "        out = self.FC(self.hidden)\n",
        "        \n",
        "        return out.view(-1, self.n_outputs) # batch_size X n_output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8Yw8fFlp8CRJ",
        "colab_type": "text"
      },
      "source": [
        "The `ImageRNN` model is doing the following:\n",
        "- The initialization function `__init__(...)` declares a few variables, and then a basic RNN layer `basic_rnn` followed by a fully-connected layer `self.FC`.\n",
        "- The `init_hidden` function initializes hidden weights with zero values. \n",
        "The `forward` function accepts an input of size `n_steps X batch_size X n_neurons`. Then the data flows through the RNN layer and then through the fully-connected layer. \n",
        "- The output are the log probabilities of the model."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bJUCKm_U8CRK",
        "colab_type": "text"
      },
      "source": [
        "#### Testing the model with some samples\n",
        "A very good practice encouraged by PyTorch developers throughout their documentation, and which I really like and highly recommend, is to always test the model with a portion of the dataset before actual training. This is to ensure that you have the correct dimension specified and that the model is outputing the information you expect. Below I show an example of how to test your model:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P6W0jwK08CRM",
        "colab_type": "code",
        "outputId": "51d9f522-1558-4628-88f5-911c2ae505c7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "source": [
        "dataiter = iter(trainloader)\n",
        "images, labels = dataiter.next()\n",
        "model = ImageRNN(BATCH_SIZE, N_STEPS, N_INPUTS, N_NEURONS, N_OUTPUTS)\n",
        "logits = model(images.view(-1, 28,28))\n",
        "print(logits[0:10])"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[-0.0918,  0.0176,  0.0474, -0.0849, -0.0141,  0.1235,  0.0070,  0.0719,\n",
            "          0.0072, -0.0563],\n",
            "        [-0.1017,  0.0276,  0.0580, -0.0783, -0.0150,  0.1127,  0.0171,  0.0749,\n",
            "          0.0092, -0.0654],\n",
            "        [-0.0941,  0.0234,  0.0402, -0.0825, -0.0402,  0.1208,  0.0117,  0.0753,\n",
            "          0.0188, -0.0701],\n",
            "        [-0.1021,  0.0160,  0.0551, -0.0886, -0.0132,  0.1113,  0.0136,  0.0634,\n",
            "         -0.0021, -0.0533],\n",
            "        [-0.0874,  0.0227,  0.0468, -0.0806, -0.0118,  0.1191,  0.0105,  0.0696,\n",
            "          0.0077, -0.0581],\n",
            "        [-0.0929,  0.0256,  0.0636, -0.0729, -0.0147,  0.1028,  0.0197,  0.0766,\n",
            "          0.0140, -0.0700],\n",
            "        [-0.0960,  0.0257,  0.0497, -0.0822, -0.0177,  0.1183,  0.0120,  0.0728,\n",
            "          0.0047, -0.0644],\n",
            "        [-0.0962,  0.0185,  0.0586, -0.0732, -0.0197,  0.1047,  0.0119,  0.0760,\n",
            "          0.0104, -0.0641],\n",
            "        [-0.0880,  0.0151,  0.0478, -0.0839, -0.0210,  0.1098,  0.0152,  0.0732,\n",
            "          0.0105, -0.0578],\n",
            "        [-0.0929,  0.0200,  0.0529, -0.0741, -0.0281,  0.1076,  0.0143,  0.0781,\n",
            "          0.0135, -0.0696]], grad_fn=<SliceBackward>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BKDkckp98CRQ",
        "colab_type": "text"
      },
      "source": [
        "#### Training"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "46Qke-c08CRR",
        "colab_type": "text"
      },
      "source": [
        "Now let's look at the code for training the image classification model. But first, let's declare a few helper functions needed to train the model:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BoDWoBuy8CRS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch.optim as optim\n",
        "\n",
        "# Device\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "# Model instance\n",
        "model = ImageRNN(BATCH_SIZE, N_STEPS, N_INPUTS, N_NEURONS, N_OUTPUTS)\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
        "\n",
        "def get_accuracy(logit, target, batch_size):\n",
        "    ''' Obtain accuracy for training round '''\n",
        "    corrects = (torch.max(logit, 1)[1].view(target.size()).data == target.data).sum()\n",
        "    accuracy = 100.0 * corrects/batch_size\n",
        "    return accuracy.item()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uvCzj28o8CRX",
        "colab_type": "text"
      },
      "source": [
        "Before training a model in PyTorch, you can programatically specify what device you want to use during training; the `torch.device(...)` function tells the program that we want to use the GPU if one is available, otherwise the CPU will be the default device.\n",
        "\n",
        "Then we create an instance of the model, `ImageRNN(...)``, with the proper parameters. The criterion represents the function we will use to compute the loss of the model. The `nn.CrossEntropyLoss()` function basically applies a log softmax followed by a negative log likelihood loss operation over the output of the model. To compute the loss, the function needs both the log probabilities and targets. We will see later in our code how to provide this to the criterion.\n",
        "\n",
        "For training, we also need an optimization algorithm which helps to update weights based on the current loss. This is achieved with the `optim.Adam` optimization function, which requires the model parameters and a learning rate. Alternatively, you can also use `optim.SGD` or any other optimization algorithm that's available. \n",
        "\n",
        "The `get_accuracy(...)` function simply computes the accuracy of the model given the log probabilities and target values. As an exercise, you can write code to test this function as we did with the model before.\n",
        "\n",
        "Let's put everything together and train our image classification model:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LaAmGhZT8CRX",
        "colab_type": "code",
        "outputId": "a53631d3-ce78-4dca-d41a-85d1f2cf9e41",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "for epoch in range(N_EPHOCS):  # loop over the dataset multiple times\n",
        "    train_running_loss = 0.0\n",
        "    train_acc = 0.0\n",
        "    model.train()\n",
        "    \n",
        "    # TRAINING ROUND\n",
        "    for i, data in enumerate(trainloader):\n",
        "         # zero the parameter gradients\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        # reset hidden states\n",
        "        model.hidden = model.init_hidden() \n",
        "        \n",
        "        # get the inputs\n",
        "        inputs, labels = data\n",
        "        inputs = inputs.view(-1, 28,28) \n",
        "\n",
        "        # forward + backward + optimize\n",
        "        outputs = model(inputs)\n",
        "\n",
        "        loss = criterion(outputs, labels)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        train_running_loss += loss.detach().item()\n",
        "        train_acc += get_accuracy(outputs, labels, BATCH_SIZE)\n",
        "         \n",
        "    model.eval()\n",
        "    print('Epoch:  %d | Loss: %.4f | Train Accuracy: %.2f' \n",
        "          %(epoch, train_running_loss / i, train_acc/i))"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch:  0 | Loss: 0.8915 | Train Accuracy: 70.15\n",
            "Epoch:  1 | Loss: 0.4040 | Train Accuracy: 87.86\n",
            "Epoch:  2 | Loss: 0.2492 | Train Accuracy: 92.83\n",
            "Epoch:  3 | Loss: 0.1951 | Train Accuracy: 94.30\n",
            "Epoch:  4 | Loss: 0.1736 | Train Accuracy: 94.93\n",
            "Epoch:  5 | Loss: 0.1598 | Train Accuracy: 95.43\n",
            "Epoch:  6 | Loss: 0.1394 | Train Accuracy: 95.97\n",
            "Epoch:  7 | Loss: 0.1339 | Train Accuracy: 96.15\n",
            "Epoch:  8 | Loss: 0.1283 | Train Accuracy: 96.30\n",
            "Epoch:  9 | Loss: 0.1250 | Train Accuracy: 96.39\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "93-DMqi58CRf",
        "colab_type": "text"
      },
      "source": [
        "We can also compute accuracy on the testing dataset to test how well the model performs on the image classification task. As you can see below, our RNN model is performing very well on the MNIST classification task.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BzeqYT4N8CRi",
        "colab_type": "code",
        "outputId": "fbeff58d-756a-474c-975a-172b30ee36b3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "test_acc = 0.0\n",
        "for i, data in enumerate(testloader, 0):\n",
        "    inputs, labels = data\n",
        "    inputs = inputs.view(-1, 28, 28)\n",
        "\n",
        "    outputs = model(inputs)\n",
        "\n",
        "    test_acc += get_accuracy(outputs, labels, BATCH_SIZE)\n",
        "        \n",
        "print('Test Accuracy: %.2f'%( test_acc/i))"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test Accuracy: 96.50\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FD98PsQi8CRv",
        "colab_type": "text"
      },
      "source": [
        "### Final Words\n",
        "Please notice that we are not using GPU in this tutorial since the models we are building are relatively simple. As an exercise, you can take a look at the [PyTorch documentation](https://pytorch.org/docs/stable/notes/cuda.html) to learn how to program specific operations to execute on the GPU. You can then try to optimize the code to run on the GPU. If you need help with this, reach out to me on [Twitter](https://twitter.com/omarsar0).\n",
        "\n",
        "That's it for this tutorial. Congratulations! You are now able to implement a basic RNN in PyTorch. You also learned how to apply RNNs to solve a real-world, image classification problem.\n",
        "\n",
        "In the next tutorial, we will do more advanced things with RNNs and try to solve even more complex problems, such as sarcasm detection and sentiment classification. Until next time!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jKAm1xB08CRx",
        "colab_type": "text"
      },
      "source": [
        "### References\n",
        "- [A Simple Neural Network from Scratch with PyTorch and Google Colab](https://github.com/omarsar/pytorch_intro_neural_network/blob/master/nn.ipynb)\n",
        "- [Hands on Machine Learning with Scikit-learn and Tensorflow](http://shop.oreilly.com/product/0636920052289.do)"
      ]
    }
  ]
}