04: SimpleTransformers_2_CoLA_Sentence_Classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "SimpleTransformers 2_CoLA_Sentence_Classification.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "fae1398dc1624b43a046c48cbc201b7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_49bd36f22f554ea7ad0f4c7877800a8b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_738b97047bdb4bf5b0cad7b1605aa017",
              "IPY_MODEL_a22fcb9ebee042ddb423cc9e07bc93b7"
            ]
          }
        },
        "49bd36f22f554ea7ad0f4c7877800a8b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "738b97047bdb4bf5b0cad7b1605aa017": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ac96ebfc8f1748cda48ba5dc3ec28849",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_936c3c53c5524039ab5f453ba5e34dcc"
          }
        },
        "a22fcb9ebee042ddb423cc9e07bc93b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d08634ed4b0546a69d8a8e04a83cb13e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:00&lt;00:00, 4.56kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_386197c6e6ff4b08aeea0476442f7792"
          }
        },
        "ac96ebfc8f1748cda48ba5dc3ec28849": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "936c3c53c5524039ab5f453ba5e34dcc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d08634ed4b0546a69d8a8e04a83cb13e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "386197c6e6ff4b08aeea0476442f7792": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c0881014a0ef4e1b9776d22fe91f2d7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5ba1bb564b36459eacc7a11af1acddb3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f17d18cae0be489b90728da6a5dc8c49",
              "IPY_MODEL_ea65881a467641258e48448ca2977f7b"
            ]
          }
        },
        "5ba1bb564b36459eacc7a11af1acddb3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f17d18cae0be489b90728da6a5dc8c49": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1ca969d5b16d4193b303ec20f24b125f",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2fe9af569c9f4a6ca144b6d272ffe4cd"
          }
        },
        "ea65881a467641258e48448ca2977f7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9710bd1e3a4841228d3932b1b46dbb35",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:06&lt;00:00, 68.4MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_19eb1083090247149ed2a55233afee60"
          }
        },
        "1ca969d5b16d4193b303ec20f24b125f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2fe9af569c9f4a6ca144b6d272ffe4cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9710bd1e3a4841228d3932b1b46dbb35": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "19eb1083090247149ed2a55233afee60": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "75d6f6f2091c4b82896d48ccfd4a1ade": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0d8c1b27043049b89752a36b3687c08f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8d9e74444f3e414e9c9743ac5eca65f3",
              "IPY_MODEL_0e688dc24c2f4097a824f1b41ea06fe7"
            ]
          }
        },
        "0d8c1b27043049b89752a36b3687c08f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8d9e74444f3e414e9c9743ac5eca65f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2f9e10a84fe249e283ac5d8cb0042aac",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b6e31b52042a4a5891d0efd40b995c45"
          }
        },
        "0e688dc24c2f4097a824f1b41ea06fe7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ba5789db37e14a6aaed972d11db912b4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 912kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_210085d9df574a09b784143eebbe0bcd"
          }
        },
        "2f9e10a84fe249e283ac5d8cb0042aac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b6e31b52042a4a5891d0efd40b995c45": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ba5789db37e14a6aaed972d11db912b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "210085d9df574a09b784143eebbe0bcd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c9fa3271a01f4012851c2bba27ff9304": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_227d0a21b7d240f3aeb1611bf493c805",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b3a6782ec67d43dcaf24096628096ee2",
              "IPY_MODEL_fafa7a9e3e514cbea6663e8a7e11f786"
            ]
          }
        },
        "227d0a21b7d240f3aeb1611bf493c805": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b3a6782ec67d43dcaf24096628096ee2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_122e8f35668442bb8e6281fcb62908d2",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 7695,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 7695,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9e16d1245bb14ca8a22ae4a13aa73b58"
          }
        },
        "fafa7a9e3e514cbea6663e8a7e11f786": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_83fad9c9c6314b188efa2a822de1e9f4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 7695/7695 [00:02&lt;00:00, 3709.21it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5de9acfa1ed74ce0ab854e28c02c14b0"
          }
        },
        "122e8f35668442bb8e6281fcb62908d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9e16d1245bb14ca8a22ae4a13aa73b58": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "83fad9c9c6314b188efa2a822de1e9f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5de9acfa1ed74ce0ab854e28c02c14b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ca441852dc6349ada8af6763e0869864": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_197c1f91ca7d45a2bd4f4d72dcc8817d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_724f288ef67d4733a3a6831649bc18ae",
              "IPY_MODEL_240ad2eef9084cd48ac84f0fe5dbc21d"
            ]
          }
        },
        "197c1f91ca7d45a2bd4f4d72dcc8817d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "724f288ef67d4733a3a6831649bc18ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a035940cbd5343f09d03f34e94188c0a",
            "_dom_classes": [],
            "description": "Epoch 4 of 4:  75%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 4,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 3,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5b6ee9cba1da455881a4c35d87f65111"
          }
        },
        "240ad2eef9084cd48ac84f0fe5dbc21d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a0730b94983042a58b6256993e0e6171",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3/4 [07:49&lt;02:36, 156.66s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_aa6682140f0b449f9e87d8760f56a154"
          }
        },
        "a035940cbd5343f09d03f34e94188c0a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5b6ee9cba1da455881a4c35d87f65111": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a0730b94983042a58b6256993e0e6171": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "aa6682140f0b449f9e87d8760f56a154": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b810c464dff4466f92e8ab167b0eb074": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4368a25c19104aa2b84a07eadd56ece3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_75b76d7d818d40bdba4fc377d2aca365",
              "IPY_MODEL_8325f226e8ed4cfbb58d0551208468df"
            ]
          }
        },
        "4368a25c19104aa2b84a07eadd56ece3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "75b76d7d818d40bdba4fc377d2aca365": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_31adc08a085e40d4a5720349a8f960ff",
            "_dom_classes": [],
            "description": "Epochs 0/4. Running Loss:    0.4106: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 241,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 241,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d4755a3822134c35ab9356914da3b46d"
          }
        },
        "8325f226e8ed4cfbb58d0551208468df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_dbf7444bd4d94c35a0dadd9a72bdf158",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 241/241 [02:19&lt;00:00,  1.73it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_80e9d26cb5b249d6b198d42724ad79a1"
          }
        },
        "31adc08a085e40d4a5720349a8f960ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d4755a3822134c35ab9356914da3b46d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dbf7444bd4d94c35a0dadd9a72bdf158": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "80e9d26cb5b249d6b198d42724ad79a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "00f3e5b04fb845cd8dafe400cbbe0b83": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ec984dc0a69c4ca797a220ddea4cca3c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a6a9bffc2f414af1b54eb8902ee09cfa",
              "IPY_MODEL_a931ed48e9b440789ea15570610f57d3"
            ]
          }
        },
        "ec984dc0a69c4ca797a220ddea4cca3c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a6a9bffc2f414af1b54eb8902ee09cfa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_32a46ed547c349a6bb65229ec700f56b",
            "_dom_classes": [],
            "description": "Epochs 1/4. Running Loss:    0.4499: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 241,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 241,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_069d9bb3da894a85aab92b2203a1ddf6"
          }
        },
        "a931ed48e9b440789ea15570610f57d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7b73c420cf994dc8aec80317cff04d0e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 241/241 [01:52&lt;00:00,  2.14it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c7efbeecabc343f1b77b6f7c85808e5a"
          }
        },
        "32a46ed547c349a6bb65229ec700f56b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "069d9bb3da894a85aab92b2203a1ddf6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7b73c420cf994dc8aec80317cff04d0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c7efbeecabc343f1b77b6f7c85808e5a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4446d04e98b8420382ea319fdcd9935e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b2ecef6e671a48009d3464b23ddfb677",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_46dd63f373ed4f20a89faea6f7eb1f83",
              "IPY_MODEL_37284221d6714d96a1e33078782ff113"
            ]
          }
        },
        "b2ecef6e671a48009d3464b23ddfb677": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "46dd63f373ed4f20a89faea6f7eb1f83": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_14788a81a8f64265a253f644dd9a7994",
            "_dom_classes": [],
            "description": "Epochs 2/4. Running Loss:    0.0956: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 241,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 241,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_423e7e920b9c4a179c82c3e5572ebfc5"
          }
        },
        "37284221d6714d96a1e33078782ff113": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bd178473c5f04701899c6bbe51de583c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 241/241 [01:20&lt;00:00,  3.00it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a119aa261bb24a258ef8aa30207a9fd7"
          }
        },
        "14788a81a8f64265a253f644dd9a7994": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "423e7e920b9c4a179c82c3e5572ebfc5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bd178473c5f04701899c6bbe51de583c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a119aa261bb24a258ef8aa30207a9fd7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5f242991fd334fb9b24da8678652e648": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9c2f541ea5ed42b78d0889b0c7d9bdec",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_cbb83cacf76f45a397737c05196a55e5",
              "IPY_MODEL_ce63ab99065e49449d62792426f84136"
            ]
          }
        },
        "9c2f541ea5ed42b78d0889b0c7d9bdec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cbb83cacf76f45a397737c05196a55e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c38350a6efa747e793c05505dc0ea7f8",
            "_dom_classes": [],
            "description": "Epochs 3/4. Running Loss:    0.1054:  48%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 241,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 116,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_01bf8c075b6b4f9f92d7f1aee9e5fee1"
          }
        },
        "ce63ab99065e49449d62792426f84136": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_691cf6f2fc6548f081bee51cd425ebf0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 116/241 [00:40&lt;00:34,  3.65it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5b167cccf92a452294ceb1c5cadf29e7"
          }
        },
        "c38350a6efa747e793c05505dc0ea7f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "01bf8c075b6b4f9f92d7f1aee9e5fee1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "691cf6f2fc6548f081bee51cd425ebf0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5b167cccf92a452294ceb1c5cadf29e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "da07a5a701544818b3cdf3ca12573137": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "state": {
            "_view_name": "VBoxView",
            "_dom_classes": [],
            "_model_name": "VBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_21db04f54fca48f7850c4bedabc44eac",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d6d2c52652344d00934ad102bd51f097",
              "IPY_MODEL_5d95b0375d144c2ebfe489928a6a0253"
            ]
          }
        },
        "21db04f54fca48f7850c4bedabc44eac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d6d2c52652344d00934ad102bd51f097": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "state": {
            "_view_name": "LabelView",
            "style": "IPY_MODEL_319bbade2104490e949895ca6d5582c0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "LabelModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 0.04MB of 0.04MB uploaded (0.00MB deduped)\r",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7f4f89ed59d449de9e76f9409c1f4935"
          }
        },
        "5d95b0375d144c2ebfe489928a6a0253": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8eaebd20acd445c58f4fe72e84ca756d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5d89c19c0f0a4a38afc36fc6744cf396"
          }
        },
        "319bbade2104490e949895ca6d5582c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7f4f89ed59d449de9e76f9409c1f4935": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8eaebd20acd445c58f4fe72e84ca756d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5d89c19c0f0a4a38afc36fc6744cf396": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "93c03626c6b74b4f86ae354d20c22a73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d6ce537c0d7146c69886962883b34c7f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_face2bd343124892a18648b38b649e0e",
              "IPY_MODEL_208be4b240e9455997a9a6eb65ab86cc"
            ]
          }
        },
        "d6ce537c0d7146c69886962883b34c7f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "face2bd343124892a18648b38b649e0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8ed1ad5ca61448888aa19c1aae124fce",
            "_dom_classes": [],
            "description": "Running Evaluation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 107,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 107,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_42e4a09e13014328b524b0df7e3097b7"
          }
        },
        "208be4b240e9455997a9a6eb65ab86cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fcab53656afa43febe2ec92f9972523b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 107/107 [00:02&lt;00:00, 36.89it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_41f435ff4e2a4adb9c352f3c69fbf39b"
          }
        },
        "8ed1ad5ca61448888aa19c1aae124fce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "42e4a09e13014328b524b0df7e3097b7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fcab53656afa43febe2ec92f9972523b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "41f435ff4e2a4adb9c352f3c69fbf39b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "35d304cc89b44b84b5d83f6da3399610": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c30c404d452742488da61fd64e15d586",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_248777aa2c1249d5ad205045ad9cc55a",
              "IPY_MODEL_261d84a5fe7c4c1d9064e3215cbf07f2"
            ]
          }
        },
        "c30c404d452742488da61fd64e15d586": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "248777aa2c1249d5ad205045ad9cc55a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7837f02e612444eebd8ff5d63be4fa6b",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 516,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 516,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fd6aacba6eee44ca867dcea3d8b99bae"
          }
        },
        "261d84a5fe7c4c1d9064e3215cbf07f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ddc6a6747b494b8d931ba6df826df0e9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 516/516 [00:06&lt;00:00, 73.90it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c8ee119378ca4e41bdf437239d9ea21a"
          }
        },
        "7837f02e612444eebd8ff5d63be4fa6b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fd6aacba6eee44ca867dcea3d8b99bae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ddc6a6747b494b8d931ba6df826df0e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c8ee119378ca4e41bdf437239d9ea21a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "568cc76da4724e7a933273565c46dcf5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f50b3922cd224a959d91e6ad5aa4a4ec",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_775d36fc2089480ea6104e145afb7abb",
              "IPY_MODEL_aad895bafcbf4a9c98481a1621229d63"
            ]
          }
        },
        "f50b3922cd224a959d91e6ad5aa4a4ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "775d36fc2089480ea6104e145afb7abb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_509d43d800ac4bb2b03256ff9b5f0e33",
            "_dom_classes": [],
            "description": "Running Evaluation: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 65,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 65,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_177e4f2e8283459692fe85ef9398a24e"
          }
        },
        "aad895bafcbf4a9c98481a1621229d63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3659634e6f254834b3e5357dfa7b891e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 65/65 [00:01&lt;00:00, 35.26it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_424e49691ecc4e37aa693c1ed6e667ae"
          }
        },
        "509d43d800ac4bb2b03256ff9b5f0e33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "177e4f2e8283459692fe85ef9398a24e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3659634e6f254834b3e5357dfa7b891e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "424e49691ecc4e37aa693c1ed6e667ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0d5db9c2b5c243fd84b6f4f9372cc389": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "state": {
            "_view_name": "VBoxView",
            "_dom_classes": [],
            "_model_name": "VBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8d4980906853428d80e5c86ac571c7ae",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3a1502bec5ee4b58826cfac5e7f4bad5",
              "IPY_MODEL_6d2cd6b24f2d4bea844e41ba87c7c873"
            ]
          }
        },
        "8d4980906853428d80e5c86ac571c7ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3a1502bec5ee4b58826cfac5e7f4bad5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "state": {
            "_view_name": "LabelView",
            "style": "IPY_MODEL_6ab2a52998b14812a0c7db0f3fcb5f8b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "LabelModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 0.03MB of 0.03MB uploaded (0.00MB deduped)\r",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b52f2c27057b468582249e42f006a956"
          }
        },
        "6d2cd6b24f2d4bea844e41ba87c7c873": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e55ab07e49fd4c8ab7c5fff65c9d5a9f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3cac61ab338b406581d0b47ffe6d12e9"
          }
        },
        "6ab2a52998b14812a0c7db0f3fcb5f8b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b52f2c27057b468582249e42f006a956": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e55ab07e49fd4c8ab7c5fff65c9d5a9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3cac61ab338b406581d0b47ffe6d12e9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b7c9d1546ced479cba6f41ec52a62d65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_6e4abe737ca34ff79f93f045afad2017",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1c6c2fb5fd9d44b1ba39ea643675bbc5",
              "IPY_MODEL_976bd0734bbe4d9aafa4e83e3dae8eae"
            ]
          }
        },
        "6e4abe737ca34ff79f93f045afad2017": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1c6c2fb5fd9d44b1ba39ea643675bbc5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9dd62fc5f7454a3ca3c8cea3d89a6f84",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7d9827c112124686854006fedc163862"
          }
        },
        "976bd0734bbe4d9aafa4e83e3dae8eae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_00b0aeea5e25488e8801635c6e59ebf4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:00&lt;00:00,  2.60it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2909f5f010b94ff8b1dc2d44f5908c5a"
          }
        },
        "9dd62fc5f7454a3ca3c8cea3d89a6f84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7d9827c112124686854006fedc163862": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "00b0aeea5e25488e8801635c6e59ebf4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2909f5f010b94ff8b1dc2d44f5908c5a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c646e1a0b2e54ace90ca2a6773593266": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_909c5ec52d794aafa675466fe6ea4cc3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_debe1c2a53274984ac7c977807dd7fde",
              "IPY_MODEL_4b5ee7c0979b434b9f10386daafc9f28"
            ]
          }
        },
        "909c5ec52d794aafa675466fe6ea4cc3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "debe1c2a53274984ac7c977807dd7fde": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_118df561cedb4b44b85beb233dfddc92",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_24015b3b265e493786d359de33953214"
          }
        },
        "4b5ee7c0979b434b9f10386daafc9f28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7a1d19cd4e5e4f6b961e33da3a4ee1d5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:00&lt;00:00,  7.68it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c028a3c185174f409cf9be4712f71e49"
          }
        },
        "118df561cedb4b44b85beb233dfddc92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "24015b3b265e493786d359de33953214": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7a1d19cd4e5e4f6b961e33da3a4ee1d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c028a3c185174f409cf9be4712f71e49": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gulabpatel/simpletransformers/blob/main/SimpleTransformers_2_CoLA_Sentence_Classification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EKOTlwcmxmej"
      },
      "source": [
        "# Simple Transformers - CoLA Sentence Classification\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BJR6t_gCQe_x"
      },
      "source": [
        "This Notebook is a port of the original \"Fine-Tuning BERT for Sentence Classification\" tutorial that Nick and I published in 2019 ([blog post](http://mccormickml.com/2019/07/22/BERT-fine-tuning/), [Notebook](https://colab.research.google.com/drive/1pTuQhug6Dhl9XalKB0zUGf4FIdYFlpcX)) to use the `Simple Transformers` library.\n",
        "\n",
        "This version has less comments and explanation than my typical Notebooks--the goal was just to get a quick sense of the `Simple Transformers` library.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jrC9__lXxTJz"
      },
      "source": [
        "# Contents"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p9MCBOq4xUpr"
      },
      "source": [
        "See \"Table of contents\" in the sidebar to the left."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RX_ZDhicpHkV"
      },
      "source": [
        "# S1. Setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nSU7yERLP_66"
      },
      "source": [
        "## 1.1. Using Colab GPU for Training\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GI0iOY8zvZzL"
      },
      "source": [
        "\n",
        "Google Colab offers free GPUs and TPUs! Since we'll be training a large neural network it's best to take advantage of this (in this case we'll attach a GPU), otherwise training will take a very long time.\n",
        "\n",
        "A GPU can be added by going to the menu and selecting:\n",
        "\n",
        "`Edit 🡒 Notebook Settings 🡒 Hardware accelerator 🡒 (GPU)`\n",
        "\n",
        "Then run the following cell to confirm that the GPU is detected."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DEfSbAA4QHas",
        "outputId": "d143e6a8-8eb3-4b93-f492-537b6de2003b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "# Get the GPU device name.\n",
        "device_name = tf.test.gpu_device_name()\n",
        "\n",
        "# The device name should look like the following:\n",
        "if device_name == '/device:GPU:0':\n",
        "    print('Found GPU at: {}'.format(device_name))\n",
        "else:\n",
        "    raise SystemError('GPU device not found')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cqG7FzRVFEIv"
      },
      "source": [
        "In order for torch to use the GPU, we need to identify and specify the GPU as the device. Later, in our training loop, we will load data onto the device. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oYsV4H8fCpZ-",
        "outputId": "3049d47d-7e05-4333-bdb3-6f6184ce8395",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "import torch\n",
        "\n",
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():    \n",
        "\n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla T4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2ElsnSNUridI"
      },
      "source": [
        "## 1.2. Installing Simple Transformers\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G_N2UDLevYWn"
      },
      "source": [
        "* Install Simple Transformers\n",
        "* Seeing some disconcerting errors..."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b50CCOvhAXfk",
        "outputId": "72792a78-c98f-4527-8d4c-d595d1fa680f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "!pip install simpletransformers"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting simpletransformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/56/35/31022262786f4aa070fe472677cea66fade8d221181a86825096af021e2c/simpletransformers-0.48.14-py3-none-any.whl (214kB)\n",
            "\r\u001b[K     |█▌                              | 10kB 24.1MB/s eta 0:00:01\r\u001b[K     |███                             | 20kB 2.8MB/s eta 0:00:01\r\u001b[K     |████▋                           | 30kB 3.5MB/s eta 0:00:01\r\u001b[K     |██████                          | 40kB 3.9MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 51kB 3.4MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 61kB 3.7MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 71kB 4.0MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 81kB 4.3MB/s eta 0:00:01\r\u001b[K     |█████████████▊                  | 92kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████████▎                | 102kB 4.5MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 112kB 4.5MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 122kB 4.5MB/s eta 0:00:01\r\u001b[K     |███████████████████▉            | 133kB 4.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████▍          | 143kB 4.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 153kB 4.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████▍       | 163kB 4.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 174kB 4.5MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 184kB 4.5MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 194kB 4.5MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▌ | 204kB 4.5MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 215kB 4.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (1.18.5)\n",
            "Collecting streamlit\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c7/d2/3228e647441606a38ce4f73a30ba8a5d36a7be421123d129f7e40348dc02/streamlit-0.69.2-py2.py3-none-any.whl (7.4MB)\n",
            "\u001b[K     |████████████████████████████████| 7.4MB 8.1MB/s \n",
            "\u001b[?25hCollecting tqdm>=4.47.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bd/cf/f91813073e4135c1183cadf968256764a6fe4e35c351d596d527c0540461/tqdm-4.50.2-py2.py3-none-any.whl (70kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 8.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (1.4.1)\n",
            "Collecting wandb\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/af/5a/59c6f39d8aceb2c38d6090f947fd77c1cedddf44b1be728d1ae682d09d15/wandb-0.10.7-py2.py3-none-any.whl (1.7MB)\n",
            "\u001b[K     |████████████████████████████████| 1.7MB 52.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (2.23.0)\n",
            "Collecting tokenizers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7c/a5/78be1a55b2ac8d6a956f0a211d372726e2b1dd2666bb537fea9b03abd62c/tokenizers-0.9.2-cp36-cp36m-manylinux1_x86_64.whl (2.9MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9MB 51.7MB/s \n",
            "\u001b[?25hCollecting transformers>=3.0.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/2c/4e/4f1ede0fd7a36278844a277f8d53c21f88f37f3754abf76a5d6224f76d4a/transformers-3.4.0-py3-none-any.whl (1.3MB)\n",
            "\u001b[K     |████████████████████████████████| 1.3MB 51.4MB/s \n",
            "\u001b[?25hCollecting seqeval\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c4/47/f85f522f1f2532ca650474088e4024a9f52d524379bff058eaadf53cb663/seqeval-1.2.1.tar.gz (43kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 8.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (0.22.2.post1)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (2019.12.20)\n",
            "Collecting tensorboardx\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/af/0c/4f41bcd45db376e6fe5c619c01100e9b7531c55791b7244815bac6eac32c/tensorboardX-2.1-py2.py3-none-any.whl (308kB)\n",
            "\u001b[K     |████████████████████████████████| 317kB 48.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from simpletransformers) (1.1.2)\n",
            "Collecting gitpython\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c0/d7/b2b0672e0331567157adf9281f41ee731c412ee518ca5e6552c27fa73c91/GitPython-3.1.9-py3-none-any.whl (159kB)\n",
            "\u001b[K     |████████████████████████████████| 163kB 56.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (3.12.4)\n",
            "Requirement already satisfied: pyarrow in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (0.14.1)\n",
            "Collecting boto3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/30/3c/c9b65c398de6589c93b3c65aa27a695a6cdf7588c50a7934ad3ac6599a8a/boto3-1.16.2-py2.py3-none-any.whl (129kB)\n",
            "\u001b[K     |████████████████████████████████| 133kB 57.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: astor in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (0.8.1)\n",
            "Requirement already satisfied: tzlocal in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (1.5.1)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (7.1.2)\n",
            "Collecting enum-compat\n",
            "  Downloading https://files.pythonhosted.org/packages/55/ae/467bc4509246283bb59746e21a1a2f5a8aecbef56b1fa6eaca78cd438c8b/enum_compat-0.0.3-py3-none-any.whl\n",
            "Collecting validators\n",
            "  Downloading https://files.pythonhosted.org/packages/41/4a/3360ff3cf2b4a1b9721ac1fbff5f84663f41047d9874b3aa1ac82e862c44/validators-0.18.1-py3-none-any.whl\n",
            "Requirement already satisfied: altair>=3.2.0 in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (4.1.0)\n",
            "Requirement already satisfied: toml in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (0.10.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (20.4)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (2.8.1)\n",
            "Collecting blinker\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1b/51/e2a9f3b757eb802f61dc1f2b09c8c99f6eb01cf06416c0671253536517b6/blinker-1.4.tar.gz (111kB)\n",
            "\u001b[K     |████████████████████████████████| 112kB 56.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: cachetools>=4.0 in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (4.1.1)\n",
            "Requirement already satisfied: tornado>=5.0 in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (5.1.1)\n",
            "Collecting base58\n",
            "  Downloading https://files.pythonhosted.org/packages/3c/03/58572025c77b9e6027155b272a1b96298e711cd4f95c24967f7137ab0c4b/base58-2.0.1-py3-none-any.whl\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.6/dist-packages (from streamlit->simpletransformers) (7.0.0)\n",
            "Collecting botocore>=1.13.44\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1f/96/35fa364675cf17e3a190ae08716ac4b078ca86a62ef071d32cd8b86d52bc/botocore-1.19.2-py2.py3-none-any.whl (6.7MB)\n",
            "\u001b[K     |████████████████████████████████| 6.7MB 49.6MB/s \n",
            "\u001b[?25hCollecting pydeck>=0.1.dev5\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/51/1e/296f4108bf357e684617a776ecaf06ee93b43e30c35996dfac1aa985aa6c/pydeck-0.5.0b1-py2.py3-none-any.whl (4.4MB)\n",
            "\u001b[K     |████████████████████████████████| 4.4MB 49.2MB/s \n",
            "\u001b[?25hCollecting watchdog\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0e/06/121302598a4fc01aca942d937f4a2c33430b7181137b35758913a8db10ad/watchdog-0.10.3.tar.gz (94kB)\n",
            "\u001b[K     |████████████████████████████████| 102kB 14.7MB/s \n",
            "\u001b[?25hCollecting shortuuid>=0.5.0\n",
            "  Downloading https://files.pythonhosted.org/packages/25/a6/2ecc1daa6a304e7f1b216f0896b26156b78e7c38e1211e9b798b4716c53d/shortuuid-1.0.1-py3-none-any.whl\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.6/dist-packages (from wandb->simpletransformers) (3.13)\n",
            "Requirement already satisfied: six>=1.13.0 in /usr/local/lib/python3.6/dist-packages (from wandb->simpletransformers) (1.15.0)\n",
            "Collecting docker-pycreds>=0.4.0\n",
            "  Downloading https://files.pythonhosted.org/packages/f5/e8/f6bd1eee09314e7e6dee49cbe2c5e22314ccdb38db16c9fc72d2fa80d054/docker_pycreds-0.4.0-py2.py3-none-any.whl\n",
            "Collecting subprocess32>=3.5.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/32/c8/564be4d12629b912ea431f1a50eb8b3b9d00f1a0b1ceff17f266be190007/subprocess32-3.5.4.tar.gz (97kB)\n",
            "\u001b[K     |████████████████████████████████| 102kB 14.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.6/dist-packages (from wandb->simpletransformers) (5.4.8)\n",
            "Requirement already satisfied: promise<3,>=2.0 in /usr/local/lib/python3.6/dist-packages (from wandb->simpletransformers) (2.3)\n",
            "Collecting sentry-sdk>=0.4.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fc/f0/188b8321988c739ab79577269c169d0d5f36aeb7172c17cd45eb0917823d/sentry_sdk-0.19.1-py2.py3-none-any.whl (120kB)\n",
            "\u001b[K     |████████████████████████████████| 122kB 53.0MB/s \n",
            "\u001b[?25hCollecting configparser>=3.8.1\n",
            "  Downloading https://files.pythonhosted.org/packages/08/b2/ef713e0e67f6e7ec7d59aea3ee78d05b39c15930057e724cc6d362a8c3bb/configparser-5.0.1-py3-none-any.whl\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->simpletransformers) (2020.6.20)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->simpletransformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->simpletransformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->simpletransformers) (2.10)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers>=3.0.2->simpletransformers) (0.7)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 47.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers>=3.0.2->simpletransformers) (3.0.12)\n",
            "Collecting sentencepiece!=0.1.92\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 46.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->simpletransformers) (0.16.0)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->simpletransformers) (2018.9)\n",
            "Collecting gitdb<5,>=4.0.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/48/11/d1800bca0a3bae820b84b7d813ad1eff15a48a64caea9c823fc8c1b119e8/gitdb-4.0.5-py3-none-any.whl (63kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 11.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.0->streamlit->simpletransformers) (50.3.0)\n",
            "Collecting s3transfer<0.4.0,>=0.3.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/69/79/e6afb3d8b0b4e96cefbdc690f741d7dd24547ff1f94240c997a26fa908d3/s3transfer-0.3.3-py2.py3-none-any.whl (69kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 11.0MB/s \n",
            "\u001b[?25hCollecting jmespath<1.0.0,>=0.7.1\n",
            "  Downloading https://files.pythonhosted.org/packages/07/cb/5f001272b6faeb23c1c9e0acc04d48eaaf5c862c17709d20e3469c6e0139/jmespath-0.10.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: decorator>=3.4.0 in /usr/local/lib/python3.6/dist-packages (from validators->streamlit->simpletransformers) (4.4.2)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.6/dist-packages (from altair>=3.2.0->streamlit->simpletransformers) (0.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.6/dist-packages (from altair>=3.2.0->streamlit->simpletransformers) (2.11.2)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.6/dist-packages (from altair>=3.2.0->streamlit->simpletransformers) (0.11.1)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.6/dist-packages (from altair>=3.2.0->streamlit->simpletransformers) (2.6.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->streamlit->simpletransformers) (2.4.7)\n",
            "Requirement already satisfied: ipywidgets>=7.0.0 in /usr/local/lib/python3.6/dist-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (7.5.1)\n",
            "Collecting ipykernel>=5.1.2; python_version >= \"3.4\"\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/52/19/c2812690d8b340987eecd2cbc18549b1d130b94c5d97fcbe49f5f8710edf/ipykernel-5.3.4-py3-none-any.whl (120kB)\n",
            "\u001b[K     |████████████████████████████████| 122kB 56.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: traitlets>=4.3.2 in /usr/local/lib/python3.6/dist-packages (from pydeck>=0.1.dev5->streamlit->simpletransformers) (4.3.3)\n",
            "Collecting pathtools>=0.1.1\n",
            "  Downloading https://files.pythonhosted.org/packages/e7/7f/470d6fcdf23f9f3518f6b0b76be9df16dcc8630ad409947f8be2eb0ed13a/pathtools-0.1.2.tar.gz\n",
            "Collecting smmap<4,>=3.0.1\n",
            "  Downloading https://files.pythonhosted.org/packages/b0/9a/4d409a6234eb940e6a78dfdfc66156e7522262f5f2fecca07dc55915952d/smmap-3.0.4-py2.py3-none-any.whl\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from jinja2->altair>=3.2.0->streamlit->simpletransformers) (1.1.1)\n",
            "Requirement already satisfied: ipython>=4.0.0; python_version >= \"3.3\" in /usr/local/lib/python3.6/dist-packages (from ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (5.5.0)\n",
            "Requirement already satisfied: nbformat>=4.2.0 in /usr/local/lib/python3.6/dist-packages (from ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (5.0.7)\n",
            "Requirement already satisfied: widgetsnbextension~=3.5.0 in /usr/local/lib/python3.6/dist-packages (from ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (3.5.1)\n",
            "Requirement already satisfied: jupyter-client in /usr/local/lib/python3.6/dist-packages (from ipykernel>=5.1.2; python_version >= \"3.4\"->pydeck>=0.1.dev5->streamlit->simpletransformers) (5.3.5)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.6/dist-packages (from traitlets>=4.3.2->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.2.0)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (2.6.1)\n",
            "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (4.8.0)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (1.0.18)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.7.5)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.8.1)\n",
            "Requirement already satisfied: jupyter-core in /usr/local/lib/python3.6/dist-packages (from nbformat>=4.2.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (4.6.3)\n",
            "Requirement already satisfied: notebook>=4.4.1 in /usr/local/lib/python3.6/dist-packages (from widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (5.3.1)\n",
            "Requirement already satisfied: pyzmq>=13 in /usr/local/lib/python3.6/dist-packages (from jupyter-client->ipykernel>=5.1.2; python_version >= \"3.4\"->pydeck>=0.1.dev5->streamlit->simpletransformers) (19.0.2)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.6/dist-packages (from pexpect; sys_platform != \"win32\"->ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.6.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.2.5)\n",
            "Requirement already satisfied: nbconvert in /usr/local/lib/python3.6/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (5.6.1)\n",
            "Requirement already satisfied: Send2Trash in /usr/local/lib/python3.6/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (1.5.0)\n",
            "Requirement already satisfied: terminado>=0.8.1 in /usr/local/lib/python3.6/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.9.1)\n",
            "Requirement already satisfied: testpath in /usr/local/lib/python3.6/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.4.4)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.6/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.6.0)\n",
            "Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.6/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.8.4)\n",
            "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.6/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (1.4.2)\n",
            "Requirement already satisfied: bleach in /usr/local/lib/python3.6/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (3.2.1)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.6/dist-packages (from bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit->simpletransformers) (0.5.1)\n",
            "Building wheels for collected packages: seqeval, blinker, watchdog, subprocess32, sacremoses, pathtools\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for seqeval: filename=seqeval-1.2.1-cp36-none-any.whl size=16168 sha256=ee1f78916a40d3cc42429a9125777a1adfbad3ba0cea7b3d276853a34ec1b35f\n",
            "  Stored in directory: /root/.cache/pip/wheels/b4/42/c0/e2d946497f714404c0b78a0ab4c7f45222a293f49320932463\n",
            "  Building wheel for blinker (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for blinker: filename=blinker-1.4-cp36-none-any.whl size=13450 sha256=8b13d95463005b056bcfb8a3ca6a619c8432996d4fce34b0eaf1038b385e1c89\n",
            "  Stored in directory: /root/.cache/pip/wheels/92/a0/00/8690a57883956a301d91cf4ec999cc0b258b01e3f548f86e89\n",
            "  Building wheel for watchdog (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for watchdog: filename=watchdog-0.10.3-cp36-none-any.whl size=73873 sha256=4a208d96795a66f20ca1c7b966b22fc64647afc1619bba500d6516f56cf00a06\n",
            "  Stored in directory: /root/.cache/pip/wheels/a8/1d/38/2c19bb311f67cc7b4d07a2ec5ea36ab1a0a0ea50db994a5bc7\n",
            "  Building wheel for subprocess32 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for subprocess32: filename=subprocess32-3.5.4-cp36-none-any.whl size=6489 sha256=e87c0c0612107cabeb82fa5c7dd77948a31329075f592dca738ad8b40bf3e6e3\n",
            "  Stored in directory: /root/.cache/pip/wheels/68/39/1a/5e402bdfdf004af1786c8b853fd92f8c4a04f22aad179654d1\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893257 sha256=ff07d221ac8614241d9e9e7110469ce89d9477548143be64985aecae61136a05\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pathtools: filename=pathtools-0.1.2-cp36-none-any.whl size=8785 sha256=8d3a41f43736897ae066dec04efb265bd45ef4b3ddcbad62ed51e8002e77f933\n",
            "  Stored in directory: /root/.cache/pip/wheels/0b/04/79/c3b0c3a0266a3cb4376da31e5bfe8bba0c489246968a68e843\n",
            "Successfully built seqeval blinker watchdog subprocess32 sacremoses pathtools\n",
            "\u001b[31mERROR: google-colab 1.0.0 has requirement ipykernel~=4.10, but you'll have ipykernel 5.3.4 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: botocore 1.19.2 has requirement urllib3<1.26,>=1.25.4; python_version != \"3.4\", but you'll have urllib3 1.24.3 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: seqeval 1.2.1 has requirement numpy==1.19.2, but you'll have numpy 1.18.5 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: seqeval 1.2.1 has requirement scikit-learn==0.23.2, but you'll have scikit-learn 0.22.2.post1 which is incompatible.\u001b[0m\n",
            "Installing collected packages: smmap, gitdb, gitpython, jmespath, botocore, s3transfer, boto3, enum-compat, validators, blinker, base58, ipykernel, pydeck, pathtools, watchdog, streamlit, tqdm, shortuuid, docker-pycreds, subprocess32, sentry-sdk, configparser, wandb, tokenizers, sacremoses, sentencepiece, transformers, seqeval, tensorboardx, simpletransformers\n",
            "  Found existing installation: ipykernel 4.10.1\n",
            "    Uninstalling ipykernel-4.10.1:\n",
            "      Successfully uninstalled ipykernel-4.10.1\n",
            "  Found existing installation: tqdm 4.41.1\n",
            "    Uninstalling tqdm-4.41.1:\n",
            "      Successfully uninstalled tqdm-4.41.1\n",
            "Successfully installed base58-2.0.1 blinker-1.4 boto3-1.16.2 botocore-1.19.2 configparser-5.0.1 docker-pycreds-0.4.0 enum-compat-0.0.3 gitdb-4.0.5 gitpython-3.1.9 ipykernel-5.3.4 jmespath-0.10.0 pathtools-0.1.2 pydeck-0.5.0b1 s3transfer-0.3.3 sacremoses-0.0.43 sentencepiece-0.1.91 sentry-sdk-0.19.1 seqeval-1.2.1 shortuuid-1.0.1 simpletransformers-0.48.14 smmap-3.0.4 streamlit-0.69.2 subprocess32-3.5.4 tensorboardx-2.1 tokenizers-0.9.2 tqdm-4.50.2 transformers-3.4.0 validators-0.18.1 wandb-0.10.7 watchdog-0.10.3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "ipykernel",
                  "tqdm"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "guw6ZNtaswKc"
      },
      "source": [
        "# S2. Loading CoLA Dataset\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_9ZKxKc04Btk"
      },
      "source": [
        "We'll use [The Corpus of Linguistic Acceptability (CoLA)](https://nyu-mll.github.io/CoLA/) dataset for single sentence classification. It's a set of sentences labeled as grammatically correct or incorrect. It was first published in May of 2018, and is one of the tests included in the \"GLUE Benchmark\" on which models like BERT are competing.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4JrUHXms16cn"
      },
      "source": [
        "## 2.1. Download & Extract"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3ZNVW6xd0T0X"
      },
      "source": [
        "We'll use the `wget` package to download the dataset to the Colab instance's file system. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5m6AnuFv0QXQ",
        "outputId": "a9f47946-78e2-41b3-91f4-1799b4ef6695",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 190
        }
      },
      "source": [
        "!pip install wget"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting wget\n",
            "  Downloading https://files.pythonhosted.org/packages/47/6a/62e288da7bcda82b935ff0c6cfe542970f04e29c756b0e147251b2fb251f/wget-3.2.zip\n",
            "Building wheels for collected packages: wget\n",
            "  Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for wget: filename=wget-3.2-cp36-none-any.whl size=9682 sha256=83f5f7ab1ae6203ad49cee9409334414319afafaf624d90d34add7fee4038489\n",
            "  Stored in directory: /root/.cache/pip/wheels/40/15/30/7d8f7cea2902b4db79e3fea550d7d7b85ecb27ef992b618f3f\n",
            "Successfully built wget\n",
            "Installing collected packages: wget\n",
            "Successfully installed wget-3.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "08pO03Ff1BjI"
      },
      "source": [
        "The dataset is hosted on GitHub in this repo: https://nyu-mll.github.io/CoLA/"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pMtmPMkBzrvs",
        "outputId": "81bc25b2-36a1-48a3-92c9-74231c9c8518",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import wget\n",
        "import os\n",
        "\n",
        "print('Downloading dataset...')\n",
        "\n",
        "# The URL for the dataset zip file.\n",
        "url = 'https://nyu-mll.github.io/CoLA/cola_public_1.1.zip'\n",
        "\n",
        "# Download the file (if we haven't already)\n",
        "if not os.path.exists('./cola_public_1.1.zip'):\n",
        "    wget.download(url, './cola_public_1.1.zip')"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading dataset...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_mKctx-ll2FB"
      },
      "source": [
        "Unzip the dataset to the file system. You can browse the file system of the Colab instance in the sidebar on the left."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Yv-tNv20dnH",
        "outputId": "3de46ac2-c721-47bd-e9e0-5a90e8ef52f1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "# Unzip the dataset (if we haven't already)\n",
        "if not os.path.exists('./cola_public/'):\n",
        "    !unzip cola_public_1.1.zip"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  cola_public_1.1.zip\n",
            "   creating: cola_public/\n",
            "  inflating: cola_public/README      \n",
            "   creating: cola_public/tokenized/\n",
            "  inflating: cola_public/tokenized/in_domain_dev.tsv  \n",
            "  inflating: cola_public/tokenized/in_domain_train.tsv  \n",
            "  inflating: cola_public/tokenized/out_of_domain_dev.tsv  \n",
            "   creating: cola_public/raw/\n",
            "  inflating: cola_public/raw/in_domain_dev.tsv  \n",
            "  inflating: cola_public/raw/in_domain_train.tsv  \n",
            "  inflating: cola_public/raw/out_of_domain_dev.tsv  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oQUy9Tat2EF_"
      },
      "source": [
        "## 2.2. Parse"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xeyVCXT31EZQ"
      },
      "source": [
        "We can see from the file names that both `tokenized` and `raw` versions of the data are available. \n",
        "\n",
        "We can't use the pre-tokenized version because, in order to apply the pre-trained BERT, we *must* use the tokenizer provided by the model. This is because (1) the model has a specific, fixed vocabulary and (2) the BERT tokenizer has a particular way of handling out-of-vocabulary words."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MYWzeGSY2xh3"
      },
      "source": [
        "We'll use pandas to parse the \"in-domain\" training set and look at a few of its properties and data points."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_UkeC7SG2krJ",
        "outputId": "72e825ad-75fc-49fe-93bc-0e53a4af72db",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427
        }
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Load the dataset into a pandas dataframe.\n",
        "df = pd.read_csv(\"./cola_public/raw/in_domain_train.tsv\", delimiter='\\t', header=None, names=['sentence_source', 'label', 'label_notes', 'sentence'])\n",
        "\n",
        "# Report the number of sentences.\n",
        "print('Number of training sentences: {:,}\\n'.format(df.shape[0]))\n",
        "\n",
        "# Display 10 random rows from the data.\n",
        "df.sample(10)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of training sentences: 8,551\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence_source</th>\n",
              "      <th>label</th>\n",
              "      <th>label_notes</th>\n",
              "      <th>sentence</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>8216</th>\n",
              "      <td>ad03</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Evan's every desire</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5980</th>\n",
              "      <td>c_13</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Frank has drunk too much.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5107</th>\n",
              "      <td>ks08</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>It was not until I was perhaps twenty-five or ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8484</th>\n",
              "      <td>ad03</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Romeo sent letters to Juliet.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2014</th>\n",
              "      <td>rhl07</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Felicia kicked the ball out the window.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>512</th>\n",
              "      <td>bc01</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>The medium contains ions.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>607</th>\n",
              "      <td>bc01</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>The window broke from the force of the wind.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5785</th>\n",
              "      <td>c_13</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>the book of poems from Blackwell by Robert Bur...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3670</th>\n",
              "      <td>ks08</td>\n",
              "      <td>0</td>\n",
              "      <td>*</td>\n",
              "      <td>The squirrel ran right quickly.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4095</th>\n",
              "      <td>ks08</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Free enterprise is compatible with American va...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     sentence_source  ...                                           sentence\n",
              "8216            ad03  ...                                Evan's every desire\n",
              "5980            c_13  ...                          Frank has drunk too much.\n",
              "5107            ks08  ...  It was not until I was perhaps twenty-five or ...\n",
              "8484            ad03  ...                      Romeo sent letters to Juliet.\n",
              "2014           rhl07  ...            Felicia kicked the ball out the window.\n",
              "512             bc01  ...                          The medium contains ions.\n",
              "607             bc01  ...       The window broke from the force of the wind.\n",
              "5785            c_13  ...  the book of poems from Blackwell by Robert Bur...\n",
              "3670            ks08  ...                    The squirrel ran right quickly.\n",
              "4095            ks08  ...  Free enterprise is compatible with American va...\n",
              "\n",
              "[10 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kfWzpPi92UAH"
      },
      "source": [
        "The two properties we actually care about are the the `sentence` and its `label`, which is referred to as the \"acceptibility judgment\" (0=unacceptable, 1=acceptable)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IC1tBYxRA6iU"
      },
      "source": [
        "## 2.3. Prep Data for SimpleTransformers"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cKi5LJzTBo4s"
      },
      "source": [
        "Logging setup - Makes the `transformers` logger less verbose."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z_nhVZkOBnzk",
        "outputId": "51904913-41ff-40c9-fb62-f2b8ebcafb1a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import logging\n",
        "\n",
        "logging.basicConfig(level=logging.INFO)\n",
        "\n",
        "# Get root logger (all other loggers will be derived from this logger's\n",
        "# properties)\n",
        "logger = logging.getLogger()\n",
        "logger.warning(\"Is this working?\") \n",
        "\n",
        "# Get the logger for the huggingface/transformers library.\n",
        "transformers_logger = logging.getLogger(\"transformers\")\n",
        "\n",
        "# Set the logging level to warning, meaning display warnings and worse, but \n",
        "# don't display any `INFO` logs.\n",
        "transformers_logger.setLevel(logging.WARNING)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:root:Is this working?\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "82WJCtDRCYWV"
      },
      "source": [
        "Convert the dataset into a simple list of [text, label] pairs.\n",
        "\n",
        "I'm following this code snippet from the documentation as my reference:\n",
        "```python\n",
        "# Train and Evaluation data needs to be in a Pandas Dataframe containing at least two columns. If the Dataframe has a header, it should contain a 'text' and a 'labels' column. If no header is present, the Dataframe should contain at least two columns, with the first column is the text with type str, and the second column in the label with type int.\n",
        "train_data = [['Example sentence belonging to class 1', 1], ['Example sentence belonging to class 0', 0], ['Example eval senntence belonging to class 2', 2]]\n",
        "train_df = pd.DataFrame(train_data)\n",
        "\n",
        "eval_data = [['Example eval sentence belonging to class 1', 1], ['Example eval sentence belonging to class 0', 0], ['Example eval senntence belonging to class 2', 2]]\n",
        "eval_df = pd.DataFrame(eval_data)\n",
        "\n",
        "```"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oeA6OK4H2eVL"
      },
      "source": [
        "# Select just the text and its label.\n",
        "df = df[['sentence', 'label']]\n",
        "\n",
        "# Rename 'sentence' column to 'text' (this is the name SimpleTransformers \n",
        "# expects).\n",
        "df.columns = ['text', 'label']"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wv7xPZlNCvr1"
      },
      "source": [
        "Split off 10% for validation."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "loJK0iLaCXDZ"
      },
      "source": [
        "# Use train_test_split to split our data into train and validation sets for\n",
        "# training\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Use 90% for training and 10% for validation.\n",
        "train_df, validation_df = train_test_split(df, random_state=2018, test_size=0.1)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lIwnAga6hgs4",
        "outputId": "d6fc0fa3-e84f-45ea-d738-d0b76aab2ff6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(train_df)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7695"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "37FcR_NOhiGN",
        "outputId": "9c504c19-c2e3-441b-a1ae-8510eae51c1b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(validation_df)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "856"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "osB0HTxX-_Yp"
      },
      "source": [
        "**Test Set**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JsF7SfJ1_ckU"
      },
      "source": [
        "**TODO** - The .tsv file doesn't include headers, so I've specified them here. This means that I can directly name the 'sentence' column 'text', but I think it's more illustrative this way?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zXpnGmUa_EIj",
        "outputId": "03c42da7-8e2d-43b9-b886-ab61c6ab6780",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 393
        }
      },
      "source": [
        "# Load the dataset into a pandas dataframe.\n",
        "df_test = pd.read_csv(\"./cola_public/raw/out_of_domain_dev.tsv\", delimiter='\\t', header=None, names=['sentence_source', 'label', 'label_notes', 'sentence'])\n",
        "\n",
        "# Report the number of sentences.\n",
        "print('Number of test sentences: {:,}\\n'.format(df.shape[0]))\n",
        "\n",
        "# Select just the text and its label.\n",
        "df_test = df_test[['sentence', 'label']]\n",
        "\n",
        "# Rename 'sentence' column to 'text' (this is the name SimpleTransformers \n",
        "# expects).\n",
        "df_test.columns = ['text', 'label']\n",
        "\n",
        "df_test.sample(10)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of test sentences: 8,551\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>159</th>\n",
              "      <td>I know the person whose mother died.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>290</th>\n",
              "      <td>Terry delighted in my pain.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>202</th>\n",
              "      <td>Which boy's mother did you meet who you liked?</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>333</th>\n",
              "      <td>Few dog barked.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>122</th>\n",
              "      <td>Tom will smoke the grass, and Reuben has the h...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>90</th>\n",
              "      <td>John was spoken to by Mary.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>335</th>\n",
              "      <td>I walk and Dana runs.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>270</th>\n",
              "      <td>Some people consider the dogs in my neighborho...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>87</th>\n",
              "      <td>The argument was summed by the coach up.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>97</th>\n",
              "      <td>The table was wiped by John clean.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                  text  label\n",
              "159               I know the person whose mother died.      1\n",
              "290                        Terry delighted in my pain.      1\n",
              "202     Which boy's mother did you meet who you liked?      0\n",
              "333                                    Few dog barked.      0\n",
              "122  Tom will smoke the grass, and Reuben has the h...      0\n",
              "90                         John was spoken to by Mary.      1\n",
              "335                              I walk and Dana runs.      1\n",
              "270  Some people consider the dogs in my neighborho...      1\n",
              "87            The argument was summed by the coach up.      0\n",
              "97                  The table was wiped by John clean.      0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kFiArS_C4DlQ"
      },
      "source": [
        "# S3. Fine-Tuning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QjKKHxGnaZsB"
      },
      "source": [
        "##3.1. wandb\n",
        "\n",
        "according to wandb documentation, when you are in Colab, wandb will authomatically authenticate your runtime if you are currentlylogged in to W&B in your browser [here](https://wandb.ai/login?signup=true)!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MXLwZ5ZLb3Eo",
        "outputId": "7f4b367b-89fb-480a-9f35-86c4363da06e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        }
      },
      "source": [
        "import wandb\n",
        "\n",
        "wandb.login()"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:wandb:setting login settings: {}\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "wandb: Paste an API key from your profile and hit enter: ··········\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n",
            "INFO:wandb:setting login settings: {'api_key': 'c849e83dfd93de2a3d09d6a8b1567489db733957'}\n",
            "INFO:wandb:setting user settings: {'email': 'gulabpatel923@gmail.com'}\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bizaNCg74HcL"
      },
      "source": [
        "## 3.2. Load Pre-Trained Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aH5bIzq_DXct"
      },
      "source": [
        "What's this reprocess_input_data argument?\n",
        "\n",
        "Documentation for arguments is [here](https://github.com/ThilinaRajapakse/simpletransformers#default-settings).\n",
        "\n",
        "> reprocess_input_data: bool\n",
        "If True, the input data will be reprocessed even if a cached file of the input data exists in the cache_dir."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ITLUMV1CBmQS",
        "outputId": "a3f86bd0-b243-48b7-e7e5-2a58f2761261",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371,
          "referenced_widgets": [
            "fae1398dc1624b43a046c48cbc201b7b",
            "49bd36f22f554ea7ad0f4c7877800a8b",
            "738b97047bdb4bf5b0cad7b1605aa017",
            "a22fcb9ebee042ddb423cc9e07bc93b7",
            "ac96ebfc8f1748cda48ba5dc3ec28849",
            "936c3c53c5524039ab5f453ba5e34dcc",
            "d08634ed4b0546a69d8a8e04a83cb13e",
            "386197c6e6ff4b08aeea0476442f7792",
            "c0881014a0ef4e1b9776d22fe91f2d7c",
            "5ba1bb564b36459eacc7a11af1acddb3",
            "f17d18cae0be489b90728da6a5dc8c49",
            "ea65881a467641258e48448ca2977f7b",
            "1ca969d5b16d4193b303ec20f24b125f",
            "2fe9af569c9f4a6ca144b6d272ffe4cd",
            "9710bd1e3a4841228d3932b1b46dbb35",
            "19eb1083090247149ed2a55233afee60",
            "75d6f6f2091c4b82896d48ccfd4a1ade",
            "0d8c1b27043049b89752a36b3687c08f",
            "8d9e74444f3e414e9c9743ac5eca65f3",
            "0e688dc24c2f4097a824f1b41ea06fe7",
            "2f9e10a84fe249e283ac5d8cb0042aac",
            "b6e31b52042a4a5891d0efd40b995c45",
            "ba5789db37e14a6aaed972d11db912b4",
            "210085d9df574a09b784143eebbe0bcd"
          ]
        }
      },
      "source": [
        "from simpletransformers.classification import ClassificationModel\n",
        "\n",
        "args = {\n",
        "    'reprocess_input_data': True, \n",
        "    'overwrite_output_dir': True\n",
        "}\n",
        "\n",
        "# Create a ClassificationModel\n",
        "model = ClassificationModel('bert', 'bert-base-uncased', num_labels=2, args=args)\n",
        "\n",
        "# You can set class weights by using the optional weight argument\n"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:filelock:Lock 140638701139560 acquired on /root/.cache/torch/transformers/4dad0251492946e18ac39290fcfe91b89d370fee250efe9521476438fe8ca185.7156163d5fdc189c3016baca0775ffce230789d7fa2a42ef516483e4ca884517.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fae1398dc1624b43a046c48cbc201b7b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:filelock:Lock 140638701139560 released on /root/.cache/torch/transformers/4dad0251492946e18ac39290fcfe91b89d370fee250efe9521476438fe8ca185.7156163d5fdc189c3016baca0775ffce230789d7fa2a42ef516483e4ca884517.lock\n",
            "INFO:filelock:Lock 140638701139560 acquired on /root/.cache/torch/transformers/f2ee78bdd635b758cc0a12352586868bef80e47401abe4c4fcc3832421e7338b.36ca03ab34a1a5d5fa7bc3d03d55c4fa650fed07220e2eeebc06ce58d0e9a157.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c0881014a0ef4e1b9776d22fe91f2d7c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:filelock:Lock 140638701139560 released on /root/.cache/torch/transformers/f2ee78bdd635b758cc0a12352586868bef80e47401abe4c4fcc3832421e7338b.36ca03ab34a1a5d5fa7bc3d03d55c4fa650fed07220e2eeebc06ce58d0e9a157.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "INFO:filelock:Lock 140638658618200 acquired on /root/.cache/torch/transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "75d6f6f2091c4b82896d48ccfd4a1ade",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:filelock:Lock 140638658618200 released on /root/.cache/torch/transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084.lock\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wwHNCEdHEaSz"
      },
      "source": [
        "## 3.3. Inspect Training Arguments"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4YSBeEWdEcIL"
      },
      "source": [
        "All of the cool features we're interested in are managed through this `model.args` object. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K4rNROrD4qAD",
        "outputId": "8bac98ba-429c-4ac5-c07e-1c54bf988768",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "type(model.args)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "simpletransformers.config.model_args.ClassificationArgs"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HeBRWPzmqUze"
      },
      "source": [
        "The following cell retrieves all of the arguments and their values, and prints them in a table.\n",
        "\n",
        "There is also some brief documentation on them [here](https://simpletransformers.ai/docs/usage/#configuring-a-simple-transformers-model)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ADMTSlPv4tsh",
        "outputId": "61bb110c-d3e2-4eec-a45c-1c433e244514",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "arg_values = []\n",
        "\n",
        "pd.set_option('display.max_rows', 80)\n",
        "\n",
        "# For all of the arguments...\n",
        "for arg in dir(model.args):\n",
        "    \n",
        "    # Skip over the special attributes and any functions.\n",
        "    if (not arg[0:2] == '__') and (not callable(getattr(model.args, arg))):\n",
        "    \n",
        "        # Store the argument and its value as a tuple.\n",
        "        arg_values.append((arg, str(getattr(model.args, arg))))\n",
        "\n",
        "# Store as a dataframe just to get the pretty printout.\n",
        "df_args = pd.DataFrame(arg_values)        \n",
        "\n",
        "df_args"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>adam_epsilon</td>\n",
              "      <td>1e-08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>best_model_dir</td>\n",
              "      <td>outputs/best_model</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>cache_dir</td>\n",
              "      <td>cache_dir/</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>config</td>\n",
              "      <td>{}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>custom_layer_parameters</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>custom_parameter_groups</td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>dataloader_num_workers</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>do_lower_case</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>dynamic_quantize</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>early_stopping_consider_epochs</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>early_stopping_delta</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>early_stopping_metric</td>\n",
              "      <td>eval_loss</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>early_stopping_metric_minimize</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>early_stopping_patience</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>encoding</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>eval_batch_size</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>evaluate_during_training</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>evaluate_during_training_silent</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>evaluate_during_training_steps</td>\n",
              "      <td>2000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>evaluate_during_training_verbose</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>evaluate_each_epoch</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>fp16</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>gradient_accumulation_steps</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>labels_list</td>\n",
              "      <td>[0, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>labels_map</td>\n",
              "      <td>{}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>lazy_delimiter</td>\n",
              "      <td>\\t</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>lazy_labels_column</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>lazy_loading</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>lazy_loading_start_line</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>lazy_text_a_column</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>lazy_text_b_column</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>lazy_text_column</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>learning_rate</td>\n",
              "      <td>4e-05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>local_rank</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>logging_steps</td>\n",
              "      <td>50</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>manual_seed</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>max_grad_norm</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>max_seq_length</td>\n",
              "      <td>128</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>model_class</td>\n",
              "      <td>ClassificationModel</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>model_name</td>\n",
              "      <td>bert-base-uncased</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>model_type</td>\n",
              "      <td>bert</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>multiprocessing_chunksize</td>\n",
              "      <td>500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>n_gpu</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>no_cache</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>no_save</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>num_train_epochs</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>onnx</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>output_dir</td>\n",
              "      <td>outputs/</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>overwrite_output_dir</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>process_count</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50</th>\n",
              "      <td>quantized_model</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>51</th>\n",
              "      <td>regression</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>52</th>\n",
              "      <td>reprocess_input_data</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53</th>\n",
              "      <td>save_best_model</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>54</th>\n",
              "      <td>save_eval_checkpoints</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>55</th>\n",
              "      <td>save_model_every_epoch</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56</th>\n",
              "      <td>save_optimizer_and_scheduler</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>57</th>\n",
              "      <td>save_steps</td>\n",
              "      <td>2000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>58</th>\n",
              "      <td>silent</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>59</th>\n",
              "      <td>skip_special_tokens</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>60</th>\n",
              "      <td>sliding_window</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>61</th>\n",
              "      <td>stride</td>\n",
              "      <td>0.8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>62</th>\n",
              "      <td>tensorboard_dir</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>63</th>\n",
              "      <td>thread_count</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>64</th>\n",
              "      <td>tie_value</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65</th>\n",
              "      <td>train_batch_size</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>66</th>\n",
              "      <td>train_custom_parameters_only</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>67</th>\n",
              "      <td>use_cached_eval_features</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>68</th>\n",
              "      <td>use_early_stopping</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69</th>\n",
              "      <td>use_multiprocessing</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>70</th>\n",
              "      <td>wandb_kwargs</td>\n",
              "      <td>{}</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>71</th>\n",
              "      <td>wandb_project</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>72</th>\n",
              "      <td>warmup_ratio</td>\n",
              "      <td>0.06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>73</th>\n",
              "      <td>warmup_steps</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>74</th>\n",
              "      <td>weight_decay</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                   0                    1\n",
              "0                       adam_epsilon                1e-08\n",
              "1                     best_model_dir   outputs/best_model\n",
              "2                          cache_dir           cache_dir/\n",
              "3                             config                   {}\n",
              "4            custom_layer_parameters                   []\n",
              "5            custom_parameter_groups                   []\n",
              "6             dataloader_num_workers                    1\n",
              "7                      do_lower_case                False\n",
              "8                   dynamic_quantize                False\n",
              "9     early_stopping_consider_epochs                False\n",
              "10              early_stopping_delta                    0\n",
              "11             early_stopping_metric            eval_loss\n",
              "12    early_stopping_metric_minimize                 True\n",
              "13           early_stopping_patience                    3\n",
              "14                          encoding                 None\n",
              "15                   eval_batch_size                    8\n",
              "16          evaluate_during_training                False\n",
              "17   evaluate_during_training_silent                 True\n",
              "18    evaluate_during_training_steps                 2000\n",
              "19  evaluate_during_training_verbose                False\n",
              "20               evaluate_each_epoch                 True\n",
              "21                              fp16                 True\n",
              "22       gradient_accumulation_steps                    1\n",
              "23                       labels_list               [0, 1]\n",
              "24                        labels_map                   {}\n",
              "25                    lazy_delimiter                   \\t\n",
              "26                lazy_labels_column                    1\n",
              "27                      lazy_loading                False\n",
              "28           lazy_loading_start_line                    1\n",
              "29                lazy_text_a_column                 None\n",
              "30                lazy_text_b_column                 None\n",
              "31                  lazy_text_column                    0\n",
              "32                     learning_rate                4e-05\n",
              "33                        local_rank                   -1\n",
              "34                     logging_steps                   50\n",
              "35                       manual_seed                 None\n",
              "36                     max_grad_norm                  1.0\n",
              "37                    max_seq_length                  128\n",
              "38                       model_class  ClassificationModel\n",
              "39                        model_name    bert-base-uncased\n",
              "40                        model_type                 bert\n",
              "41         multiprocessing_chunksize                  500\n",
              "42                             n_gpu                    1\n",
              "43                          no_cache                False\n",
              "44                           no_save                False\n",
              "45                  num_train_epochs                    1\n",
              "46                              onnx                False\n",
              "47                        output_dir             outputs/\n",
              "48              overwrite_output_dir                 True\n",
              "49                     process_count                    1\n",
              "50                   quantized_model                False\n",
              "51                        regression                False\n",
              "52              reprocess_input_data                 True\n",
              "53                   save_best_model                 True\n",
              "54             save_eval_checkpoints                 True\n",
              "55            save_model_every_epoch                 True\n",
              "56      save_optimizer_and_scheduler                 True\n",
              "57                        save_steps                 2000\n",
              "58                            silent                False\n",
              "59               skip_special_tokens                 True\n",
              "60                    sliding_window                False\n",
              "61                            stride                  0.8\n",
              "62                   tensorboard_dir                 None\n",
              "63                      thread_count                 None\n",
              "64                         tie_value                    1\n",
              "65                  train_batch_size                    8\n",
              "66      train_custom_parameters_only                False\n",
              "67          use_cached_eval_features                False\n",
              "68                use_early_stopping                False\n",
              "69               use_multiprocessing                 True\n",
              "70                      wandb_kwargs                   {}\n",
              "71                     wandb_project                 None\n",
              "72                      warmup_ratio                 0.06\n",
              "73                      warmup_steps                    0\n",
              "74                      weight_decay                    0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fZKbpevQ7a-h"
      },
      "source": [
        "### Noteworthy Args"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xmwuXj2urCVA"
      },
      "source": [
        "Below are what I'd consider the most critical arguments--they'll have the most immediate impact on your accuracy.\n",
        "\n",
        "What's displayed are their default values (except model-name, which we specified earlier)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0g7vCk6l7doI",
        "outputId": "cb8924ee-ce23-493e-93bd-84489726a496",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "model.args.train_batch_size"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "8"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lL-QZ_2w7fzL",
        "outputId": "9e7c6b43-5fb4-486e-a7b1-bb9b1c9b39fc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "model.args.num_train_epochs"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bI5rAPTA7jK0",
        "outputId": "b41d88b1-5c1a-4eac-fcb2-a33a6b0156e5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "model.args.max_seq_length"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "128"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bk6-Ye7x7poE",
        "outputId": "acda56ce-c8aa-4c1a-beed-331f0ade2246",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "model.args.learning_rate"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4e-05"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AK4ReVMV7mp0",
        "outputId": "87bfcea2-1450-498e-b4d1-437e5b7b5607",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "model.args.model_name"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'bert-base-uncased'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aJyF8Jb86w-6"
      },
      "source": [
        "## 3.3. Run Training"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6QGPheuGXqIV"
      },
      "source": [
        "#### Choose Hyperparameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4mHLBtrE-XTa"
      },
      "source": [
        "# These are the values we used in our Notebook:\n",
        "model.args.max_seq_length = 128\n",
        "model.args.num_train_epochs = 4\n",
        "model.args.train_batch_size = 32\n",
        "model.args.learning_rate = 2e-5"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wOz-aOhucHlf"
      },
      "source": [
        "#### Configure Validation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dMSDFNtEcLIt"
      },
      "source": [
        "Periodically evaluate on our 10% validation set during training to monitor over-fitting.\n",
        "\n",
        "Calculate number of steps so we can specify how often to evaluate."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ynY7BMUSVxFf",
        "outputId": "84751f74-6109-4e77-b676-1b12a3bddf49",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "# Num steps in epoch = num training samples / batch size\n",
        "steps_per_epoch = int(np.ceil(len(train_df) / float(model.args.train_batch_size)))\n",
        "\n",
        "print('Each epoch will have {:,} steps.'.format(steps_per_epoch))"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Each epoch will have 241 steps.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QSz3ovrXcZhe"
      },
      "source": [
        "Turn on validation."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W3h4WNIPcJe7"
      },
      "source": [
        "# Run evaluation periodically during training to monitor progress.\n",
        "model.args.evaluate_during_training = True\n",
        "\n",
        "# \"Print results from evaluation during training.\"\n",
        "model.args.evaluate_during_training_verbose = True\n",
        "\n",
        "# \"Perform evaluation at every specified number of steps. A checkpoint model and\n",
        "#  the evaluation results will be saved.\"\n",
        "model.args.evaluate_during_training_steps = 120\n",
        "\n",
        "# We only need to tokenize our validation set once, then we can read it from the\n",
        "# cache.\n",
        "model.args.use_cached_eval_features = True"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AHM3fSwmXs39"
      },
      "source": [
        "#### Configure Early Stopping"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OT1V5LmrXzwu"
      },
      "source": [
        "There's a nice intro to early stopping in their docs here:\n",
        "    \n",
        "* https://simpletransformers.ai/docs/usage/#using-early-stopping\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6m3mcaBnXjI_"
      },
      "source": [
        "# Turn on early stopping.\n",
        "model.args.use_early_stopping = True\n",
        "\n",
        "# \"The improvement over best_eval_loss necessary to count as a better checkpoint.\"\n",
        "model.args.early_stopping_delta = 0.01\n",
        "\n",
        "# What metric to use in calculating score for evaluation set (plus whether a low\n",
        "# vs. high value is better for this metric).\n",
        "\n",
        "#model.args.early_stopping_metric = \"mcc\"\n",
        "#model.args.early_stopping_metric_minimize = False\n",
        "\n",
        "model.args.early_stopping_metric = \"eval_loss\"\n",
        "model.args.early_stopping_metric_minimize = True\n",
        "\n",
        "# \"Terminate training after this many evaluations without an improvement in the\n",
        "#  evaluation metric greater then early_stopping_delta.\"\n",
        "model.args.early_stopping_patience = 2\n"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uagAitE4cMLt"
      },
      "source": [
        "## Specify wandb project\n",
        "\n",
        "For the simple transformers documentation. wandb_project:\n",
        "\n",
        "\"Name of W&B project. This will log all hypermeter values, training and evaluation metrics to the given project.\"\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qC1r3ndOc9nB"
      },
      "source": [
        "model.args.wandb_project = \"CoLA Sentence Classification\""
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VHbXqbckfqEo"
      },
      "source": [
        "#### Kick-Off Training"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AXqIwuILs3kO"
      },
      "source": [
        "The log statements aren't easy to interpret, but you can infer that it's running validation twice per epoch. \n",
        "\n",
        "There are also some statements related to early stopping, and if you look at the validation loss at each checkpoint, you can make sense of the early stopping behavior.\n",
        "\n",
        "I'm hoping/assuming that `wandb` integration will make this much easier to look at!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XDTTZ6n7DECX",
        "outputId": "9a8c76e1-3f44-4eb4-95e1-6f57b26d838f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "c9fa3271a01f4012851c2bba27ff9304",
            "227d0a21b7d240f3aeb1611bf493c805",
            "b3a6782ec67d43dcaf24096628096ee2",
            "fafa7a9e3e514cbea6663e8a7e11f786",
            "122e8f35668442bb8e6281fcb62908d2",
            "9e16d1245bb14ca8a22ae4a13aa73b58",
            "83fad9c9c6314b188efa2a822de1e9f4",
            "5de9acfa1ed74ce0ab854e28c02c14b0",
            "ca441852dc6349ada8af6763e0869864",
            "197c1f91ca7d45a2bd4f4d72dcc8817d",
            "724f288ef67d4733a3a6831649bc18ae",
            "240ad2eef9084cd48ac84f0fe5dbc21d",
            "a035940cbd5343f09d03f34e94188c0a",
            "5b6ee9cba1da455881a4c35d87f65111",
            "a0730b94983042a58b6256993e0e6171",
            "aa6682140f0b449f9e87d8760f56a154",
            "b810c464dff4466f92e8ab167b0eb074",
            "4368a25c19104aa2b84a07eadd56ece3",
            "75b76d7d818d40bdba4fc377d2aca365",
            "8325f226e8ed4cfbb58d0551208468df",
            "31adc08a085e40d4a5720349a8f960ff",
            "d4755a3822134c35ab9356914da3b46d",
            "dbf7444bd4d94c35a0dadd9a72bdf158",
            "80e9d26cb5b249d6b198d42724ad79a1",
            "00f3e5b04fb845cd8dafe400cbbe0b83",
            "ec984dc0a69c4ca797a220ddea4cca3c",
            "a6a9bffc2f414af1b54eb8902ee09cfa",
            "a931ed48e9b440789ea15570610f57d3",
            "32a46ed547c349a6bb65229ec700f56b",
            "069d9bb3da894a85aab92b2203a1ddf6",
            "7b73c420cf994dc8aec80317cff04d0e",
            "c7efbeecabc343f1b77b6f7c85808e5a",
            "4446d04e98b8420382ea319fdcd9935e",
            "b2ecef6e671a48009d3464b23ddfb677",
            "46dd63f373ed4f20a89faea6f7eb1f83",
            "37284221d6714d96a1e33078782ff113",
            "14788a81a8f64265a253f644dd9a7994",
            "423e7e920b9c4a179c82c3e5572ebfc5",
            "bd178473c5f04701899c6bbe51de583c",
            "a119aa261bb24a258ef8aa30207a9fd7",
            "5f242991fd334fb9b24da8678652e648",
            "9c2f541ea5ed42b78d0889b0c7d9bdec",
            "cbb83cacf76f45a397737c05196a55e5",
            "ce63ab99065e49449d62792426f84136",
            "c38350a6efa747e793c05505dc0ea7f8",
            "01bf8c075b6b4f9f92d7f1aee9e5fee1",
            "691cf6f2fc6548f081bee51cd425ebf0",
            "5b167cccf92a452294ceb1c5cadf29e7"
          ]
        }
      },
      "source": [
        "print('Training on {:,} samples...'.format(len(train_df)))\n",
        "\n",
        "# Train the model, testing against the validation set periodically.\n",
        "out = model.train_model(train_df, eval_df=validation_df)"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training on 7,695 samples...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/simpletransformers/classification/classification_model.py:353: UserWarning: Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\n",
            "  \"Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\"\n",
            "INFO:simpletransformers.classification.classification_model: Converting to features started. Cache is not used.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c9fa3271a01f4012851c2bba27ff9304",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=7695.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ca441852dc6349ada8af6763e0869864",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Epoch', max=4.0, style=ProgressStyle(description_width='i…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:wandb:setting login settings: {}\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mgulabpatel923-gmail-com\u001b[0m (use `wandb login --relogin` to force relogin)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Tracking run with wandb version 0.10.7<br/>\n",
              "                Syncing run <strong style=\"color:#cdcd00\">glorious-disco-5</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://wandb.ai/gulabpatel923-gmail-com/CoLA%20Sentence%20Classification\" target=\"_blank\">https://wandb.ai/gulabpatel923-gmail-com/CoLA%20Sentence%20Classification</a><br/>\n",
              "                Run page: <a href=\"https://wandb.ai/gulabpatel923-gmail-com/CoLA%20Sentence%20Classification/runs/iy50j3o5\" target=\"_blank\">https://wandb.ai/gulabpatel923-gmail-com/CoLA%20Sentence%20Classification/runs/iy50j3o5</a><br/>\n",
              "                Run data is saved locally in <code>wandb/run-20201022_055648-iy50j3o5</code><br/><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b810c464dff4466f92e8ab167b0eb074",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 0 of 4', max=241.0, style=ProgressStyle(des…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/optim/lr_scheduler.py:231: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
            "  warnings.warn(\"To get the last learning rate computed by the scheduler, \"\n",
            "/usr/local/lib/python3.6/dist-packages/simpletransformers/classification/classification_model.py:851: UserWarning: Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\n",
            "  \"Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\"\n",
            "INFO:simpletransformers.classification.classification_model: Converting to features started. Cache is not used.\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.18859792415161353, 'tp': 577, 'tn': 31, 'fp': 233, 'fn': 15, 'eval_loss': 0.5942512350243943}\n",
            "/usr/local/lib/python3.6/dist-packages/torch/optim/lr_scheduler.py:200: UserWarning: Please also save or load the state of the optimzer when saving or loading the scheduler.\n",
            "  warnings.warn(SAVE_STATE_WARNING, UserWarning)\n",
            "INFO:simpletransformers.classification.classification_model: No improvement in eval_loss\n",
            "INFO:simpletransformers.classification.classification_model: Current step: 1\n",
            "INFO:simpletransformers.classification.classification_model: Early stopping patience: 2\n",
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.48512568824435076, 'tp': 511, 'tn': 161, 'fp': 103, 'fn': 81, 'eval_loss': 0.48290322358920196}\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.49601389547899344, 'tp': 521, 'tn': 157, 'fp': 107, 'fn': 71, 'eval_loss': 0.47961354046781485}\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "00f3e5b04fb845cd8dafe400cbbe0b83",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 1 of 4', max=241.0, style=ProgressStyle(des…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.4301113924027745, 'tp': 539, 'tn': 123, 'fp': 141, 'fn': 53, 'eval_loss': 0.5054554238079865}\n",
            "INFO:simpletransformers.classification.classification_model: No improvement in eval_loss\n",
            "INFO:simpletransformers.classification.classification_model: Current step: 1\n",
            "INFO:simpletransformers.classification.classification_model: Early stopping patience: 2\n",
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.4943898227690984, 'tp': 519, 'tn': 158, 'fp': 106, 'fn': 73, 'eval_loss': 0.46376883910498884}\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.5017520185720736, 'tp': 518, 'tn': 161, 'fp': 103, 'fn': 74, 'eval_loss': 0.464010445488112}\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4446d04e98b8420382ea319fdcd9935e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 2 of 4', max=241.0, style=ProgressStyle(des…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.48221668588615024, 'tp': 536, 'tn': 141, 'fp': 123, 'fn': 56, 'eval_loss': 0.608484154654162}\n",
            "INFO:simpletransformers.classification.classification_model: No improvement in eval_loss\n",
            "INFO:simpletransformers.classification.classification_model: Current step: 1\n",
            "INFO:simpletransformers.classification.classification_model: Early stopping patience: 2\n",
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.4971682272680467, 'tp': 533, 'tn': 148, 'fp': 116, 'fn': 59, 'eval_loss': 0.545273852891454}\n",
            "INFO:simpletransformers.classification.classification_model: No improvement in eval_loss\n",
            "INFO:simpletransformers.classification.classification_model: Current step: 2\n",
            "INFO:simpletransformers.classification.classification_model: Early stopping patience: 2\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.49191585896432594, 'tp': 531, 'tn': 148, 'fp': 116, 'fn': 61, 'eval_loss': 0.5418517862002705}\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5f242991fd334fb9b24da8678652e648",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Running Epoch 3 of 4', max=241.0, style=ProgressStyle(des…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n",
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.49258640547224786, 'tp': 530, 'tn': 149, 'fp': 115, 'fn': 62, 'eval_loss': 0.6406358941066488}\n",
            "INFO:simpletransformers.classification.classification_model: Patience of 2 steps reached\n",
            "INFO:simpletransformers.classification.classification_model: Training terminated.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model: Training of bert model complete. Saved to outputs/.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hzu_I6j1e9vA",
        "outputId": "60dd1ecb-cf03-44b0-aaec-7ec756af903c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "#To see the current running project\n",
        "wandb.run.name"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'glorious-disco-5'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8axvKzRhfGrd",
        "outputId": "7f9d98ca-d70e-4143-9663-d4d11a737344",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "#Names are not unique you can have multiple run with the same name but ids are globally unique\n",
        "wandb.run.id"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'iy50j3o5'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g16D8lrJfikX",
        "outputId": "79ed92e3-a3d7-489d-ea33-36246e6bc171",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 869,
          "referenced_widgets": [
            "da07a5a701544818b3cdf3ca12573137",
            "21db04f54fca48f7850c4bedabc44eac",
            "d6d2c52652344d00934ad102bd51f097",
            "5d95b0375d144c2ebfe489928a6a0253",
            "319bbade2104490e949895ca6d5582c0",
            "7f4f89ed59d449de9e76f9409c1f4935",
            "8eaebd20acd445c58f4fe72e84ca756d",
            "5d89c19c0f0a4a38afc36fc6744cf396"
          ]
        }
      },
      "source": [
        "#It is optional, if you close the notebook will do the same things.\n",
        "wandb.finish()"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<br/>Waiting for W&B process to finish, PID 254<br/>Program ended successfully."
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "da07a5a701544818b3cdf3ca12573137",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "VBox(children=(Label(value=' 0.03MB of 0.03MB uploaded (0.00MB deduped)\\r'), FloatProgress(value=1.0, max=1.0)…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "Find user logs for this run at: <code>wandb/run-20201022_055648-iy50j3o5/logs/debug.log</code>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "Find internal logs for this run at: <code>wandb/run-20201022_055648-iy50j3o5/logs/debug-internal.log</code>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<h3>Run summary:</h3><br/><style>\n",
              "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: right }\n",
              "    </style><table class=\"wandb\">\n",
              "<tr><td>Training loss</td><td>0.06438</td></tr><tr><td>lr</td><td>0.0</td></tr><tr><td>global_step</td><td>840</td></tr><tr><td>_step</td><td>25</td></tr><tr><td>_runtime</td><td>470</td></tr><tr><td>_timestamp</td><td>1603346678</td></tr><tr><td>tp</td><td>530</td></tr><tr><td>tn</td><td>149</td></tr><tr><td>fp</td><td>115</td></tr><tr><td>fn</td><td>62</td></tr><tr><td>mcc</td><td>0.49259</td></tr><tr><td>train_loss</td><td>0.10537</td></tr><tr><td>eval_loss</td><td>0.64064</td></tr></table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<h3>Run history:</h3><br/><style>\n",
              "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: right }\n",
              "    </style><table class=\"wandb\">\n",
              "<tr><td>Training loss</td><td>██▆▇▇▅▆▇▃▃▁▄▄▅▃▁</td></tr><tr><td>lr</td><td>▇█▇▇▆▆▅▅▅▄▃▃▃▂▁▁</td></tr><tr><td>global_step</td><td>▁▁▂▂▂▃▃▃▃▄▄▄▅▅▅▅▅▆▆▆▇▇▇▇██</td></tr><tr><td>_step</td><td>▁▁▂▂▂▂▃▃▃▄▄▄▄▅▅▅▅▆▆▆▇▇▇▇██</td></tr><tr><td>_runtime</td><td>▁▁▁▂▂▂▃▄▄▄▄▄▄▅▆▆▆▆▆▆▇▇▇▇██</td></tr><tr><td>_timestamp</td><td>▁▁▁▂▂▂▃▄▄▄▄▄▄▅▆▆▆▆▆▆▇▇▇▇██</td></tr><tr><td>tp</td><td>█▁▂▄▂▂▄▃▃▃</td></tr><tr><td>tn</td><td>▁██▆██▇▇▇▇</td></tr><tr><td>fp</td><td>█▁▁▃▁▁▂▂▂▂</td></tr><tr><td>fn</td><td>▁█▇▅▇▇▅▆▆▆</td></tr><tr><td>mcc</td><td>▁██▆██████</td></tr><tr><td>train_loss</td><td>▇█▇▆▃▇▄▁▁▁</td></tr><tr><td>eval_loss</td><td>▆▂▂▃▁▁▇▄▄█</td></tr></table><br/>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "Synced 5 W&B file(s), 1 media file(s), 0 artifact file(s) and 0 other file(s)"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                    <br/>Synced <strong style=\"color:#cdcd00\">glorious-disco-5</strong>: <a href=\"https://wandb.ai/gulabpatel923-gmail-com/CoLA%20Sentence%20Classification/runs/iy50j3o5\" target=\"_blank\">https://wandb.ai/gulabpatel923-gmail-com/CoLA%20Sentence%20Classification/runs/iy50j3o5</a><br/>\n",
              "                "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GjoegimYIPZa"
      },
      "source": [
        "## 3.4. Inspect generated files"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CAIOLClZtoxG"
      },
      "source": [
        "Helper function to print the contents of a directory, with file sizes in MB."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D87pElNqYqVu"
      },
      "source": [
        "import os \n",
        "import pandas as pd\n",
        "\n",
        "def list_files_info(data_dir):\n",
        "    '''\n",
        "    Prints out the files in a directory along with their sizes in MB.\n",
        "    '''\n",
        "\n",
        "    # Check out the sizes on the saved files.\n",
        "    files = list(os.listdir(data_dir))\n",
        "\n",
        "    print(data_dir)\n",
        "\n",
        "    rows = []\n",
        "\n",
        "    # For each file in the directory...\n",
        "    for f in files:\n",
        "        # Get the file size, in MB\n",
        "        f_size = float(os.stat(data_dir + '/' + f).st_size) / 2**20\n",
        "        \n",
        "        # Print the filename and its size.\n",
        "        print(\"     {:25s}    {:>8.2f} MB\".format(f, f_size))\n",
        "\n",
        "        rows.append([f, '{:.2f} MB'.format(f_size)])\n",
        "\n",
        "    print('')\n",
        "\n",
        "    return pd.DataFrame(rows, columns=['File', 'Size'])\n"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aNjybmHcttGf"
      },
      "source": [
        "This cache folder stores the tokenized and encoded text data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cxMF67nqIbVC",
        "outputId": "15a942f9-ed1c-4b87-ecbf-f8b23d70e148",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 179
        }
      },
      "source": [
        "list_files_info('./cache_dir')"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "./cache_dir\n",
            "     cached_dev_bert_128_2_856        0.68 MB\n",
            "     cached_train_bert_128_2_7695        6.07 MB\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>File</th>\n",
              "      <th>Size</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>cached_dev_bert_128_2_856</td>\n",
              "      <td>0.68 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>cached_train_bert_128_2_7695</td>\n",
              "      <td>6.07 MB</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                           File     Size\n",
              "0     cached_dev_bert_128_2_856  0.68 MB\n",
              "1  cached_train_bert_128_2_7695  6.07 MB"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1Jsgufn6t2zW"
      },
      "source": [
        "The `outputs` folder contains the final model, plus all of the checkpoints."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EMIqYvSXI59a",
        "outputId": "07c39622-9681-4b7b-8d67-6ebafc2b5c15",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "list_files_info('./outputs/')"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "./outputs/\n",
            "     vocab.txt                        0.22 MB\n",
            "     checkpoint-120                   0.00 MB\n",
            "     checkpoint-482-epoch-2           0.00 MB\n",
            "     checkpoint-720                   0.00 MB\n",
            "     checkpoint-241-epoch-1           0.00 MB\n",
            "     checkpoint-840                   0.00 MB\n",
            "     special_tokens_map.json          0.00 MB\n",
            "     tokenizer_config.json            0.00 MB\n",
            "     checkpoint-360                   0.00 MB\n",
            "     checkpoint-480                   0.00 MB\n",
            "     eval_results.txt                 0.00 MB\n",
            "     model_args.json                  0.00 MB\n",
            "     checkpoint-723-epoch-3           0.00 MB\n",
            "     checkpoint-240                   0.00 MB\n",
            "     best_model                       0.00 MB\n",
            "     training_args.bin                0.00 MB\n",
            "     config.json                      0.00 MB\n",
            "     pytorch_model.bin              417.73 MB\n",
            "     checkpoint-600                   0.00 MB\n",
            "     training_progress_scores.csv        0.00 MB\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>File</th>\n",
              "      <th>Size</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>vocab.txt</td>\n",
              "      <td>0.22 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>checkpoint-120</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>checkpoint-482-epoch-2</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>checkpoint-720</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>checkpoint-241-epoch-1</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>checkpoint-840</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>special_tokens_map.json</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>tokenizer_config.json</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>checkpoint-360</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>checkpoint-480</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>eval_results.txt</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>model_args.json</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>checkpoint-723-epoch-3</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>checkpoint-240</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>best_model</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>training_args.bin</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>config.json</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>pytorch_model.bin</td>\n",
              "      <td>417.73 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>checkpoint-600</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>training_progress_scores.csv</td>\n",
              "      <td>0.00 MB</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                            File       Size\n",
              "0                      vocab.txt    0.22 MB\n",
              "1                 checkpoint-120    0.00 MB\n",
              "2         checkpoint-482-epoch-2    0.00 MB\n",
              "3                 checkpoint-720    0.00 MB\n",
              "4         checkpoint-241-epoch-1    0.00 MB\n",
              "5                 checkpoint-840    0.00 MB\n",
              "6        special_tokens_map.json    0.00 MB\n",
              "7          tokenizer_config.json    0.00 MB\n",
              "8                 checkpoint-360    0.00 MB\n",
              "9                 checkpoint-480    0.00 MB\n",
              "10              eval_results.txt    0.00 MB\n",
              "11               model_args.json    0.00 MB\n",
              "12        checkpoint-723-epoch-3    0.00 MB\n",
              "13                checkpoint-240    0.00 MB\n",
              "14                    best_model    0.00 MB\n",
              "15             training_args.bin    0.00 MB\n",
              "16                   config.json    0.00 MB\n",
              "17             pytorch_model.bin  417.73 MB\n",
              "18                checkpoint-600    0.00 MB\n",
              "19  training_progress_scores.csv    0.00 MB"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cLbK5JgBFkua"
      },
      "source": [
        "## 3.5. Evaluate on Test Set\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0QarqhLuxfHN"
      },
      "source": [
        "Let's load the model from the checkpoint which performed best on the validation set. \n",
        "\n",
        "This is how we combat overfitting--the \"final\" model (at the end of all training epochs) will perform best on the training set, but may not generalize as well to new data. \n",
        "\n",
        "So, instead, we use an earlier checkpoint where the training loss was higher but the validation loss was at its lowest! "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "shHc3hNfh1OG"
      },
      "source": [
        "model = ClassificationModel(\n",
        "    \"bert\", \"outputs/best_model\"\n",
        ")"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "apzWNC1pAHO7"
      },
      "source": [
        "**On Validation Set**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l0b1d39OEY4e",
        "outputId": "25589c82-8e81-4e6b-ccb3-39452d5e26ad",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 273,
          "referenced_widgets": [
            "93c03626c6b74b4f86ae354d20c22a73",
            "d6ce537c0d7146c69886962883b34c7f",
            "face2bd343124892a18648b38b649e0e",
            "208be4b240e9455997a9a6eb65ab86cc",
            "8ed1ad5ca61448888aa19c1aae124fce",
            "42e4a09e13014328b524b0df7e3097b7",
            "fcab53656afa43febe2ec92f9972523b",
            "41f435ff4e2a4adb9c352f3c69fbf39b"
          ]
        }
      },
      "source": [
        "# Evaluate the model\n",
        "result, model_outputs, wrong_predictions = model.eval_model(validation_df)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/simpletransformers/classification/classification_model.py:851: UserWarning: Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\n",
            "  \"Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\"\n",
            "INFO:simpletransformers.classification.classification_model: Features loaded from cache at cache_dir/cached_dev_bert_128_2_856\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "93c03626c6b74b4f86ae354d20c22a73",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=107.0, style=ProgressStyle(descr…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Tracking run with wandb version 0.10.7<br/>\n",
              "                Syncing run <strong style=\"color:#cdcd00\">cool-dream-6</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://wandb.ai/gulabpatel923-gmail-com/CoLA%20Sentence%20Classification\" target=\"_blank\">https://wandb.ai/gulabpatel923-gmail-com/CoLA%20Sentence%20Classification</a><br/>\n",
              "                Run page: <a href=\"https://wandb.ai/gulabpatel923-gmail-com/CoLA%20Sentence%20Classification/runs/o7xe4bf4\" target=\"_blank\">https://wandb.ai/gulabpatel923-gmail-com/CoLA%20Sentence%20Classification/runs/o7xe4bf4</a><br/>\n",
              "                Run data is saved locally in <code>wandb/run-20201022_060511-o7xe4bf4</code><br/><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.4943898227690984, 'tp': 519, 'tn': 158, 'fp': 106, 'fn': 73, 'eval_loss': 0.46376883910498884}\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u24h6JjSFsJ_",
        "outputId": "501c20fc-30fb-4e62-f5c5-683f371051e8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "print(result)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'mcc': 0.4943898227690984, 'tp': 519, 'tn': 158, 'fp': 106, 'fn': 73, 'eval_loss': 0.46376883910498884}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GbE5RYSfAKM8"
      },
      "source": [
        "**On Test Set**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-riuk2sEDuva",
        "outputId": "09128ef3-9173-4ba3-ebb3-594be987e1bf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        }
      },
      "source": [
        "df_test.sample(10)"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>179</th>\n",
              "      <td>I believe there to be no way out.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>166</th>\n",
              "      <td>I want goes to the store.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>245</th>\n",
              "      <td>If only Superman would stop flying planes!</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>250</th>\n",
              "      <td>I'm creating a committee. Kim – you're in charge.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>230</th>\n",
              "      <td>We appeared to them to vote for themselves.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>437</th>\n",
              "      <td>Of whom are you thinking?</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>247</th>\n",
              "      <td>I know you like the back of my hand.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92</th>\n",
              "      <td>The book was seen by Mary.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>124</th>\n",
              "      <td>Simon quickly dropped the gold, and Jack the d...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>425</th>\n",
              "      <td>Nobody saw Pat.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                  text  label\n",
              "179                  I believe there to be no way out.      1\n",
              "166                          I want goes to the store.      0\n",
              "245         If only Superman would stop flying planes!      1\n",
              "250  I'm creating a committee. Kim – you're in charge.      1\n",
              "230        We appeared to them to vote for themselves.      0\n",
              "437                          Of whom are you thinking?      1\n",
              "247               I know you like the back of my hand.      1\n",
              "92                          The book was seen by Mary.      1\n",
              "124  Simon quickly dropped the gold, and Jack the d...      1\n",
              "425                                    Nobody saw Pat.      1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NsTOCd8fALl5",
        "outputId": "6a25515d-f3f3-4848-995b-ae665feee004",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 771,
          "referenced_widgets": [
            "35d304cc89b44b84b5d83f6da3399610",
            "c30c404d452742488da61fd64e15d586",
            "248777aa2c1249d5ad205045ad9cc55a",
            "261d84a5fe7c4c1d9064e3215cbf07f2",
            "7837f02e612444eebd8ff5d63be4fa6b",
            "fd6aacba6eee44ca867dcea3d8b99bae",
            "ddc6a6747b494b8d931ba6df826df0e9",
            "c8ee119378ca4e41bdf437239d9ea21a",
            "568cc76da4724e7a933273565c46dcf5",
            "f50b3922cd224a959d91e6ad5aa4a4ec",
            "775d36fc2089480ea6104e145afb7abb",
            "aad895bafcbf4a9c98481a1621229d63",
            "509d43d800ac4bb2b03256ff9b5f0e33",
            "177e4f2e8283459692fe85ef9398a24e",
            "3659634e6f254834b3e5357dfa7b891e",
            "424e49691ecc4e37aa693c1ed6e667ae",
            "0d5db9c2b5c243fd84b6f4f9372cc389",
            "8d4980906853428d80e5c86ac571c7ae",
            "3a1502bec5ee4b58826cfac5e7f4bad5",
            "6d2cd6b24f2d4bea844e41ba87c7c873",
            "6ab2a52998b14812a0c7db0f3fcb5f8b",
            "b52f2c27057b468582249e42f006a956",
            "e55ab07e49fd4c8ab7c5fff65c9d5a9f",
            "3cac61ab338b406581d0b47ffe6d12e9"
          ]
        }
      },
      "source": [
        "# Evaluate the model\n",
        "result, model_outputs, wrong_predictions = model.eval_model(df_test)"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/simpletransformers/classification/classification_model.py:851: UserWarning: Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\n",
            "  \"Dataframe headers not specified. Falling back to using column 0 as text and column 1 as labels.\"\n",
            "INFO:simpletransformers.classification.classification_model: Converting to features started. Cache is not used.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "35d304cc89b44b84b5d83f6da3399610",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=516.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "568cc76da4724e7a933273565c46dcf5",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Running Evaluation', max=65.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Calling wandb.login() after wandb.init() has no effect.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<br/>Waiting for W&B process to finish, PID 384<br/>Program ended successfully."
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0d5db9c2b5c243fd84b6f4f9372cc389",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "VBox(children=(Label(value=' 0.01MB of 0.01MB uploaded (0.00MB deduped)\\r'), FloatProgress(value=1.0, max=1.0)…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "Find user logs for this run at: <code>wandb/run-20201022_060511-o7xe4bf4/logs/debug.log</code>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "Find internal logs for this run at: <code>wandb/run-20201022_060511-o7xe4bf4/logs/debug-internal.log</code>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<h3>Run summary:</h3><br/><style>\n",
              "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: right }\n",
              "    </style><table class=\"wandb\">\n",
              "<tr><td>_step</td><td>2</td></tr><tr><td>_runtime</td><td>4</td></tr><tr><td>_timestamp</td><td>1603346715</td></tr></table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<h3>Run history:</h3><br/><style>\n",
              "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: right }\n",
              "    </style><table class=\"wandb\">\n",
              "<tr><td>_step</td><td>▁▅█</td></tr><tr><td>_runtime</td><td>▁▁▁</td></tr><tr><td>_timestamp</td><td>▁▁▁</td></tr></table><br/>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "Synced 5 W&B file(s), 3 media file(s), 0 artifact file(s) and 0 other file(s)"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                    <br/>Synced <strong style=\"color:#cdcd00\">cool-dream-6</strong>: <a href=\"https://wandb.ai/gulabpatel923-gmail-com/CoLA%20Sentence%20Classification/runs/o7xe4bf4\" target=\"_blank\">https://wandb.ai/gulabpatel923-gmail-com/CoLA%20Sentence%20Classification/runs/o7xe4bf4</a><br/>\n",
              "                "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Tracking run with wandb version 0.10.7<br/>\n",
              "                Syncing run <strong style=\"color:#cdcd00\">drawn-dew-7</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://wandb.ai/gulabpatel923-gmail-com/CoLA%20Sentence%20Classification\" target=\"_blank\">https://wandb.ai/gulabpatel923-gmail-com/CoLA%20Sentence%20Classification</a><br/>\n",
              "                Run page: <a href=\"https://wandb.ai/gulabpatel923-gmail-com/CoLA%20Sentence%20Classification/runs/orv3hk2v\" target=\"_blank\">https://wandb.ai/gulabpatel923-gmail-com/CoLA%20Sentence%20Classification/runs/orv3hk2v</a><br/>\n",
              "                Run data is saved locally in <code>wandb/run-20201022_060517-orv3hk2v</code><br/><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model:{'mcc': 0.4580946050400422, 'tp': 324, 'tn': 79, 'fp': 83, 'fn': 30, 'eval_loss': 0.5217112677601667}\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KfhF0PFffBPY"
      },
      "source": [
        "Without SimpleTransformers, original score: `Total MCC: 0.498`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3z8W6lwnBhr8",
        "outputId": "42b40f8e-4cd5-43ae-bbe5-906161e6c742",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "result"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'eval_loss': 0.5217112677601667,\n",
              " 'fn': 30,\n",
              " 'fp': 83,\n",
              " 'mcc': 0.4580946050400422,\n",
              " 'tn': 79,\n",
              " 'tp': 324}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zQBwmsNhAT90",
        "outputId": "4187ddca-7554-43fb-d15a-12b686af3efa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print('MCC: %.3f' % result['mcc'])"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "MCC: 0.458\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Id2KBRzOB1-C",
        "outputId": "58fd02f1-bd04-4300-bdb6-2e51e197d054",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "model_outputs"
      ],
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-0.73535156,  0.93603516],\n",
              "       [-0.89111328,  1.04394531],\n",
              "       [-1.06542969,  1.23339844],\n",
              "       ...,\n",
              "       [-1.74023438,  1.52050781],\n",
              "       [-0.33227539,  0.42602539],\n",
              "       [-0.30151367,  0.51074219]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gp5B7bc5hq9Q"
      },
      "source": [
        "**Run on New Text**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qu724mANFjbp",
        "outputId": "f798f887-88b6-43ea-bd2b-2dfdc51f7e7a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 152,
          "referenced_widgets": [
            "b7c9d1546ced479cba6f41ec52a62d65",
            "6e4abe737ca34ff79f93f045afad2017",
            "1c6c2fb5fd9d44b1ba39ea643675bbc5",
            "976bd0734bbe4d9aafa4e83e3dae8eae",
            "9dd62fc5f7454a3ca3c8cea3d89a6f84",
            "7d9827c112124686854006fedc163862",
            "00b0aeea5e25488e8801635c6e59ebf4",
            "2909f5f010b94ff8b1dc2d44f5908c5a",
            "c646e1a0b2e54ace90ca2a6773593266",
            "909c5ec52d794aafa675466fe6ea4cc3",
            "debe1c2a53274984ac7c977807dd7fde",
            "4b5ee7c0979b434b9f10386daafc9f28",
            "118df561cedb4b44b85beb233dfddc92",
            "24015b3b265e493786d359de33953214",
            "7a1d19cd4e5e4f6b961e33da3a4ee1d5",
            "c028a3c185174f409cf9be4712f71e49"
          ]
        }
      },
      "source": [
        "predictions, raw_outputs = model.predict([\"Some arbitary sentence\"])"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:simpletransformers.classification.classification_model: Converting to features started. Cache is not used.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b7c9d1546ced479cba6f41ec52a62d65",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c646e1a0b2e54ace90ca2a6773593266",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IKzLS9ohzGVu"
      },
      "source": [
        "# S4. Conclusion"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FL_NnDGxRpEI"
      },
      "source": [
        "## 4.1. Observations\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vv_uUVM8bcd3"
      },
      "source": [
        "**Less Code, More Documentation**\n",
        "\n",
        "* No more:\n",
        "    * Tokenization code\n",
        "    * PyTorch taining loop code\n",
        "\n",
        "* Instead --> Careful argument selection!\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fu0sZSOse_qb"
      },
      "source": [
        "\n",
        "**Colab Compatibility**\n",
        "\n",
        "* Alarming install errors, but no problems yet...\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bwDtGNIjfAtW"
      },
      "source": [
        "**Checkpoints**\n",
        "\n",
        "* Automatically saves checkpoints!\n",
        "* Loading back a model from a checkpoint is trivial!\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zkNa5MBlfCI-"
      },
      "source": [
        "\n",
        "**Early Stopping**\n",
        "\n",
        "* Harder than it sounds:\n",
        "    * You must specify:\n",
        "        * eval frequency\n",
        "        * minimum \"delta\"        \n",
        "        * \"patience\"\n",
        "\n",
        "* Didn't immediately improve our CoLA score...\n",
        "     * But we're not done exploring the technique yet!\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "--wEDqN9dExi"
      },
      "source": [
        "## 4.2. Next Steps"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8DT0bZZUdGMF"
      },
      "source": [
        "**Weights & Biases - \"wandb\"**\n",
        "\n",
        "* Set this up and try it out!\n",
        "    * Does it plot the learning curve for us?\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jdqzhWoYfDdU"
      },
      "source": [
        "\n",
        "**Make Early Stopping Work**\n",
        "\n",
        "* Is it easier to apply to a different dataset?\n",
        "* Was the final model better than the \"best model\"?\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Rb5-Jgu-fEPH"
      },
      "source": [
        "\n",
        "**Resume Training from Checkpoint**\n",
        "\n",
        "* Checkpoints should allow us to resume training in case we get disconnected from Colab...\n",
        "    * How do you resume?\n",
        "    * Save our checkpoints to GDrive (otherwise they'll be lost!)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KkM5QS7-BWrU"
      },
      "source": [
        "# A.1. Over-Fitting from Original Notebook"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6O_NbXFGMukX",
        "outputId": "10c12baf-329f-40ee-c210-4b61b5abf111",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 390
        }
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Display floats with two decimal places.\n",
        "pd.set_option('precision', 2)\n",
        "\n",
        "# Create a DataFrame from our training statistics.\n",
        "#df_stats = pd.DataFrame(data=training_stats)\n",
        "df_stats = pd.read_csv(\"/content/outputs/training_progress_scores.csv\")\n",
        "# Use the 'epoch' as the row index.\n",
        "df_stats = df_stats.set_index('global_step')\n",
        "\n",
        "# A hack to force the column headers to wrap.\n",
        "#df = df.style.set_table_styles([dict(selector=\"th\",props=[('max-width', '70px')])])\n",
        "\n",
        "# Display the table.\n",
        "df_stats"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tp</th>\n",
              "      <th>tn</th>\n",
              "      <th>fp</th>\n",
              "      <th>fn</th>\n",
              "      <th>mcc</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>eval_loss</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>global_step</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>120</th>\n",
              "      <td>577</td>\n",
              "      <td>31</td>\n",
              "      <td>233</td>\n",
              "      <td>15</td>\n",
              "      <td>0.19</td>\n",
              "      <td>0.46</td>\n",
              "      <td>0.59</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>240</th>\n",
              "      <td>511</td>\n",
              "      <td>161</td>\n",
              "      <td>103</td>\n",
              "      <td>81</td>\n",
              "      <td>0.49</td>\n",
              "      <td>0.49</td>\n",
              "      <td>0.48</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>241</th>\n",
              "      <td>521</td>\n",
              "      <td>157</td>\n",
              "      <td>107</td>\n",
              "      <td>71</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.41</td>\n",
              "      <td>0.48</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>360</th>\n",
              "      <td>539</td>\n",
              "      <td>123</td>\n",
              "      <td>141</td>\n",
              "      <td>53</td>\n",
              "      <td>0.43</td>\n",
              "      <td>0.40</td>\n",
              "      <td>0.51</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>480</th>\n",
              "      <td>519</td>\n",
              "      <td>158</td>\n",
              "      <td>106</td>\n",
              "      <td>73</td>\n",
              "      <td>0.49</td>\n",
              "      <td>0.19</td>\n",
              "      <td>0.46</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>482</th>\n",
              "      <td>518</td>\n",
              "      <td>161</td>\n",
              "      <td>103</td>\n",
              "      <td>74</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.46</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>600</th>\n",
              "      <td>536</td>\n",
              "      <td>141</td>\n",
              "      <td>123</td>\n",
              "      <td>56</td>\n",
              "      <td>0.48</td>\n",
              "      <td>0.28</td>\n",
              "      <td>0.61</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>720</th>\n",
              "      <td>533</td>\n",
              "      <td>148</td>\n",
              "      <td>116</td>\n",
              "      <td>59</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.55</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>723</th>\n",
              "      <td>531</td>\n",
              "      <td>148</td>\n",
              "      <td>116</td>\n",
              "      <td>61</td>\n",
              "      <td>0.49</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.54</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>840</th>\n",
              "      <td>530</td>\n",
              "      <td>149</td>\n",
              "      <td>115</td>\n",
              "      <td>62</td>\n",
              "      <td>0.49</td>\n",
              "      <td>0.11</td>\n",
              "      <td>0.64</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              tp   tn   fp  fn   mcc  train_loss  eval_loss\n",
              "global_step                                                \n",
              "120          577   31  233  15  0.19        0.46       0.59\n",
              "240          511  161  103  81  0.49        0.49       0.48\n",
              "241          521  157  107  71  0.50        0.41       0.48\n",
              "360          539  123  141  53  0.43        0.40       0.51\n",
              "480          519  158  106  73  0.49        0.19       0.46\n",
              "482          518  161  103  74  0.50        0.45       0.46\n",
              "600          536  141  123  56  0.48        0.28       0.61\n",
              "720          533  148  116  59  0.50        0.10       0.55\n",
              "723          531  148  116  61  0.49        0.10       0.54\n",
              "840          530  149  115  62  0.49        0.11       0.64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1-G03mmwH3aI"
      },
      "source": [
        "Notice that, while the the training loss is going down with each epoch, the validation loss is increasing! This suggests that we are training our model too long, and it's over-fitting on the training data. \n",
        "\n",
        "(For reference, we are using 7,695 training samples and 856 validation samples).\n",
        "\n",
        "Validation Loss is a more precise measure than accuracy, because with accuracy we don't care about the exact output value, but just which side of a threshold it falls on. \n",
        "\n",
        "If we are predicting the correct answer, but with less confidence, then validation loss will catch this, while accuracy will not."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "68xreA9JAmG5",
        "outputId": "99008d1d-dd88-4bb5-af07-91f4d2e54a8d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 365
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "% matplotlib inline\n",
        "\n",
        "import seaborn as sns\n",
        "\n",
        "# Use plot styling from seaborn.\n",
        "sns.set(style='darkgrid')\n",
        "\n",
        "# Increase the plot size and font size.\n",
        "sns.set(font_scale=1.5)\n",
        "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
        "\n",
        "# Plot the learning curve.\n",
        "plt.plot(df_stats['train_loss'], 'b-o', label=\"Training\")\n",
        "plt.plot(df_stats['eval_loss'], 'g-o', label=\"Validation\")\n",
        "\n",
        "# Label the plot.\n",
        "plt.title(\"Training & Validation Loss\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "plt.xticks([1, 2, 3, 4])\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuUAAAGaCAYAAACopj13AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd1iUZ9o28HM6vVdpIkoRARFrNLErUaOxG401Maa9mzWboptk90128+0bU9yUNRuTaNTYFVsSTVFjYqISGzZARUUQKVJnGJj6fH8Aowgoo8ADw/k7jhyGp801I8I591z3/UgEQRBARERERESikYpdABERERFRe8dQTkREREQkMoZyIiIiIiKRMZQTEREREYmMoZyIiIiISGQM5UREREREImMoJyKblZ2djYiICHz88cf3fI1FixYhIiKiCauyXQ293hEREVi0aFGjrvHxxx8jIiIC2dnZTV5fUlISIiIicOTIkSa/NhHR/ZKLXQARtR/WhNu9e/ciMDCwGatpe7RaLf773//iu+++Q35+Pjw8PJCQkIBnn30WYWFhjbrGn/70J3z//ffYvn07oqKi6j1GEAQMHToUZWVlOHjwIOzs7JryaTSrI0eOIDk5GbNnz4aLi4vY5dSRnZ2NoUOHYsaMGfjb3/4mdjlE1IowlBNRi1myZEmtr48dO4aNGzdi6tSpSEhIqLXPw8Pjvh8vICAAp06dgkwmu+dr/OMf/8Cbb75537U0hddffx3ffvstxowZg969e6OgoAD79u1DSkpKo0P5pEmT8P3332Pr1q14/fXX6z3m8OHDuHbtGqZOndokgfzUqVOQSlvmg9nk5GR88sknGD9+fJ1QPm7cOIwePRoKhaJFaiEisgZDORG1mHHjxtX62mQyYePGjejevXudfbfTaDRwcnKy6vEkEglUKpXVdd6qtQS4iooK7NmzBwMGDMD7779v2f78889Dr9c3+joDBgyAv78/du3ahVdeeQVKpbLOMUlJSQCqAnxTuN+/g6Yik8nu6w0aEVFzYk85EbU6Q4YMwcyZM3Hu3Dk88cQTSEhIwNixYwFUhfOlS5di8uTJ6NOnD7p164bhw4fjvffeQ0VFRa3r1NfjfOu2/fv3Y+LEiYiJicGAAQPwzjvvwGg01rpGfT3lNdvUajX+/ve/o1+/foiJicG0adOQkpJS5/kUFxdj8eLF6NOnD+Lj4zFr1iycO3cOM2fOxJAhQxr1mkgkEkgkknrfJNQXrBsilUoxfvx4lJSUYN++fXX2azQa/PDDDwgPD0dsbKxVr3dD6uspN5vN+OyzzzBkyBDExMRgzJgx2LlzZ73nZ2Rk4H//938xevRoxMfHIy4uDhMmTMDmzZtrHbdo0SJ88sknAIChQ4ciIiKi1t9/Qz3lRUVFePPNNzFw4EB069YNAwcOxJtvvoni4uJax9Wcf+jQIXz55ZcYNmwYunXrhpEjR2Lbtm2Nei2skZaWhueeew59+vRBTEwMRo0ahc8//xwmk6nWcdevX8fixYsxePBgdOvWDf369cO0adNq1WQ2m/HVV1/hkUceQXx8PHr06IGRI0fir3/9KwwGQ5PXTkTW40g5EbVKOTk5mD17NhITEzFixAhotVoAQF5eHrZs2YIRI0ZgzJgxkMvlSE5OxhdffIHU1FR8+eWXjbr+gQMHsG7dOkybNg0TJ07E3r17sWLFCri6uuLpp59u1DWeeOIJeHh44LnnnkNJSQlWrlyJp556Cnv37rWM6uv1esydOxepqamYMGECYmJikJ6ejrlz58LV1bXRr4ednR0effRRbN26Fd988w3GjBnT6HNvN2HCBHz66adISkpCYmJirX3ffvstKisrMXHiRABN93rf7l//+hdWr16NXr16Yc6cOSgsLMRbb72FoKCgOscmJyfj6NGjGDRoEAIDAy2fGrz++usoKirCggULAABTp06FRqPBjz/+iMWLF8Pd3R3AnecyqNVqPPbYY8jMzMTEiRPRtWtXpKamYv369Th8+DA2b95c5xOapUuXorKyElOnToVSqcT69euxaNEiBAcH12nDulenT5/GzJkzIZfLMWPGDHh5eWH//v147733kJaWZvm0xGg0Yu7cucjLy8P06dPRsWNHaDQapKen4+jRoxg/fjwA4NNPP8VHH32EwYMHY9q0aZDJZMjOzsa+ffug1+tbzSdCRO2aQEQkkq1btwrh4eHC1q1ba20fPHiwEB4eLmzatKnOOTqdTtDr9XW2L126VAgPDxdSUlIs27KysoTw8HDho48+qrMtLi5OyMrKsmw3m83C6NGjhf79+9e67quvviqEh4fXu+3vf/97re3fffedEB4eLqxfv96y7euvvxbCw8OFZcuW1Tq2ZvvgwYPrPJf6qNVqYf78+UK3bt2Erl27Ct9++22jzmvIrFmzhKioKCEvL6/W9ilTpgjR0dFCYWGhIAj3/3oLgiCEh4cLr776quXrjIwMISIiQpg1a5ZgNBot28+cOSNEREQI4eHhtf5uysvL6zy+yWQSHn/8caFHjx616vvoo4/qnF+j5vvt8OHDlm0ffPCBEB4eLnz99de1jq35+1m6dGmd88eNGyfodDrL9tzcXCE6OlpYuHBhnce8Xc1r9Oabb97xuKlTpwpRUVFCamqqZZvZbBb+9Kc/CeHh4cLvv/8uCIIgpKamCuHh4cLy5cvveL1HH31UePjhh+9aHxGJh+0rRNQqubm5YcKECXW2K5VKy6ie0WhEaWkpioqK8MADDwBAve0j9Rk6dGit1V0kEgn69OmDgoIClJeXN+oac+bMqfV13759AQCZmZmWbfv374dMJsOsWbNqHTt58mQ4Ozs36nHMZjNeeOEFpKWlYffu3XjooYfw0ksvYdeuXbWOe+ONNxAdHd2oHvNJkybBZDJh+/btlm0ZGRk4efIkhgwZYplo21Sv96327t0LQRAwd+7cWj3e0dHR6N+/f53jHRwcLP+v0+lQXFyMkpIS9O/fHxqNBpcuXbK6hho//vgjPDw8MHXq1Frbp06dCg8PD/z00091zpk+fXqtliFfX1+EhobiypUr91zHrQoLC3HixAkMGTIEkZGRlu0SiQTPPPOMpW4Alu+hI0eOoLCwsMFrOjk5IS8vD0ePHm2SGomo6bF9hYhapaCgoAYn5a1duxYbNmzAxYsXYTaba+0rLS1t9PVv5+bmBgAoKSmBo6Oj1deoaZcoKSmxbMvOzoaPj0+d6ymVSgQGBqKsrOyuj7N3714cPHgQ7777LgIDA/Hhhx/i+eefxyuvvAKj0WhpUUhPT0dMTEyjesxHjBgBFxcXJCUl4amnngIAbN26FQAsrSs1muL1vlVWVhYAoFOnTnX2hYWF4eDBg7W2lZeX45NPPsHu3btx/fr1Ouc05jVsSHZ2Nrp16wa5vPavQ7lcjo4dO+LcuXN1zmnoe+fatWv3XMftNQFA586d6+zr1KkTpFKp5TUMCAjA008/jeXLl2PAgAGIiopC3759kZiYiNjYWMt5L774Ip577jnMmDEDPj4+6N27NwYNGoSRI0daNSeBiJoPQzkRtUr29vb1bl+5ciX+7//+DwMGDMCsWbPg4+MDhUKBvLw8LFq0CIIgNOr6d1qF436v0djzG6tmYmKvXr0AVAX6Tz75BM888wwWL14Mo9GIyMhIpKSk4O23327UNVUqFcaMGYN169bh+PHjiIuLw86dO+Hn54cHH3zQclxTvd734y9/+Qt+/vlnTJkyBb169YKbmxtkMhkOHDiAr776qs4bhebWUss7NtbChQsxadIk/Pzzzzh69Ci2bNmCL7/8Ek8++SRefvllAEB8fDx+/PFHHDx4EEeOHMGRI0fwzTff4NNPP8W6dessb0iJSDwM5UTUpuzYsQMBAQH4/PPPa4WjX375RcSqGhYQEIBDhw6hvLy81mi5wWBAdnZ2o25wU/M8r127Bn9/fwBVwXzZsmV4+umn8cYbbyAgIADh4eF49NFHG13bpEmTsG7dOiQlJaG0tBQFBQV4+umna72uzfF614w0X7p0CcHBwbX2ZWRk1Pq6rKwMP//8M8aNG4e33nqr1r7ff/+9zrUlEonVtVy+fBlGo7HWaLnRaMSVK1fqHRVvbjVtVRcvXqyz79KlSzCbzXXqCgoKwsyZMzFz5kzodDo88cQT+OKLLzBv3jx4enoCABwdHTFy5EiMHDkSQNUnIG+99Ra2bNmCJ598spmfFRHdTet6u09EdBdSqRQSiaTWCK3RaMTnn38uYlUNGzJkCEwmE1avXl1r+6ZNm6BWqxt1jYEDBwKoWvXj1n5xlUqFDz74AC4uLsjOzsbIkSPrtGHcSXR0NKKiovDdd99h7dq1kEgkddYmb47Xe8iQIZBIJFi5cmWt5f3Onj1bJ2jXvBG4fUQ+Pz+/zpKIwM3+88a21QwbNgxFRUV1rrVp0yYUFRVh2LBhjbpOU/L09ER8fDz279+P8+fPW7YLgoDly5cDAIYPHw6gavWY25c0VKlUltagmtehqKiozuNER0fXOoaIxMWRciJqUxITE/H+++9j/vz5GD58ODQaDb755hurwmhLmjx5MjZs2IB///vfuHr1qmVJxD179iAkJKTOuuj16d+/PyZNmoQtW7Zg9OjRGDduHPz8/JCVlYUdO3YAqApY//nPfxAWFoaHH3640fVNmjQJ//jHP/Drr7+id+/edUZgm+P1DgsLw4wZM/D1119j9uzZGDFiBAoLC7F27VpERkbW6uN2cnJC//79sXPnTtjZ2SEmJgbXrl3Dxo0bERgYWKt/HwDi4uIAAO+99x4eeeQRqFQqdOnSBeHh4fXW8uSTT2LPnj146623cO7cOURFRSE1NRVbtmxBaGhos40gnzlzBsuWLauzXS6X46mnnsJrr72GmTNnYsaMGZg+fTq8vb2xf/9+HDx4EGPGjEG/fv0AVLU2vfHGGxgxYgRCQ0Ph6OiIM2fOYMuWLYiLi7OE81GjRqF79+6IjY2Fj48PCgoKsGnTJigUCowePbpZniMRWad1/hYjImrAE088AUEQsGXLFrz99tvw9vbGww8/jIkTJ2LUqFFil1eHUqnEqlWrsGTJEuzduxe7d+9GbGwsvvrqK7z22muorKxs1HXefvtt9O7dGxs2bMCXX34Jg8GAgIAAJCYmYt68eVAqlZg6dSpefvllODs7Y8CAAY267iOPPIIlS5ZAp9PVmeAJNN/r/dprr8HLywubNm3CkiVL0LFjR/ztb39DZmZmncmV7777Lt5//33s27cP27ZtQ8eOHbFw4ULI5XIsXry41rEJCQl46aWXsGHDBrzxxhswGo14/vnnGwzlzs7OWL9+PT766CPs27cPSUlJ8PT0xLRp0/A///M/Vt9FtrFSUlLqXblGqVTiqaeeQkxMDDZs2ICPPvoI69evh1arRVBQEF566SXMmzfPcnxERASGDx+O5ORk7Nq1C2azGf7+/liwYEGt4+bNm4cDBw5gzZo1UKvV8PT0RFxcHBYsWFBrhRciEo9EaIlZOkREVIvJZELfvn0RGxt7zzfgISIi28GeciKiZlbfaPiGDRtQVlZW77rcRETU/rB9hYiomb3++uvQ6/WIj4+HUqnEiRMn8M033yAkJARTpkwRuzwiImoF2L5CRNTMtm/fjrVr1+LKlSvQarXw9PTEwIED8cILL8DLy0vs8oiIqBVgKCciIiIiEhl7yomIiIiIRMZQTkREREQkMk70vE1hoUbsEoiIiIjIBkmlEri7O9a7j6H8NmYzW+yJiIiIqGWxfYWIiIiISGQM5UREREREImMoJyIiIiISGUM5EREREZHIGMqJiIiIiETGUE5EREREJDKGciIiIiIikTGUExERERGJjKGciIiIiEhkvKMnEREREbU7ybnHsTNjD4p1JXBXuWFsWCJ6+/UQrR6GciIiIiJqV5Jzj2Nd2lYYzAYAQLGuBOvStgKAaMGc7StERERE1K7szNhjCeQ1DGYDdmbsEakihnIiIiIiakfMghnFupJ69zW0vSWwfYWIiIiI2oXLpZnYeH57g/vdVW4tWE1tDOVEREREZNPUeg12ZuzG79f/gKvSBQMD+uH360drtbAopAqMDUsUrUaGciIiIiKySWbBjIPXjmDXpT2oNOkwLHggHu44FHZyO3R0DWlVq69IBEEQRHv0VqigQC12CURERER0ny6XXsXG89uQpb6GcLcwTIl4FP6OvqLWJJVK4OnpVO8+jpQTERERkc24vVVlXvR09PCJg0QiEbu0O2IoJyIiIqI2706tKm0BQzkRERERtWmtsVXFWgzlRERERNQmtdVWlfowlBMRERFRm2IWzPgt5wh2ZlS1qgwNfgijOg5rM60q9WEoJyIiIqI2wxZaVerDUE5ERERErZ5GX44dGbvx+/VkuCpdMDd6OhLaaKtKfRjKiYiIiKjVssVWlfowlBMRERFRq3S59Co2nd+GqzbWqlIfhnIiIiKqV3Lu8VZ1G3JqP2y9VaU+DOVERERUR3LucaxL2wqD2QAAKNaVYF3aVgBgMKdm015aVeojaijX6/X48MMPsWPHDpSVlSEyMhILFy5Ev379GnX+rl27sGrVKly8eBFKpRLh4eF45ZVXEBsb28yVExER2badGXssgbyGwWzAzow9DOXULK6UXcXG9PbRqlIfUUP5okWL8MMPP2DWrFkICQnBtm3bMH/+fKxZswbx8fF3PHfp0qX44osvMHbsWEydOhVarRZpaWkoKChooeqJiIhsV7GuxKrtRPeqplXl0PU/4KJ0ahetKvWRCIIgiPHAp06dwuTJk7F48WLMmTMHAKDT6TBmzBj4+Phg7dq1DZ57/PhxTJ8+HR9//DGGDx/epHUVFKib9HpERERt0V8P/hOl+rJ694U4B6GvfwISfLvDUeHQwpWRrahqVUnGzozdqDTpMDhogM23qkilEnh6OtW7T7SR8j179kChUGDy5MmWbSqVCpMmTcLSpUuRn58PHx+fes9dvXo1YmJiMHz4cJjNZlRUVMDR0bGlSiciIrJpGSVXUGGoqLNdIVWgu1c0rpXnYuP57dh6YRdivKPR1y8BUR7hkEllIlRLbdGtrSpd3DphSvij6ODkJ3ZZohItlKempiI0NLROmI6NjYUgCEhNTW0wlB86dAijR4/GBx98gDVr1kCr1SIgIAB//vOfMXbs2JYon4iIyCYdy0vB6tSN8LBzw4AOfbE/+2Cd1VcEQUC2JgdHrh/DH3kncCL/FFyUzujt1wN9/BLafbiihmn05dh5aTd+z2nfrSr1ES2UFxQUwNe3bvO+t7c3ACA/P7/e80pLS1FSUoJvv/0WMpkML730Etzc3LB27Vq8/PLLsLe3b/KWFiIiIlsnCAJ+unoA2zO+QyfXjlgQOxtOCkcMDXmozrESiQRBzgEIcg7Ao51H4WxhGg5fP4Z9Wb/ip6sHEOwciL7+PdGT7S1UraZVZVfGHlSYKjEk6EGMCrXtVhVriRbKKysroVAo6mxXqVQAqvrL66PVagEAJSUl2LRpE+Li4gAAw4cPx/Dhw/Gf//znvkK5t7fzPZ9LDfs1MxnrT+1AobYIng4eeCx2HB4M6S12WUREBMBkNmHF8Y34MeNX9AtKwHN9ZkMpq/s7uiH+vv0wrGs/lFaW4WDmH/j5ymFsOr8dSRe/Qc8OsRgU2hdxfl3Z3tJOXSy8gi+PbUBGcSaifcIxr8dUBLl2ELusVke0UG5nZweDwVBne00Yrwnnt6vZHhgYaAnkAKBUKjFy5EisXr0a5eXl99xjzomeTe/2tW5vaIvw3+SvUVZWwWW1iIhEVmnUYcXZtThbmIbhwYMwNiwRpUWVACrv4WoS9Pbojd4evZGlzsGR3KP4I/cEDmcfh7PSCb19e6Cvf0+2t7QTdVpVuj6GBN/ukOgl7TZvtcqJnt7e3vW2qNQsadhQP7mbmxuUSiW8vLzq7PPy8oIgCNBoNJz42YpwrVsiotapRFeK/6asRLbmOqZFjMeDAY27T0hjBDl3QJDzWDwaNgpnC9Nx5PpR7M8+iL1ZvyDYOQB9qttbnBT8fW1r6mtVeTh0GOzZqnJHooXyyMhIrFmzps6odkpKimV/faRSKaKiopCXl1dnX25uLmQyGVxdXZunaLonXOuWiKj1ydHkYlnKCpQbtXg6dg66eUU1y+PIpXLEeUcjzjsaar0GR/NO4sj1o9h8fgeSLnyDGK+u6OufgK4eEWxvsQFVq6psx1V1NldVsZJooTwxMRErVqzA5s2bLeuU6/V6JCUloUePHpZJoDk5OaioqEBYWFitc9955x389ttv6N+/PwBAo9Fg9+7diI+Ph50d34m1Ju4qt3oDuIPcHoIgcMY1EVELSyu6gM9Pr4FKpsDCHk8j2DmwRR7XWemEwUEDMDhoALLVOTiSewzJucdxsuA0nBVO6OUXj77+PRHg5N8i9VDTabBVhb/jG020mwcBwAsvvIC9e/di9uzZCA4OxrZt23DmzBmsWrUKCQkJAICZM2ciOTkZ6enplvMqKiowYcIE5OXlYc6cOXBxccHWrVtx+fLlWufei/ba49Scbu8pBwAJJBAgINy9M2ZEToKXvYeIFRIRtR+Hrx/F2rQt8HXwxrNx8+Bh5y5qPSazqWr1ltxjOHMjFSbBhCDnAPT1q25vUbK9pTW7vVVlcOAAtqrcwZ16ykUN5TqdDv/+97+xa9culJaWIiIiAi+++CIeeOAByzH1hXKgqvd8yZIlOHDgACorKxEdHY0XX3wRvXr1uq+aGMqbR3LucezM2HNzrdtOI6E3G7Dt4rcwQ8C4sIfxUEA/SCVSsUslIrJJgiDguys/4bvLPyLCvTPmx8yEvdxe7LJq0ejLcTTvJA7nHkWW+hpkEhm6eUWhr18Coj0j2d7SyrBVxXqtNpS3RgzlLau4sgTr0rbiXFE6wlxD8XjUJPg4eItdFhGRTTGajVifloTDuUfRxy8B0yMnQi4VrYO1Ua5pruPw9arVW9QGDZwUjujt14PtLa3A7a0qEzqPYatKIzGUW4GhvOUJgoDDucew9cJOGM0mPNJpJAYHDeCoORFRE9AaKvDFmTVIL76IUaHDMarjsDYVnkxmE84VpePw9aM4XdPe4tQBffx7opdvPNtbWtDtrSqDAvtjVOhwtqpYgaHcCgzl4inRlWJDehJO30hFqEsIHo+aDD/H+pfGJCKiuyuqLMaylBXI0xZgRuQk9PXvKXZJ90VjKLes3nK1pr3FMxJ9/HuiG9tbmlVmWRY2pG9jq8p9Yii3AkO5uARBwB95J7Dl/E7ozHqMDh2OoUEP8QctEZGVrqqz8WnKSuhNBsyPmYlIjy5il9Skrmmu48j1Y0jOOw61vqq9pZdfPPr69USgM+8W2VTYqtK0GMqtwFDeOpTq1Nh0fhtOFpxBsHMgZkZN4TtyIqJGOnMjFV+eXQtHuQOejZtn0z8/b7a3HMPpG+dgEkwIdOqAvtU3J3JW1h+A6M7Mghm/5yRjJ1tVmhRDuRUYyluX4/mnsDF9GyqMlXi441CMCBnMUXMiojv49dohbEzfjkDnDngmdi5cVS5il9RiNIZyHMtLweHrR3FVnQ2pRIpunlHo61+1ektrn9zaWrBVpfkwlFuBobz1Ues12Hx+B47lpyDQqQMej5qCIH40SURUi1kwY2fGHvx49WdEe0ZiXvQM2MlVYpclmhxNLg7nHkVy7i3tLb7x6OPfk79DGqAxlGNnxh78npMMF6UTxnceg55sVWlSDOVWYChvvVIKzmB9ehLKDVqMDBmMxI5DOepBRATAYDJgTeomHMtPwYCAvpjSZRw/VaxmMpuQWnQeh3OP4XTBWRgFEwKc/NG3evUWtrewVaUlMZRbgaG8dSs3aLHlwk4k5x5HB0c/PB41GSEuQWKXRUQkGo2hHMtPrUJG6RU8GjYKw4IHcmSzAeUGLY7lncTh68eQqc6CVCJFtGck+lav3tIeB3oyy7KwMX07MtVZbFVpAQzlVmAobxvO3EjFurStKNOrMSx4IEaHDodCphC7LCKiFlWgLcSylC9RpCvBrKipSPCNE7ukNiNHk4sjuceQnHscZXo1HBUO6OUbj77+PRHo1MHm39jc2qriXL2qCltVmh9DuRUYytsOraEC2y5+g9+v/wFfBx88HjUZnVxDxC6LiKhFXC7NxH9PfQVBEPBU7Gx0dgsVu6Q2yWQ2Ia34Ag5fP4pTt7S39PFLQC+/eLgoncUusUmxVUVcDOVWYChve1ILz2Nt2haU6EoxOGgAHuk0EkqZUuyyiIiazcn80/jq3Hq4qlzxbNw8+Dp4i12STahqb0nB4dyjyCyraW+JQF+/nujmFdXm21vYqiI+hnIrMJS3TRXGSmzP+A4Hrx2Gt70nHo+awlEjIrI5giBgf9avSLr4LTq6BGFB7BxOVGwm18vzqm5OlHsMpdXtLT1949HXPwFBTgFtqs2DrSqtB0O5FRjK27b0ootYm7YFRZXFeCjwAYwLexgqjpoTkQ0wC2ZsubALB7J/Q3fvGMzuOg1KzqVpdlXtLRdx5PpRpNw4C6PZiA6Ofujjn4Bevj3gqmq97S1mwYxDOX9gR8Zutqq0EgzlVmAob/t0Jj12ZuzGgezf4WHnjsejJiHcvbPYZRER3TOdSY+VZ9fh9I1zGBL0IMZ3Hg2pRCp2We2O1qDFsfwUHL5+DFfKrkIqkaKrR0TV6i1eUVC0ovaWW1tVOruFYmr4eLaqtAIM5VZgKLcdF0suY23qZuRX3MCAgL4YHzYKdhwdIKI2pkyvxn9TvsJVdTYmhY/FoMD+YpdEAHLL83Ek9xiOXD+GUn0ZHOUO6OnXHX39eiLIWbz2Fo2hHLsy9uA3tqq0SgzlVmAoty16kx7fXPoB+7J+hZvKFdMjJ6KrZ4TYZRERNUpueR6WpaxAmV6DedHTEesdLXZJdBuzYEZaUdXqLTXtLf6OvtU3J2q59ha2qrQNDOVWYCi3TZdLM7EmdTPytPno598LEzqPgYPCXuyyiIgadKE4A5+dXg25RIZn4ubyRmltgNZQgWP5KThy/SguW9pbwtHHvydivLo2W3tLZlkWNp7fjswytqq0dgzlVmAot10GkwHfXfkJP109AGeFEx6LnIAYr65il0VEVEdy7nF8nQTwk88AACAASURBVLoZXvaeeDZuHrzsPcQuiayUV56Pw9U3JyrRlcJBbo+evt3R178ngp0Dm6SdhK0qbQ9DuRUYym1fZlkWvk7djJzyXPTy7YHJ4WPhqHAQuywiIgiCgO8z92HXpe/Rxa0TnoqZBQf+fGrTzIIZ6UUXcTj3KFIKzsBgNsLP0Rd9/RLQ268HXFUu93TNQzl/YMel3agwslWlLWEotwJDeftgNBux58o+fJ+5D44KB0yLmIDu3t3ELouI2jGT2YQN6dvw+/Vk9PKNx4yoya1qNQ+6f1pDBY5Xr95yuSwTEkjQ1TMCffwSEOvVFScKTmNnxh4U60rgrnLD2LBE9PbrUesabFVp2xjKrcBQ3r5kqXPwdeomZGtykOATh8nh43gjDiJqcRXGSnx55mukFp1HYsehGBM6gi0INi6vPB9Hco/jSO4xlOhKoZAoYIIJZsFsOUYhVWB65ET09utRp1VlfOfR6OUbz++TNoah3AoM5e2PyWzCj1d/xneXf4K93A5Twh9FD59Y/qAjohZRXFmCT0+txPXyPDwWMQEPdOgtdknUgsyCGenFF7H81CrozYY6+6WQoI9/T5y6cZatKjaAodwKDOXtV44mF1+nbkamOgvdvbthasR4uChb753aiKjty1bn4NNTK1FprMST3WYiyjNc7JJIJM/te+WO+33svfBkzEwEOPm3UEXUHO4Uynk7MKJqHZz88JeEZ/Fo2CicKUzDPw+/j+Tc4+D7ViJqDucK07H0+KcAgIU9nmEgb+fcVW533G8wGxnIbRxDOdEtZFIZhocMwuJef4aPgzdWnduAz05/hRJdqdilEZEN+T0nGZ+eWglPew+8lPAcAp07iF0SiWxsWCIUUkWD+4t1JS1YDYmB7Su3YfsK1TALZvycdRA7L30PuVSGiZ0fQV//nuw1J6J7JggCvrn0PfZk7kOURzie6PY4e4PJIjn3ONac2wgz6kYzd5Ub/tn/ryJURU2JPeVWYCin2+Vrb2Bt2mZcLLmMrh4RmB45Ee52d/6YkYjodgazEV+nbsLRvJN4wL83pkWMh0wqE7ssamWSc49jXdpWGG6Z9HnrKizUtjGUW4GhnOpjFsz45doh7MjYDSkkGN95NPp36MNRcyJqFK1Bi+WnV+NCySU80ikRI0MG8+cHNSg59/hd1yuntomh3AoM5XQnNyqKsDZtC84XX0S4e2fMiJzE218T0R3dqCjCspQVKKwoxONRU9DLL17skohIJAzlVmAop7sRBAG/5RzBtovfwgwB48IexkMB/SCVcN40EdWWWZaFT1NWwiiYsCBmFrq4h4ldEhGJiKHcCgzl1FhFlcVYl7YVqUXn0dktFDMiJ8PHwUvssoiolThVcBYrz66Ds9IJz8bNg5+jr9glEZHIGMqtwFBO1hAEAYdzj2HrhZ0wmk0Y22kkBgUN4Kg5UTv3c9Zv2HJhJ4KcA/BM3FzeiIyIADCUW4WhnO5Fia4UG9KTcPpGKkJdQvB41GT4OfqIXRYRtTCzYMa2i99iX9aviPWKxpzox6CSKcUui4haCYZyKzCU070SBAF/5J3AlvM7oTPrMSZ0BIYEPcglz4jaCb3JgFXn1uNkwRkMCuyPiV0e4admRFQLQ7kVGMrpfpXq1Nh0fhtOFpxBiHMQHo+ajA5OfmKXRUTNSK3X4LNTX+FKWRYmdBmDIUEPil0SEbVCDOVWYCinpiAIAo7nn8Km89tRYazEwx2HYUTIII6aE9mgPG0BlqWsQKmuFHO6PobuPjFil0RErRRDuRUYyqkpqfUabD6/A8fyUxDk1AGPR01BoHMHscsioiaSUXIFn536ChKJBE/HzkGoa4jYJRFRK8ZQbgWGcmoOJwvOYEN6EsoNWowMGYLEjkMgl8rFLouI7sOxvJNYnboJHio3PBv3BLwdPMUuiYhaOYZyKzCUU3MpN2ix5cJOJOceRwdHP8yMmoJgl0CxyyIiKwmCgB+v/owdGbvRybUjFsTOhpPCUeyyiKgNYCi3AkM5NbfTN85hfVoS1AYNhgUPxKiOw6CQKcQui4gawWQ2YdOFHTh47TB6+MRiVtRU/vslokZjKLcCQzm1BK2hAtsufoPfr/8BPwcfPB41mb2oRK1cpVGHFWfX4mxhGoYHD8LYsEQueUhEVmm1oVyv1+PDDz/Ejh07UFZWhsjISCxcuBD9+vW743kff/wxPvnkkzrbvby88Ntvv91XTQzl1JJSC89jbdoWlOhKMSToQYzpNBJKjrq1Ocm5x7EzYw+KdSVwV7lhbFgievv1ELssakIlulL8N2UlsjXXMTViPB4M6Ct2SUTUBt0plIs602zRokX44YcfMGvWLISEhGDbtm2YP38+1qxZg/j4+Lue/9Zbb8HOzs7y9a3/T9QWRHmG47U+L2J7xnfYm/ULTt84hxlRk9HZLVTs0qiRknOPY13aVhjMBgBAsa4E69K2AgCDuY3I0eRiWcoKlBu1eDp2Drp5RYldEhHZINFGyk+dOoXJkydj8eLFmDNnDgBAp9NhzJgx8PHxwdq1axs8t2ak/I8//oCLi0uT1sWRchJLetFFrE3bgqLKYgwMfABjwx7m7blbEZPZhBsVhcjTFtzyXz4ul16FgLo/Rp0Ujni9z1/grKx/RITahrSiC/j89BqoZAo8HTcXwc6cnE1E965VjpTv2bMHCoUCkydPtmxTqVSYNGkSli5divz8fPj4+NzxGoIgQKPRwNHRERKJpLlLJmpWER6d8dfeC7Hz0h78nP0bTt9IxeNRkxDu3lns0tqVcoMWedp85JXXDt8FFYUwC2bLcc5KJ/g6eNcbyAFAYyjHooNvwc/RF13cOqGLWyg6u4XBVeXcUk+F7tPh60exNm0LfB288WzcPHjYuYtdEhHZMNFCeWpqKkJDQ+HoWHsZqdjYWAiCgNTU1LuG8kGDBkGr1cLR0REjR47Eq6++Cjc3t+Ysm6hZ2clVmBI+Dj18YvF16iZ8eGI5BgT0xfiwUbCTsz2rqZjMJhRWFt0M3eX5lv/XGMotx8kkMnjbe8LP0Rdx3t3g6+ANXwcf+Dp4w0FhDwB4/bf/h2JdSZ3HcFY6YUjgg7hQcgnJucfw67VDAABfB290dutUFdTdO8FN5doyT5oaTRAEfHflJ3x3+UdEuHfG/JiZsJfbi10WEdk40UJ5QUEBfH1962z39vYGAOTn5zd4rouLC2bOnIm4uDgoFAocPnwYGzduxLlz57B582Yolff+kb+3N0exSHze3rHoERqJjWd24dv0vUgrPo8FvWYgzq+r2KW1KRp9OXLK8pCjrv6vLA/X1LnI1RTAZDZZjnNROaGDsy96B3VHB2dfBLj4oYOzL3wcPSGTyu74GI/Hj8dnf6yF3qS3bFPKlJjTYzIeDOkNoOpNwOXiLJwrOI9z+RdwvCAFv+UcAQD4OXmjq3cXdPUJR1fvLvBy9GiGV4Iay2gy4rOja3HgymEM7NgXC3rOgFzGG30RUfMT7SdNZWUlFIq6q0yoVCoAVf3lDZk9e3atrxMTE9GlSxe89dZb2L59O6ZMmXLPdbGnnFqThwNGINIpAmtSN+PtAx/jAf9emNBlDEftbmEWzCisKK5qOaluNaka/S6A2qCxHCeVSOFt7wVfB290DYysGvV2rBr1dlQ41L1wJVBUqb3r40c6ROGxiAl1Vl+JdIiq9fPEFZ7o59kP/Tz7wSyYka3OwYWSS7hQcgmHs05g3+XfAQCedh7o4tYJnd07IdytEzztGdJbitZQgc/PrMH54osYHTocD3cchuKiCrHLIiIb0ip7yu3s7GAwGOpsrwnjNeG8sR577DG8++67OHTo0H2FcqLWJtQ1BIt7vYDvrvyEn64ewLmi83gsYkK7WwGiwlhhCdu3hu8C7Q0YhZuj3o4KB/g6+KCbV1R1u0lV+Pay87jrqPe96u3Xw6qVVqQSKYJdAhHsEoihwQ/BLJiRo8m1hPTThedwOPcoAMBd5YYu7lXtLp3dOsHb3pNzaJpBUWUxlqWsQJ62ALOipqKPf4LYJRFROyNaKPf29q63RaWgoAAA7tpPfjupVApfX1+UlpY2SX1ErYlCpsC4sIfR3bsbvk7djE9PrUQfvwRM7PJI/aO8bZRZMKOosqR6omV+rZVOyvQ3R52lEim87D3g6+CNaM/IWr3eTsq2d7tzqUSKQOcOCHTugMFBA2AWzMgtz8f5kgxcLL6Ec4XpSM49DgBwU7mis1to9eTRTvBx8GZIv09X1dn4NGUlDGYDno97EhEenFxNRC1PtFAeGRmJNWvWoLy8vNZkz5SUFMt+axgMBly/fh3dunVr0jqJWpMQlyC82utP2HNlH77P3IfUovOYFjEecd61v+8Pnc1F0oEMFJbp4OmiwoSBYegX7SdS1XVVGitrLy1YHcDzK27AaDZajnOQ28PXwQddPSLg6+htCd9e9h6QS223z1cqkaKDkx86OPlhUGB/CIKAPG1+1Uh68SWcL87A0byTAAAXpbNlFL2Leyf4OfgwpFvhzI1UfHl2LRzlDvifHvPRwan1/DshovZFtN9qiYmJWLFiBTZv3mxZp1yv1yMpKQk9evSwTALNyclBRUUFwsLCLOcWFRXBw6N2n+WXX34JnU6HBx98sMWeA5EY5FI5xnQagTjvbvg6dROWn16NBJ84TAl/FE5KRxw6m4tVu9OgN1Yt31dYpsOq3WkA0KLB3CyYUVxZUm/4LtWXWY6TQFI96u2DKI/w6vBdPeqt4HKnACCRSODn6As/R188GNAPgiAgv+IGLhZfsrS8HMuvGtBwUjhaetK7uHWCv6MvbwXfgF+vHcLG9O0IdO6AZ2LnwlXVtPe9ICKyhmg3DwKAF154AXv37sXs2bMRHByMbdu24cyZM1i1ahUSEqr6+WbOnInk5GSkp6dbzouLi8OoUaMQHh4OpVKJI0eO4Pvvv0dCQgJWr14Nufze32twoie1JSazCT9k/ozdV36CUqpCjGogDv8mgc5grnOsp4sK7z7bv8lrqDTqkK+t3eedpy1AvvaG5S6XAGAvt7OE7VsnWXrZe0Jhw6PeLUEQBNyoKMKFkku4WFI1kl6zTKOjwgGdXUPRxT0Mnd06IcDJr92HdLNgxs6MPfjx6s+I9ozEvOgZsJNbN4+JiOhetMqJngCwZMkS/Pvf/8aOHTtQWlqKiIgILF++3BLIG/LII4/g+PHj2LNnDwwGAwICAvDss89iwYIF9xXIidqCCp0RV/PUyMzTIDNXjcw8J2i1fWHoeBrJTrshhPhCVuoBeYfLkCgrIejtYMwKR2FRh3t+TLNgRomutN6JliW6m/M4JJDA084dvo4+iHDvfLPX29Ebzgonjno3E4lEAm8HT3g7eOKBDr0AAIXVIf1CySVcLL6ElBtnAQD2cnt0duuIzm6dEO4WhkDnDu0qpBtMBqxO3Yjj+acwIKAvpnQZ12wTgImIrCHqSHlrxJFyak00FYaqAJ6rRmb1n3nFN5doc3NSIsTXGSF+zgjycUQWTuHHrB8hSIBb869gkkKZ2x3/njntjo+nM+lvjnqX3zrqXQD9LaPedjK76tFu71qTLL3tPaGQ1V3qlMRXXFli6Um/WHIJ+RU3AFT9XYa5dbTczCjIKcBmQ6rGUI7PTq3CpdIreDRsFIYFD+QbRSJqUXcaKWcovw1DOYmlrFyPK9Xh+2r1nzdKKy37PV3sEOLnjBBfJ4T4uSDE1wmuTnU/cn/p5zdRYS6vs10JFZYO+QcEQbg56l0z4l09+n3rnSklkMDDzr3e8O2idGaYaeNKdKW39KRfRp62ajUslUyJTq41IT0MIc6BNhHSC7SFWJbyJYp0JZgVNRUJvnFil0RE7RBDuRUYyqm5CYKAYrXOMvJ9NU+DzDw1itU3b5jl426PEF9ndPRzRrCfM0J8neFk37gR6Of2vdLgPg+VGzRGba27T6pkylt6vX0sAdzb3gtKjnq3G6U6NS5W96RfKLmE6+V5AAClVIFOrh0tq7uEuAS1uTkAl0sz8d9TX0GAgAUxcxDm1lHskoionWIotwJDOTUlQRBwo7TyZvtJ9Sh4mbaqFUQCwM/TAR2rg3dVG4ozHOzuPfS8/tv/qzXifSu5VI4BHfrUCuCuSheOelMdar0GGSWXcb46qF/TXAcAKKRyhLqEWFZ3CXUJbtUtSyfzT+Orc+vhqnLFc3Hz4OPgLXZJRNSOMZRbgaGc7pVZEJBfXFGr//tqnhrllVXrbsukEnTwcrSE7xBfZwT5OEGlbNrWgOTc41h1bkOD+/8zZEmTPh61D+UGLS6WXK4aSS/OQLbmOgQIkEtk6OgabFkrvZNrCJQypdjlQhAE7M/6FUkXv0VHlyAsiJ0DZ2X9vwiJiFoKQ7kVGMqpMUxmM3ILtcjMU+NKbnUPeL4GOn3V7d7lMgkCvZ0s4TvEzxmB3o5QyFumN/eVX/4X5UZtne3uKjf8s/9fW6QGsm1aQwUySi9bJo9mqa9BgACZRIYQlyDLHUdDXUNafLlBs2DGlgu7cCD7N3T3jsHsrtPYikVErQJDuRUYyul2RpMZOTfKa03CzMrXWG7Oo5RLEeTrVBW+qwN4By9HyGXiLTOXnHscq89shiA1WbYppApMj5yI3n49RKuLbFeFsRKXSq9YVnfJVGfDLJghlUgR7BxoWd2lk2tH2Mvtmq0OnUmPlWfX4fSNcxgS9CDGdx7drpZ8JKLWjaHcCgzl7ZveYEJ2Qbml/SQzT41rBRoYTVX/TOyUMgRXh++aSZj+Hg6QSltfT/bfkpJQ4nQKJrkW7io3jA1LZCCnFlNp1OFyaaZlrfTMsiyYBBMkkCDIOQBdqnvSw1xD4aCwb5LHLNWp8d9TK5GlvoZJ4WMxKLDpb5ZFRHQ/GMqtwFDeflTqjcjK19zSA65Bzo1ymKv/STjayRFcHb5r2lC83e0hbSOTIt/bcAI6gwmvzewpdilE0Jv0uFSaaVnd5UrpVRirQ3qgk3/1xNEwdHYLhaPCwerr55bnYVnKCqj1GsyNno5Y7+hmeBZERPen1d7Rk6ilaCtr7oJ5cxJmbqEWNe9IXRwUCPFzQfcunpY2FE9Xuza9KolMKoXJZBS7DCIAgFKmRKRHF0R6dAEA6E0GXCm7arnj6MFrh7E/6yAkkKCDk1/1HUerJo86KR1rXSs59zg2p++A1lR1Iy07mQomsxl2ChX+3ONphLgEtfjzIyK6XwzlZHPUWr1l7e+aSZj5JTfvgunurEKIrzN6R/laesDdnJRtOoDXRyaVwGzmB2HUOillCoS7hyHcPQwIBQxmIzLLsiw96YdyknEg+zcAgL+jr2V1F42hHFvO74QZZsu1Kk1Va/yPDU5kICeiNouhnNq0Uo3OEr5rliAsLLt5Ex4v16q7YA6I9be0oLg4ir9cW0uQSSUwMZRTG6GQytHZLRSd3UIBDIXRbMRV9TVcKM7AhZJLOJJ7DL9cO3THa+zL+hVDgh9smYKJiJoYQzm1CYIgoKhMV2sCZmaeGqWam3em9PVwQFiAK4YkOKOjrzOCrLgLpi2SyRjKqe2SS+Xo5BqCTq4hGIkhMJlNyNJcw7tHP2nwnIZumkVE1BYwlFOrIwgCCkoqkJmnqXUjHk1F9V0wJUAHT0d0DfGwTMIM8nGCvYrfzreSSiUwmc13P5CoDZBJZejoEgx3lVuD4dtd5dbCVRERNR2mGGoRh87mIulABgrLdPB0UWHCwDD0i/aD2Swgr1hbK3xn5mlQobt5F8wAb0fEd/GytJ8E+jhBpWiZm/C0ZWxfIVs0NiwRa85tqtVTDgAyiQxjwxJFqoqI6P4xlFOzO3Q2F6t2p1lutlNYpsOX36Rix8HLKNXooTPU3AVTiiAfJ/Tp6osQ36q7YQZ4OUEh540/7oVMKmUoJ5tTs9b+rauvOModMCl8LNfhJ6I2jaGcml3SgQxLIK9hFgQUlVViYPcAy414/DwdRL0Lpq2RSSUwmRjKyfb09uvBAE5ENoehnJrdrauh3MpoEjBjeHgLV9N+cElEsmUNtcQREbVVHJakZufporJqOzUNrr5CtqqmJa7mDX9hmQ6rdqfh0NlckSsjIrp3DOXU7CYMDIPytr5wpVyKCQPDRKqofeDqK2Sr6muJ0xvNSDqQIVJFRET3j+0r1OxqPlLmR80tixM9yVY11BLX0HYioraAoZxaRL9oP4bwFiaXSiAIVZNqpRKJ2OUQNRlPF1W9AdzdmS1xRNR2sX2FyEZJpVVBnJM9ydbU1xIHVLXF6fQmESoiIrp/DOVENkomqwrlXBaRbE2/aD/MfjgS8urvcU8XFYYmBCK/pAIfbkmx3PuAiKgtYfsKkY2SVbesVE325B1Qybb0i/bD/hPXoJBJ8fJj8QCATh1c8MWuc/gk6TT+NDEGCjm/74mo7eBIOZGNklXfiImTPclWCWYB0lumS/SL9sOcUZE4e7kI/9l2BkYTVx8ioraDoZzIRsmkNSPlDOVkm8wCIJHWnsT8YGwHzEqMwKmMQny6ncGciNoOhnIiG8WJnmTrGlpZaFD3AMwYHo4TF27g813nuF4/EbUJ7CknslE1I+VGhnKyUVXtK/Uv9zk0IRBGkxkb912ETCbBk6O7Wt6oEhG1RgzlRDbq5uorHCUk22QWBNxpCf6RvYNhNJmx9cAlyKVSzBkVyTX7iajVYignslEyaVV3GttXyFYJAu46+j26X0cYTQJ2HLwMuUyCmSMjIGEwJ6JWiKGcyEZxoifZuqqR8rsH7LH9O8JoMuPbQ5mQyaSYPqwLgzkRtToM5UQ2SspQTjbOfNuSiA2RSCSY8FAnGIxm/PBHFuQyCaYM7sxgTkStCkM5kY2SM5STjTMLQqMnb0okEkwd0hkmk4Dvk7Mgl0kx4aFODOZE1GowlBPZKEv7Cid6ko0SBFg1cVMikWD68C4wmqtaWeQyKcYNCG3GComIGo+hnMhGcZ1ysnV3W32lPhJJ1WRPo8lsmfw5ul/HZqmPiMgaDOVENkomq1p9he0rZKvMd1in/E6kEgnmPhwFk1moWi5RJsXI3sHNUCERUeMxlBPZKK6+QrbO3IglERsilUrwxOgoGE0CNu67CLlMiqEJgU1cIRFR4zGUE9mos5cLAQAfbjkFTxcVJgwMQ79oP5GrImo6gnBvI+U1ZFIpnnqkK0wmM9b+eB4ymQSDugc0YYVERI0nFbsAImp6h87mYtdvmZavC8t0WLU7DYfO5opYFVHTMput7ym/nVwmxdPjuiE2zBOr96Tj4KnrTVMcEZGVGMqJbFDSgQwYblt1RW80Y/1PF3DuShGy8zUoLdfDZObKLNR2ma1cfaUhCrkUz43vhuhQD6z8LpVvXolIFKK2r+j1enz44YfYsWMHysrKEBkZiYULF6Jfv35WXWf+/Pn45ZdfMGvWLLz22mvNVC1R21FYpqt3u6bCgPc2nLR8LQHgaK+As4MCzg5KuDgo4OyohLO9Ai6OSrg4KG/uc1TCwU7eJCGIqCkIVqxTfjcKuQzPT4jBh5tT8MU35yCXSdEr0qdJrk1E1BiihvJFixbhhx9+wKxZsxASEoJt27Zh/vz5WLNmDeLj4xt1jZ9//hlHjx5t5kqJ2hZPF1W9wdzVSYmnx0ajTGuAWqtHWbke6pr/1xpw7UY51FdLoKkw1HtdqUQCJwdFVXivDuyW4H5LiK/6Uwl7lYw3Z6Fmcy9LIt6JSiHDC5Pi8MGmk1i+8yxkUgl6hHs33QMQEd2BaKH81KlT+Pbbb7F48WLMmTMHAPDoo49izJgxeO+997B27dq7XkOv1+Nf//oXnnjiCXz88cfNXDFR2zFhYBhW7U6D3nizPUUpl2LK4M6ICHa/6/kmsxkarQFqrQFlWj3KtLeE9/KqP9VaA67kqqHWGlChM9Z7HZlUApfqkfeq0H57mFfC2fFmsFcpGOKp8czmpmlfuZVKKcOfJ8fh/Y0n8en2M3h+QgziOns16WMQEdVHtFC+Z88eKBQKTJ482bJNpVJh0qRJWLp0KfLz8+Hjc+ePDlevXo3KykqGcqLb1KyyknQgA4VlOqtXX5FJpXB1UsHVSdWo4w1GsyWoq28J8WVaPdTlN0fi84q0UGsN0BlM9V5HKZfeDO2Ot7TO1GqjuRniFXJZ414QsklN2b5yK3uVHC9OicO7G07iP9vO4E+TYtAt1LPJH4eI6FaihfLU1FSEhobC0dGx1vbY2FgIgoDU1NQ7hvKCggIsW7YMf/vb32Bvb9/c5RK1Of2i/VpsCUSFXAoPFzt4uNg16nid3lQV4isMKCuvCvGamlH5cgPUFXqUavTIytdArdXDaKp/rXU7paxOYHeuHoGvNSrvqISTvQJyGee225Kq9pXm+WTFwU6Bv0ztjnfXn8DHW0/jz5PjEBVy90+ZiIjulWihvKCgAL6+vnW2e3tX9e/l5+ff8fwPPvgAoaGhGDduXLPUR0TNR6WUQaW0h5fb3d9QC4KASr3pZgtN+W0j8dWj8zdKK3H5ehnUWgPMQv0h3tFOXrt95pZJrbdPdnWyUzR6FPbQ2dx7/lSC7o0gCBAEoBkGyi2c7BX4y7TueHfdCXy4JQUvTumO8CC35ntAImrXRAvllZWVUCgUdbarVFUfl+t09a8eAVT1o2/fvh1r1qxp8lESb2/nJr0eEbUss1lAeaUBJWodysr1KNHoUKrRoVSjR6lGhxKNDmUaPfJLK3HhWinUWj3qy/ASCeDiqKxq43FUwdVJCTcnFVycVHBzUlb/qUJ6ZhHWfp8GvaGqf7+wTIfVe9Lh4myHQQlBLfzs24+aO9U6Odk1689tbwD/en4AFv/nN3y4JQVvLXgAkSEezfZ4RNR+NUkoNxqN2Lt3L0pLSzF48GDLaPed2NnZwWCou8JDTRivCee3EwQBb7/9NkaMGIGePXveX+H1KChQN/k1iajl2UkBO2clfJyVABoObWazAE2F4bbJrLVXpVFr9cgv1kJdroe2gUmtt9IZTPjqm7OIDuaoanMxs0neMQAAIABJREFUVq/DX1Ghb5Gf2y9OicM7647jb58dwkvTuiPU36XZH5OIbI9UKoGnp1O9+6wO5UuWLMGRI0ewdetWAFUhee7cuTh69CgEQYCbmxs2bdqE4ODgO17H29u73haVgoICAGiwn/zHH3/EqVOnsHDhQmRnZ9fap9FokJ2dDS8vL9jZNa63lYjaN2n1CjEujspGHW80mS2BXa014P2NJ+s9rqG14qlpCNUfbzRn+8qt3J1VeOWxePzf2uP4YONJvPxYPIJ9+ckqETUdq2c9/frrr7VGqPft24c//vgDTzzxBN5//30AwPLly+96ncjISFy+fBnl5eW1tqekpFj21ycnJwdmsxmzZ8/G0KFDLf8BQFJSEoYOHYrk5GRrnxYRUaPIZVK4O6sQ7OuM6FAPeLrU/6leQ9upadTcjLYlb2bl4WKHlx+Lh0opw3sbTiK7QNNij01Ets/qkfLc3FyEhIRYvt6/fz8CAwPx0ksvAQAuXLiAXbt23fU6iYmJWLFiBTZv3mxZp1yv1yMpKQk9evSwTALNyclBRUUFwsLCAABDhgxBYGBgnes999xzGDx4MCZNmoTo6GhrnxYR0T2pb014CYBxA0LFK6odqJnM29Lr2nu72ePlx+LxztrjeG/DSbw6PR7+no53P5GI6C6sDuUGgwFy+c3Tjhw5ggceeMDydVBQkKUF5U7i4uKQmJiI9957DwUFBQgODsa2bduQk5ODf/3rX5bjXn31VSQnJyM9PR0AEBwc3GBrTFBQEIYNG2btUyIiume3rwnvbK+AusKA9Ksl6B/jz5shNZOaUN4c65Tfja+7Q1UwX3cCS9afwKIZPeDr7tDidRCRbbG6fcXPzw8nTpwAUDUqnpWVhV69eln2FxYWwsGhcT+clixZgpkzZ2LHjh345z//CaPRiOXLlyMhIcHasoiIRNMv2g/vPtsfKxYNwYcvPIix/TvitzO52Hss++4n0z2pWTFHhEwOAPD3dMTL07rDZBLw7voTKCipEKcQIrIZVo+Ujx49GsuWLUNRUREuXLgAJycnDBw40LI/NTX1rpM8a6hUKrz66qt49dVXGzxmzZo1jbpWzUg6EZHYxg4IxdU8DTbsvYhAbydE8qYzTc5sFqd95VYB3k54aVrVDYbeXX8Cr07vAU9XLjJARPfG6pHyBQsWYPz48Th58iQkEgneeecduLhULQ2lVquxb98+9OvXr8kLJSJqK6QSCeY/8v/bu/OwqMu9f+Dv2Yd9k00EF0hQdtdMc18Qd8W9ckkrt1yy7bFzns7vZJ3UjhaalvqczMwdxX23MvVkKrIoaOCKiCCyb7P+/iAmEURQ4MvMvF/X1WVzz/c788EI3tx87vtuC1dHC3y9OwEPcjmLWteEbF95lJerDd4ZF4LCEg2Wbo5Bdj533SGiZyPS659w9N0z0Ol0KCwshFKprPJgIGPAfcqJqK7cyyrEJ9+fh7O9BT58pT0UMonQJZmM7PxSvLPqNCaF+aJHiIfQ5SDlbi6Wbb0EB2sF3p8QCjtr7r5DRJVVt095rWfKq6PRaGBjY2O0gZyIqC65O1nhjSH+uHO/ABsOJaEO50DMXmNoX3mUt4cd5o8OxsP8Eizbcgl5RSqhSyIiI1PrUP7zzz8jMjKywtimTZvQrl07hISE4J133qnypE4iInMU7NMEw7u3wn8v38eR3+8IXY7JMLSvNJJQDgCtPe0xNyIYGTnF+GLLJRQU83shEdVcrUP5+vXrcf36dcPjlJQUfPrpp3BxccFLL72EAwcOYNOmTXVaJBGRMRvcpTna+zpj28lkXL75UOhyTMJfPeUCF/KYNs0dMGdUIO5lFeGLrZdQVMJgTkQ1U+svZ9evX0dAQIDh8YEDB6BQKLBjxw6sW7cO4eHh2L17d50WSURkzEQiEV4f1AZNnaywZncCt8+rA39tidh4ZsrLBbR0wuyRAUjNKMC/t8WiuFQjdElEZARqHcpzc3Ph4PDX9l5nzpzBiy++CGvrsqb1Tp06ITWVe/MSET1KKZdi9qhA6PVA5M54lKq0Qpdk1BpbT/njgrybYObwANxKz8eK7bEoUTGYE1H1ah3KHRwckJaWBgAoKChAfHw8OnToYHheo9FAq+U3GyKix7k6WOKtYf64+6AA/3cgkQs/n0Nj2RKxOqGtnfHGUH8k383FVzviUKrm90YierJah/KQkBBs2bIFhw4dwqeffgqtVovu3bsbnr916xZcXFzqtEgiIlMR0MoJET288XtSBg7+dlvocoxW+Ux5I87kAICOfi6YPrgtrt7OwcqdcVBrGMyJqGq1DuVvv/02dDod5s2bh6ioKAwfPhw+Pj4AAL1ej2PHjqFdu3Z1XigRkakI6+yFTm1csPOnFMRfzxK6HKPUmHvKH/eivxumhLfB5ZvZWLUrAWqNTuiSiKgRktb2Bh8fHxw4cAAXL16EjY0NOnbsaHguLy8PkyZNQufOneu0SCIiUyISiTBlYBukPSjCN9GX8bfJHeDqYCl0WUalvH2lsfaUP65bkDs0Oh2+P3QVa6ITMGN4AKSSRrZ1DBEJ6pm+Itjb26N3794VAjkA2NnZYdKkSfDz86uT4oiITJVCLsGcUYEQiYCVO+O5Q0ctNdYtEavTM8QDE/u1RswfD/Dt3ivQ6jhjTkR/qfVMebnbt2/j+PHjuHOn7DAMT09P9OnTB15eXnVWHBGRKXO2t8CM4QH4YuslrN+fiJkjAoyiHaMx0P+ZZ43t76tP+2bQaHXYeiIZUrEI0wa3bdSLVYmo4TxTKF+xYgXWrl1baZeVpUuX4s0338TcuXPrpDgiIlPXtoUjxvbywZYTydh/5iaGdG0pdElGwdC+YoSBdkAnL2i0Ouz8+TokEhGmhLcxuh8uiKju1TqU79ixA2vWrEFoaCimTZuGF154AQDwxx9/YP369VizZg08PT0xcuTIOi+WiMgU9evoiZv387H71A14utogxKeJ0CU1euXbSRpR90oFg7q0gEarR/SvNyCViPHaAF+j6Y8novpR61D+448/Ijg4GBs3boRU+tftXl5e6NGjByZOnIgffviBoZyIqIZEIhEmh/nh3oMirN17GR+91gHuTlZCl9WoGbZENMKZ8nJDu7aARqvD/rO3IBWLMaHfCwzmRGas1pMMKSkpCA8PrxDIy0mlUoSHhyMlJaVOiiMiMhdymQSzRwZCKhEjcmc8ikq48LM65UskjTnEikQijOzeCgM6eeL4xVRsPZHMA6WIzFitQ7lMJkNRUdETny8sLIRMJnuuooiIzJGTnRIzhwcgI7sY6/ZdMfRNU2V6E5gpB8qC+ZhePujTvhmO/H4HO3++zmBOZKZqHcoDAwOxdetWPHjwoNJzWVlZ2LZtG4KDg+ukOCIic+Pr5YBxfXxwKfkB9vx6Q+hyGq2/9ikXuJA6IBKJMKHvC+gZ0hQH/nsL0fzvTmSWat1TPnPmTEyePBnh4eEYNWqU4TTP5ORkREVFobCwEMuWLavzQomIzEWf9s1w634+9py+CU8XG7T3dRa6pEZHZ6RbIj6JSCTCKwN8odHqsef0TUglYgx+qYXQZRFRA6p1KO/YsSMiIyPxz3/+E//5z38qPNe0aVN8/vnn6NChQ50VSERkbkQiEV4b4Iu0B4VYt/8K3Jw6wKMJF34+yrD7iomEcqDsY5k80A9anQ5Rv1yHVCJGWGee/UFkLp5pn/LevXujZ8+eSEhIQGpqKoCyw4P8/f2xbds2hIeH48CBA3VaKBGROZFJJZg1IhD/b8N5RO6Mw98ndYClkut1yplS+8qjxGIRpg5qA7VWj20nkyGViNC3g6fQZRFRA3jmEz3FYjGCgoIQFBRUYTw7Oxs3brAfjojoeTnaKjFrRACW/BiDb/ZcwdyIIKNf2FhX/lznaZJ/HxKxGG8MaQutVocfj/0BqUSMnqEeQpdFRPXMWM9dICIyCy80s8eEfq0Rfz0Lu05dF7qcRsOwT7mpTZX/SSoRY8bwAAR5O+H7w1dxKi5N6JKIqJ4xlBMRNXI9Q5qie3BT7D97C78nZQhdTqNg6Ck3wZnyclKJGLNGBMC/pSO+O5CEswnpQpdERPWIoZyIqJETiUSY2K81vD1ssX7/FdzJKBC6JMGZak/542RSCeaMDISvlz3W7b+Cc4n3hS6JiOoJQzkRkRGQScWYNSIQFgopInfGoaBYLXRJgjK1LRGrI5dJMDciGC942OHbPVdw4Wqm0CURUT2o0ULPx7c+rM7FixefuRgiInoye2sFZo8IxOc/XsQ30QmYNyYYErF5zq3oTHBLxOoo5BLMHR2Mf2+9hDXRCZg1MhAhPk2ELouI6lCNQvnnn39eqxcVmckXSSKihubtYYdX+vviu4NJ2PnTdYzp7SN0SYIwh57yx1kopJg/JhjLtlzC17vi8faoIAS0chK6LCKqIzUK5d9//31910FERDXUPbgpbt3Px6Fzt+HlZo0X27oJXVKDK98S0dzmgCyVMiwYG4Jlm2MQGRWPeRFBaNPCUeiyiKgO1CiUd+rUqb7rICKiWhjf5wXczSjAdweS4O5oheZuNkKX1KBMfUvE6lhbyPDOuBAs2RyDL3fGYcGYELT2tBe6LCJ6TubZjEhEZOSkEjFmjAiElYUMK6PikFekErqkBmWO7SuPsrGUY+G4UDjZKrF8eyyS7+YKXRIRPSeGciIiI2VnJcfskYHILVRjze4EaMu3JDEDhhM9zTOTAyj7779wXCjsrORYvu0SbtzLE7okInoODOVEREaspbstJoX5Iul2DradSBG6nAZT3r5i7hsLONgo8N74UFgpZfhiyyXcvp8vdElE9IwYyomIjFzXQHf07dAMR8/fwen4e0KX0yD0ZrYlYnUcbZV4b3wolAoJlm25hNRMHi5FZIwYyomITMCYXj7w87LHhkNXzaKNwbBPOb+LAQCa2Fvg3fGhkEpEWLY5BveyCoUuiYhqiV/OiIhMgFQixlvDA2BnJcPKqHjkFpr2ws+/tkTkTHk5VwdLvDs+FBCJsGRzDO4/LBK6JCKqBYZyIiITYWspx+yRQSgsVmP1rnhotKa78FNvxlsiVsfdyQrvjguBVqvHks0xyMwpFrokIqohhnIiIhPS3M0Gkwf64VpqLrYc/0PocupNefsKM3llHs7WWDguBCq1Fkt+jEFWbonQJRFRDTCUExGZmBf93RDWyQsnLt7FL7FpQpdTL3T6skDO9pWqebna4J1xISgq1WDp5hhk55cKXRIRPYWgoVylUmHp0qXo1q0bgoKCMGbMGJw9e/ap9+3ZswevvfYaunbtioCAAPTu3Rsffvgh7t692wBVExE1fqN6toJ/Cwf8cOQqUkzwYBm9Xs/Wlado4WaLBWODkVekwpLNMcgtYDAnaswEDeUffPABNmzYgKFDh2LRokUQi8WYPn06YmJiqr0vKSkJrq6umDp1Kj7++GMMHz4cp06dQkREBDIzMxuoeiKixksiFuPNYQGwt1Zg5a545JhYINPp9JwlrwHvpnaYNzoYOfmlWLrlktmd/EpkTET68s1eG1hcXBxGjx6NDz/8EJMnTwYAlJaWYvDgwXBxccGmTZtq9XqXL1/GyJEj8d577+H1119/5royM3nwAhGZjjsZBVi88Tw8Xazx3vh2kElNo2tx64k/cDLmLta801PoUoxC0q1srNgeCxcHS7w3IRTWFjKhSyIyS2KxCE5O1lU/18C1GBw6dAgymQyjR482jCkUCkRERODChQvIyMio1es1bdoUAJCXZ/r78xIR1ZSnizVeH9QWKXfz8OOxa0KXU2f0eu68Uht+zR0wZ1QQ0h8W4Ystl1BUoha6JCJ6jGChPDExES1btoSVlVWF8aCgIOj1eiQmJj71NXJycpCVlYX4+Hh8+OGHAIAuXbrUS71ERMaqo58LBnVpjp8vpeGnGNNYe8P2ldrzb+mI2SMDkJpZgC+2xqK4VCN0SUT0CKlQb5yZmQlXV9dK487OzgBQo5nyAQMGICcnBwBgb2+Pv//973jxxRfrtlAiIhMw4uVWuH2/AJuOXoOHsxVeaGYvdEnPRafXQ8xMXmtB3k0wc3gAvt6dgOXbY7FgTDCUcsGiABE9QrD/E0tKSiCTVe5pUygUAMr6y59m5cqVKCoqwo0bN7Bnzx4UFj7/scLOzjbP/RpERI3R/0zphAVf/oLV0ZexYn4PONlZCF3SM1MoZZBKxfya/Qz6O9vAylqJJT+cx+roK/j7tM4M5kSNgGD/FyqVSqjVlXvaysN4eTivTseOHQEAPXr0QJ8+fTBkyBBYWlrilVdeeea6uNCTiEzZzGH++GTjBfxj7X/xwcRQyKQSoUt6JkVFKkDPr9nPqnVTG0wb1AZr917Bx9+cwdsRQUb7uUBkTBrlQk9nZ+cqW1TKtzR0cXGp1et5enrC398fe/furZP6iIhMkYezNaYNaosb9/Lw/eGrEGgDrudW1lMudBXG7UV/N0wd1AZXbmZj1a4EqDU6oUsiMmuChXI/Pz/cuHGjUstJbGys4fnaKikpQX4+Z02IiKrT3tcZQ7u2wOn4dJy4aJwLP3V6PcRsKn9uXQPd8VqYL+JSsrAmOgEaLYM5kVAEC+VhYWFQq9XYvn27YUylUiEqKgrt2rUzLAJNS0tDSkpKhXsfPnxY6fUSEhKQlJQEf3//+i2ciMgEDO3WEiE+TbD52B+4ejtb6HJqjVsi1p0eIR6Y2K81Yv54gG/3XIZWx2BOJATBesqDg4MRFhaGZcuWITMzE15eXti1axfS0tLw2WefGa57//33ce7cOVy9etUw1qtXLwwcOBCtW7eGpaUlkpOTsXPnTlhZWWHmzJlCfDhEREZFLBJh2uC2+OT78/h6dwL+PqkjnOyUQpdVY2W7rzCU15U+7ZtBq9Vhy4lkSPYlYvrgtvxNBFEDE3S59ZIlS7BixQpER0cjNzcXvr6++Pbbb9G+fftq75swYQLOnj2LY8eOoaSkBM7OzggLC8PMmTPh6enZQNUTERk3S6UUc0YF4pPvz2NlVDw+fKUd5DLjWOzHnvK617+TFzQ6PXb8lAKpWIQpg9rwBx+iBiTSG+sqn3rClfxEZG4uJT9A5I44vOjvimmD2xrFoTxf707A3cwCLJ7Osynq2p5fb2D3rzfQPbgpXgvzZTAnqkPV7b7CjUmJiMxciE8TDHu5JXafuoHmrjbo38lL6JKeSs/2lXozpGsLqLU67D97Cw9yinA/uxhZeaVwslVgZA9vdPF3E7pEIpPEUE5ERBj8Ugvcvl+AbSdT0MzFGm1bOApdUrXYvlJ/RCIRRnZvhTv38xF3/a+NFbLySrHhYBIAMJgT1QPBdl8hIqLGQywS4fVBbeDmZIk10ZeRmVMsdEnV4u4r9UskEuHug8qnZKs0OkT9nFLFHUT0vBjKiYgIAGChKFv4qdPpsTIqHqUqrdAlPZFOr4eIu4PUq6y80lqNE9HzYSgnIiIDVwdLvDnMH6kZBfjPwcRGe+Int0Ssf062ilqNE9HzYSgnIqIKAls5YWSPVjiXmIFDv90Wupwq6XV6cKK8fo3s4Q25tHJM8GvuIEA1RKaPoZyIiCoJf7E5Ovi5YMfPKUi4niV0OZXo9GD7Sj3r4u+GSQP9DDPjjrYKeLpY4Ux8Os4k3BO4OiLTw91XiIioEpFIhNfD2yA9qxBroi/jb5M7wNXBUuiyDLglYsPo4u9WYacVtUaLFdvjsH5/IuRSCTr4uQhYHZFp4Uw5ERFVSSGXYPaoIIhEwMqd8ShRaYQuyUDH9hVByKQSzBkVCO+mdvhmz2XEpTwQuiQik8FQTkRET+Rib4G3hgcgLasQ6/c3noWfOj2M4uRRU6SUSzFvdBCaOVtj1a4EJN7KFrokIpPAUE5ERNXyb+GI0T19cOFqJvafvSV0OQD+3H2FU+WCsVTKsGBsMFzsLfDVjjgk380VuiQio8dQTkRETzWgkydebOuKXb9cR2yy8C0L7CkXno2lHO+MC4GdtRzLt8XiVnq+0CURGTWGciIieiqRSIRJA/3g6WqNb/dexr2syqc9NiSdDuwpbwTsrRV4d1woLBUSfLH1UpWngBJRzTCUExFRjShkEsweGQiJWIyVUfEoLhVu4adOr2dPeSPhZKfEwvGhkIhFWLYlBvezi4QuicgoMZQTEVGNNbGzwMzhAbj/sBhr916BTqCFn+wpb1xcHSyxcFwItFo9lm2OQVZuidAlERkdhnIiIqoVv+YOGNvHB5eSH2Dv6ZuC1KDXs32lsfFwtsY7Y0NQVKrFsi0xyC0oFbokIqPCUE5ERLXWt30zdA1wQ/SvNxBzLbPB31+nY/tKY9TczQbzRwcjp0CFZVsvoaBYLXRJREaDoZyIiGpNJBLhtTBftHS3wbf7rjT4Aj+2rzRePs3s8PaoQNx/WIwvtl5CUUnjOXSKqDFjKCciomcik0owa0QgFFIxVu6MQ1FJw82Klm2J2GBvR7XUpoUjZo0IQGpGAVbsiEWpSit0SUSNHkM5ERE9M0dbJWaOCMSD3BJ8u/cKdLqGWfhZtiUiU3ljFuzTBG8O9UfK3VxERsVBrWEwJ6oOQzkRET2X1p72mND3BcSlZGH3r9cb5D25JaJx6ODngqnhbXDlZja+3pUAjVYndElEjRZDORERPbeeoR7oHuyOfWdu4XxSRr2/X1lPeb2/DdWBroHueLV/a8SmZJVto9lAv00hMjZSoQsgIiLjJxKJMLGfL+5mFmL9/kS4OVmimbN1vb1f2ZaInCk3Fr3aNUOpWodtJ5Mhl4kxJbwN//sRPYbzDEREVCdkUjFmjgiEUi5B5M64et0OT6fTQ8SVnkYlrLMXhnVridPx6fjx6DXoBTp4iqixYignIqI642CjwKyRgXiYV4pv9lyut1YFvV4PMRjKjc3Qri0Q1skLJy7exY6fUhjMiR7BUE5ERHXKx8MOrw7wxeUbD7Hz55R6eQ+dXg8Rv4MZHZFIhNG9vNEr1AMHf7uNvWduCl0SUaPBnnIiIqpz3YOb4lZ6Pg7+dhterjbo3Na1Tl9fx55yoyUSiTCxf2uUqrXYfeoGFDIJBnTyErosIsExlBMRUb0Y3/cFpGYW4D8HEuHuZAkvV5s6e229jid6GjOxSIQp4X5QaXTYeiIZCpkEPUM9hC6LSFD85R8REdULqUSMmcMDYGUhw8qoeOQXqerstcv2Ka+zlyMBSMRivDGkLYK8nbDx8FWcSbgndElEgmIoJyKiemNnrcDskYHIKVBhTfRlaHV1c3gMT/Q0DeU/uPk1d8D6/YkNssc9UWPFUE5ERPWqpbstXhvgi8Rb2dh+sm4Wfur1eoZyEyGXSTBnVCC8m9rhmz2XEZfyQOiSiATBUE5ERPWuW5A7+rZvhiO/33nuNgW9Xg89wPYVE6KUSzFvdBCaOVtj1a4EJN7KFrokogbHUE5ERA1iTG8f+HraY8Ohq7iZnvfMr6P7c29rLvQ0LZZKGRaMDYaLvQW+2hGH5Lu5QpdE1KAYyomIqEFIJWLMGBEAW8uyhZ95hc+28LP8vBm2r5geG0s53hkXAjtrOZZvi8Wt9HyhSyJqMAzlRETUYGwt5Zg9Mgj5RWp8vTsBGm3tF36WnxLKmXLTZG+twLvjQmGpkOCLrZdw90Gh0CURNQiGciIialDN3WwweaAfrt3JwdbjybW+v7x9hRPlpsvJTomF40MhEYuwbEsMMrKLhC6JqN4xlBMRUYPr4u+G/h09cfxiKk7FpdXq3vJdFdm+YtpcHSyxcFwItFo9lm6+hKzcEqFLIqpXDOVERCSI0b280aa5AzYevoqUtJov6ivbe4Wh3Bx4OFvjnbEhKCpVY9mWGOQWlApdElG9YSgnIiJBSMRizBgeAHtrBVZFxdc4cLGn3Lw0d7PB/NEhyClQYdnWSygoVgtdElG9YCgnIiLBWFvIMHtkIIpKNVi1q2YLP//M5OwpNyM+zezw9qhA3H9YjC+2XkJRiUbokojqHEM5EREJysvVBlPD2yD5bi5+PHrtqdcbZsqZys1KmxaOmDUiAKkZBVixIxalKq3QJRHVKUFDuUqlwtKlS9GtWzcEBQVhzJgxOHv27FPvO3LkCObNm4fevXsjODgYYWFh+Pzzz5Gfz/1MiYiMUac2rhj4ohd+upSGny7drfZaPQ8PMlvBPk3wxlB/pNzNRWRUHNQaBnMyHYKG8g8++AAbNmzA0KFDsWjRIojFYkyfPh0xMTHV3ve3v/0NKSkpGDZsGD766CN069YNGzduxPjx41FaykUgRETGaFR3bwS0dMSmI9eQnPrkhZ/cEtG8dfRzwdTwNrhyMxurd19+pr3uiRojkb58yqGBxcXFYfTo0fjwww8xefJkAEBpaSkGDx4MFxcXbNq06Yn3/vbbb+jcuXOFsd27d+P999/HZ599hpEjRz5zXZmZnG0nIhJKYYka//zuPErVWvx9ckc42CgqXZORU4wP1pzF64PaoGuguwBVUmNw8mIqNh65ho5+LnhzqD9/c0JGQSwWwcnJuurnGrgWg0OHDkEmk2H06NGGMYVCgYiICFy4cAEZGRlPvPfxQA4Affv2BQCkpKTUfbFERNQgrJQyzBkViBKVFiuj4qtsT9Czp5wA9GrXDGN6+eD3pAz852Ci4TcoRMZKsFCemJiIli1bwsrKqsJ4UFAQ9Ho9EhMTa/V6Dx48AAA4ODjUWY1ERNTwPJytMW1wG9y4l4eNR67h8V/oGtpXuFWB2Qvr7IVh3VridHw6fjxa+XOFyJhIhXrjzMxMuLq6Vhp3dnYGgGpnyquydu1aSCQS9O/fv07qIyIi4bT3dcHgl1pg35mbaOFmg97tmhmeK98SkTPlBABDu7ZAqUqLQ+duQyGTIKKnN0T83CAjJFgoLykpgUwmqzSuUJT1D9ZmwebevXuxY8cOvPnmm/Dy8nquupydbZ7rfiIiqhvTRwThfk4xNh/7A/4+zgjwbgIAKNSUpXJ7O0t+zSYAwMwxIRBJxTht7lxDAAAgAElEQVR45iYcHSwxrp+v0CUR1ZpgoVypVEKtrnwqV3kYLw/nT3P+/HksWrQIPXv2xNy5c5+7Li70JCJqPCb198Wd9Hx8+t05/O/kjnC0VSIrqwAAkJ9fwq/ZZDDq5ZbIzSvBpkNJ0Kg0GNDp+SbpiOpDo1zo6ezsXGWLSmZmJgDAxcXlqa+RlJSEGTNmwNfXF8uXL4dEIqnzOomISDiWSinmjAqEWqNDZFQ8VGotytuGxewpp0eIRSJMCfdDB19nbD2R/NT97okaG8G+pPn5+eHGjRsoLCysMB4bG2t4vjq3b9/GtGnT4OjoiG+++QaWlpb1VisREQnH3ckK04e0xa30fHx/+Ooj+5Szb5gqkojFeGOoP4K8nbDx0FWcTUgXuiSiGhMslIeFhUGtVmP79u2GMZVKhaioKLRr186wCDQtLa3SNoeZmZmYOnUqRCIR1q9fD0dHxwatnYiIGlboC84Y3q0lziSU7bIBAF/tiMO7X5/G2csMXvQXqUSMmcMD4NfcAev2X8H5pNptHEEkFMEODwKAuXPn4vjx45g0aRK8vLywa9cuJCQkYMOGDWjfvj0A4NVXX8W5c+dw9epVw33Dhg1DUlISpk2bhtatW1d4TS8vL4SGhj5zTexPJCJqnHR6Pf753e+4db+gwrhcKsakgX7o4u8mUGXUGJWoNPj31ljcuJeHOaOCEOTtJHRJRNX2lAu20BMAlixZghUrViA6Ohq5ubnw9fXFt99+awjkT5KUlAQAWLduXaXnRowY8Vyh/EnUahXy83Og0aig01U+zIIIACQSKayt7WFhYfX0i4moVsQiEfKKKm8QoNLoEPVzCkM5VaCUSzFvdBCWbr6EVbviMX90MPya8ywTarwEnSlvjKqaKS8uLkR+fjasre2gUFhALJawl5Eq0ev1UKtVyMnJhI2NA4M5UT2Y+q8TT3zu/z7o3YCVkLHIL1Lh8x9jkJVbgnfGhcDHw07oksiMNcrdV4xJQUEu7O2bwNLSBhKJlIGcqiQSiSCXK2Bv74yCghyhyyEySU62VW+X+6RxIhtLORaOC4GdtRzLt8XiVjrbVKlxYiivAa1WDZmMX/CpZmQyObRajdBlEJmkkT28IZdW/NYll4oxsoe3QBWRMbC3VuDdcaGwVEjwxdZLuPug8Ok3ETUwhvIa4uw41RQ/V4jqTxd/N0wa6GeYGXeyVXCRJ9WIk50SC8eFQiIWYdmWGGRkFwldElEF7Cl/TFU95enpt+Dm1lyAashY8XOGiKhxuptZgM9/jIFCJsGHr7SDo61S6JLIjLCnnAQze/YbmD37jQa/l4iIqCoeztZ4Z2wIikrVWLo5BrkFpUKXRARA4C0RSTjdunWo0XXbt++Bu3vTeq6GiIio4TR3s8H80SFYtjUGy7ZewvsT2sHaQiZ0WWTm2L7yGHNpXzl8+ECFx9u2bcb9+/cwZ86CCuPdu/eChYXFM7+PWl22p7BMVvsvds9zr9BM8XOGiMjUXLn5ECu2x8HD2apsIaiSc5VUvxrt4UEknAEDwis8/umn48jNzak0/riSkhIolTXvv3ueQG2MYZyIiIxH2xaOmDUiACuj4rFiRyzeGRMChVwidFlkpthTTk80e/YbmDx5Aq5cScCMGa+jd++u2LRpAwDg1Kmf8O67czFsWBh69eqCMWOG4bvv1kGr1VZ6jUf7wi9ePI9u3Trg559P4Lvv1mH48IHo3fslzJ07A6mpd+rsXgDYuXMbRo8eht69u2L69NcQGxvDPnUiIqog2KcJ3hjqj5S7uYiMioNaw1O7SRicKRfI2cvpiPo5BVl5pXCyVWBkD+9GuaVXTk423ntvPvr3D0NY2CC4upbVeODAPlhYWGLs2ImwtLTAhQvnsW7dGhQWFmLWrLlPfd0NG9ZDLJZgwoTXkJ+fh82bN+If//gIa9duqJN7d+3ageXLlyAkpB3Gjh2Pe/fu4cMPF8LGxgbOzi7P/hdCREQmp6OfC1TqNli/PxGrd1/GzBEBkEo4b0kNi6FcAGcvp2PDwSSoNDoAQFZeKTYcTAKARhfMHzzIxAcf/A2DBw+rMP7xx59AofirjWX48AgsXfopdu3ajunTZ0Aul1f7uhqNBv/3fxsglZZ9Ctra2uHLL5fh+vVktGrl81z3qtVqrFu3Gv7+gVix4mvDdT4+L2Dx4o8ZyomIqJKuge5QqbXYeOQa1u27gjeG+EMs5rkT1HAYyp/D6fh7+DXuXq3vS0nLhUZbcX2tSqPDfw4k4pdLabV+vW5B7uga6F7r+2pCqVQiLGxQpfFHA3lRUSFUKjWCg0MRHR2FW7du4oUXWlf7uoMGDTWEZQAIDg4BAKSl3X1qKH/avUlJV5Cbm4uZM0dUuK5fvzB89dW/q31tIiIyX73aNUOpWodtJ5Mhk4oxJbwNxDwQjhoIQ7kAHg/kTxsXkrOzS4VgW+769RSsXbsaFy/+jsLCiscVFxYWPPV1y9tgytnY2AIA8vMr735T23vT08t+UGrWzLPCdVKpFO7u9fPDCxERmYawzl4oUWmw5/RNKGQSTOzXmic1U4NgKH8OXQOfbYb63a9PIyuv8mEFTrYKvD+xXV2UVmcenREvl5+fjzlz3oClpTVef/0teHg0g1wux7VrSVi9OhI6ne6prysWV726vSY7dD7PvURERE8zrFtLqNQ6HDp3GwqZBBE9vRnMqd4xlAtgZA/vCj3lACCXijGyh7eAVdVcTMwF5ObmYvHipQgJ+euHiHv3at96Ux/c3Mp+UEpNvYPg4FDDuEajwb179+DtXX17DBERmTeRSITRvbxRotbi4G+3oZRLMKRrS6HLIhPHpcUC6OLvhkkD/eBkqwBQNkM+aaBfo1vk+SRicdmnzaMz02q1Grt2bReqpAr8/NrCzs4Oe/bsgkajMYwfPXoI+fl5AlZGRETGQiQS4ZX+rfFSgBt2nbqBI+duC10SmTjOlAuki7+b0YTwxwUGBsHGxhaLF3+MiIixEIlEOHz4ABpL94hMJsPUqW9g+fKlmDdvJnr16oN79+7h4MG98PBoxl9BEhFRjYhFIkwJ94NKrcWWE8mQyyXoGeIhdFlkojhTTrVmZ2ePJUuWw8mpCdauXY3Nm39Ahw6dMXPm20KXZjBq1FjMm7cQ6en3sGrVl4iNjcG//vVvWFvbQC5XCF0eEREZCYlYjDeG+iPI2wkbD13F2YR0oUsiEyXSc3VcBZmZlXf/SE+/BTe35gJUQ3VJp9Nh8OB+6NGjF95//6N6fS9+zhARmRaVWosvd8Th6u0czBjuj/a+PPOCak8sFsHJybrq5xq4FqIGUVpaeXebQ4f2Iy8vF6Gh7QWoiIiIjJlcJsGcUYFo2dQGa6IvIy4lS+iSyMSwp5xMUlzcJaxeHYmePXvD1tYO164lYf/+PWjVyhu9evUVujwiIjJCSrkU80cHY+nmS1i1Kx7zRwfDr7mD0GWRieBMOZmkpk090KSJM3bs2IoVK5bi119/QVjYIHz55WrIZDKhyyMiIiNlqZRhwdhgONtb4MsdcUi5myt0SWQi2FP+GPaUU13g5wwRkWnLKSjFv364iPxiNd4bH4rmbjZCl0RGgD3lRERERHXI3lqBheNDYKGQ4Iutl5D2oFDoksjIMZQTERERPYMmdhZ4d1woJGIRlm6JQUZ2kdAlkRFjKCciIiJ6Rq6OlnhnXAi0Wj2Wbr6Eh3klQpdERoqhnIiIiOg5NHO2xoKxwSgqVWPp5hjkFlTelpfoaRjKiYiIiJ5TCzdbzB8dguyCUizbegkFxWqhSyIjw1BOREREVAd8mtnh7VFBuP+wGF9svYSiEo3QJZERYSgnIiIiqiNtWzhi1ogApGYU4MsdsShVaYUuiYwEQzkRERFRHQr2aYI3hvoj+W4uIqPioNYwmNPTMZRTnThwYC+6deuAe/fSDGMREUOwePHHz3Tv87p48Ty6deuAixfP19lrEhER1VRHPxdMDW+DKzezsXr3ZWi0OqFLokaOodxMvffefPTt2w3FxcVPvGbBgtkYMKAHSksb7yryY8cOY9u2H4Uug4iIqJKuge54pX9rXEp+gHX7rkCn4yHq9GRSoQsgYfTrNwBnzpzCr7/+jH79wio9n539EBcu/I7+/QdCoVA803v8+ONOiMX1+3Pf8eNH8Mcf1zBmzIQK4yEh7XD8+GnIZLJ6fX8iIqLq9G7XDCq1DttOJkMulWByuB/EIpHQZVEjxJlyM/Xyyz1hYWGJY8cOV/n8iRPHoNVq0b9/5cBeU3K5HFKpMD/3icViKBSKev+hgIiI6GnCOnthaNcW+DX+HjYf/QN6PWfMqTLOlJsppVKJl1/ugZMnjyEvLw+2trYVnj927DCcnJzg6dkcy5b9CxcunMP9+/ehVCrRrl0HzJo1F+7uTat9j4iIIQgNbY9Fiz42jF2/noIVK5YiISEednZ2GDZsJJo0ca5076lTP2HPnl24du0q8vJy4ezsgvDwIXj11SmQSCQAgNmz38ClSxcBAN26dQAAuLm5Y8eOvbh48TzefvstfPXVGrRr18HwusePH8EPP3yHW7duwtLSCl27vowZM96Gvb294ZrZs99AQUEB/v73/4d//3sJEhMvw8bGFqNHj8PEiZNq9xdNREQEYFi3lihVa3H43B3I5WJE9PCGiDPm9AiGcoGcS7+IPSmHkF2aAweFPYZ6h6GTW7sGraFfvzAcOXIQP/10HEOHjjCMp6ffQ0JCHCIixiEx8TISEuLQt+8AODu74N69NOzevRNz5ryJH37YDqVSWeP3y8p6gLfffgs6nQ6vvDIJSqUF9uzZVWV7zIED+2BhYYmxYyfC0tICFy6cx7p1a1BYWIhZs+YCACZNmori4mLcv38Pc+YsAABYWFg+8f0PHNiLTz/9B/z9AzFjxtvIyLiPnTu3IjHxMtau/b5CHXl5uXjnnbfRq1cf9OnTHydPHsPq1ZFo1coHXbp0rfHHTEREBAAikQhjevmgVK3Dwf/ehlImwZCuLYUuixoRhnIBnEu/iB+TdkKtKzvtK7s0Bz8m7QSABg3mHTt2hr29A44dO1whlB87dhh6vR79+g2At7cPevXqW+G+rl274623puCnn44jLGxQjd9v06YNyM3Nwbp1G+Hr6wcAGDhwMMaPH1Hp2o8//gQKxV+Bf/jwCCxd+il27dqO6dNnQC6Xo2PHFxEVtR25uTkYMCC82vfWaDRYvToSPj6tERn5DeRyOQDA19cPH3+8CHv37kJExDjD9RkZ9/G///uJod9+8OBhiIgYjP37oxnKiYjomYhEIrzSvzVUai12nboBhUyC/p28hC6LGgmG8ufw270LOHvv91rfdyP3NjT6iqd8qXVqbErcgTNp52r9el3cO6Kze/ta3yeVStG7d1/s3r0TDx48QJMmTQAAx44dQbNmnmjbNqDC9RqNBoWFBWjWzBPW1ja4di2pVqH87NnTCAwMNgRyAHBwcEC/fgOxa9f2Ctc+GsiLigqhUqkRHByK6Ogo3Lp1Ey+80LpWH2tS0hVkZz80BPpyvXv3w6pVX+LMmdMVQrm1tTX69h1geCyTydCmjT/S0u7W6n2JiIgeJRaJMCXcDyq1FltOJEMul6BniIfQZVEjwFAugMcD+dPG61O/fmGIitqOEyeOYMyYCbh58waSk69hypTpAIDS0hJs3PgdDhzYi8zMjAqLUwoKCmr1XvfvpyMwMLjSuJdX80pj16+nYO3a1bh48XcUFhZWeK6wsHbvC5S15FT1XmKxGM2aeeL+/XsVxl1cXCv1+tnY2CIlJbnW701ERPQoiViMN4b6QxUVj42HrkIhlaBLgJvQZZHABA3lKpUKX375JaKjo5GXlwc/Pz/Mnz8fXbp0qfa+uLg4REVFIS4uDteuXYNarcbVq1cbqOq/dHZv/0wz1B+d/hTZpTmVxh0U9pjX7q26KK3GAgOD4e7ugaNHD2HMmAk4evQQABjaNpYvX4oDB/Zi9OjxCAgIhLW1NQARPv74f+pt9Xh+fj7mzHkDlpbWeP31t+Dh0QxyuRzXriVh9epI6HT1fwCDWCypcpwr5omIqC5IJWLMHB6AL3fEYf3+RMhlYrT3dRG6LLNy9nI6on5OQVZeKZxsFRjZwxtd/IX74UjQ/eI++OADbNiwAUOHDsWiRYsgFosxffp0xMTEVHvfzz//jO3by9odPD09G6LUOjXUOwwyccX9s2ViGYZ6P/v2g8+jb9/+SEy8gtTUOzh+/Ah8fdsYZpTL+8bnzJmPXr36omPHFxEUFFLrWXIAcHV1Q2rqnUrjt2/fqvA4JuYCcnNzsWjR/2LMmPHo2vVldOzYGTY2tpXuBWq2ct3Nzb3K99Lr9UhNvQNXV/eafRBERER1RC6TYM6oQLRsaoM10ZcRl5IldElm4+zldGw4mISsvLIDErPySrHhYBLOXk4XrCbBQnlcXBz279+PhQsX4r333sPYsWOxYcMGuLu7Y9myZdXeO378eFy4cAFRUVHo1q1bA1Vcdzq5tcMEv1FwUJRtw+egsMcEv1ENvvtKuf79BwIAVq5cjtTUOxX2Jq9qxnjnzq3QarW1fp8uXboiPj4WV68mGcays7Nx9OjBCteV7y3+6Ky0Wq2u1HcOABYWFjX6AcHPry0cHByxe/cOqNVqw/jJk8eRmZmBl17i4k0iImp4SrkU80cHw8PZCl/tiMW8r05h6r9O4N2vTwsaEI2dXq+HSq1FbkEp7mUVIiUtFwk3svB7UgZ+iU3DpiPXoNJU/M27SqND1M8pAlUsYPvKoUOHIJPJMHr0aMOYQqFAREQEli9fjoyMDLi4VP1rnPIFicask1s7wUL441q2bAUfn9b49ddfIBaL0afPXwscX3qpGw4fPgArK2u0aNESly/H4/z5c7Czs6v1+0yYMAmHDx/AggWzEBExDgqFEnv27IKrqzsKCv4wXBcYGAQbG1ssXvwxIiLGQiQS4fDhA6iqc8TX1w9HjhxEZOS/4efXFhYWlujWrXul66RSKWbMmINPP/0H5sx5E3379kdGxn3s2LEVrVp5Y8iQyjvAEBERNQRLpQw9Qprih8PXkFdUNnFUPnMLQNCWCqGoNToUl2pQXKpBUfmfJZrHxrQoKlWjuFRb5XVaXe1bTstnzoUgWChPTExEy5YtYWVlVWE8KCgIer0eiYmJTwzlVPf69w9DcvI1hIa2r/BDz9y5CyEWi3H06EGUlqoQGBiMFStWYcGCObV+jyZNmuCrr77B8uVLsHHjdxUOD/rXv/5puM7Ozh5LlizHypUrsHbtatjY2KJ//4Ho0KETFiyYXeE1hw0bhWvXknDgwD5s3foj3NzcqwzlABAePgRyuRybNm3AqlVfwsrKCv36heGtt+ZUuVc6ERFRQzlw9hYej5DlM7fGFsq1Ot2fgVmD4pK/wrIhOD82Vj5e9Ge4Li7VQK15+voxpVwCC4UUlkopLBRS2FnJ4eZoWTamkMJCIfnzz7+uKX9u8cYLyM6vHMCdbIXLAyK9QCvXBg8eDFdXV6xfv77CeHJyMgYNGoRPPvmkwiz6kyxevBjff/99nS30zMzMrzSWnn4Lbm6VdwghehJ+zhARUW1M/deJJz43fUjbBgvmOp0exarHg/NfM9IVAnalmeuyP1XqpwdqhUwCC4XkkQBdMTiXj1saHv8VwC0VUijlUojFz34ianlP+aMtLHKpGJMG+tXr37VYLIKTk3WVzwk2U15SUgKZTFZpvHzGsrRUmF8fODvbVBrLyBBDKhV0TSwZGbFYXOXnEhERUVWcHSyQmV1c5XPfH7oKWxsleravfnMLnU6PEpUGhcUaFJWoUVCsRlGJGoUlZY8Li8v+KSrRoLDkzz+L1WX/Xlx2XXHp07dnlkvFsLSQwUophZWFDDbWCrg1sYbln4+tLGRl/66UwVIpg3X5Y4uyx5ZKKaQSYXPV0J42sLVR4vuDiXiQXYwmDhZ4bWCbp/4d1yfBQrlSqayw4K5ceRgXqp2gqplynU4HTQ1+jUJUTqfTVfm5REREVJXh3VpWmrktV6rWYvXOOCTdyKo0S/1oy0dxqaZSC8zjJGJRpVaOJrZKWDpbG2akLRVSWCgfnaWWVhh75kCt06G0qBSlRcL1bT/K38sen79ZcRvu+v7e3Shnyp2dnZGRkVFpPDMzEwDYT05ERERmo7xlYu3eK1U+X1SqwcH/3q7Q8mGplMLZXlm55UMpfSRgyyr0Vsuk4kqH41HjIFgo9/Pzw8aNG1FYWFhhsWdsbKzheSIiIiJz0cXfzXCYzeMcbRRYOvMlBmoTJlhDT1hYGNRqteEQIKDshM+oqCi0a9cOrq6uAIC0tDSkpAi3ZyQRERFRQxnZwxvyx9axyaVijOrpzUBu4gSbKQ8ODkZYWBiWLVuGzMxMeHl5YdeuXUhLS8Nnn31muO7999/HuXPnKuyucvfuXURHRwMA4uPjAQBff/01gLIZ9t69ezfgR0JERERUN8rbWBrT8e/UMAQL5QCwZMkSrFixAtHR0cjNzYWvry++/fZbtG/fvtr7UlNT8eWXX1YYK388YsSIegnler2eP6FSjQi0yygREZmILv5uDOFmSLB9yhurqlbdZmamwc7OEXK5UoCKyNioVKXIzX0AZ2cPoUshIiKiRqS63Ve4+XYNWFvbISfnAQoL86HVajgTSlXS6/VQqUqRk5MJa2t7ocshIiIiIyJo+4qxsLCwglQqQ0FBDgoLc6HTaYUuiRopiUQKGxsHWFhYPf1iIiIioj8xlNeQTCaHgwP3TiciIiKiusf2FSIiIiIigTGUExEREREJjKGciIiIiEhgDOVERERERAJjKCciIiIiEhh3X3mMWMxTO4mIiIio7lWXM3miJxERERGRwNi+QkREREQkMIZyIiIiIiKBMZQTEREREQmMoZyIiIiISGAM5UREREREAmMoJyIiIiISGEM5EREREZHAGMqJiIiIiATGUE5EREREJDCGciIiIiIigUmFLkAoGzZsQGRkJPLz8w1jdnZ2ePnll7Fv3z74+fkhOjpawAqJiIiIyFyY7Uz5li1bkJ+fD7lcDgcHBwBAXl4e9u3bB7lcLnB1RERERGROzDaUf/TRRzh16hTi4+PxySefAABCQkIAAHq9XsjSiIiIiMjMmG37SteuXSuNxcbGQiaTQavVClAREREREZkrs50pL1dQUGDoK/fw8IBarYZMJhO4KiIiIiIyJ2Y7U17uf/7nf3D48GEAQGpqKgDA3d1dyJKIiIiIyMyY/Uz5lClTYGlpCQAQiURwcnKCo6OjwFURERERkTkx+1B+7Ngxw24rDg4OsLe3x40bNwSuioiIiIjMiVmH8oyMDGzYsMGw28q0adPQpUsXZGdnQ6VSITU1Fbm5uQJXSURERESmTqQ34/3/YmNjMWbMmGqvmT59OhYuXNhAFRERERGROTLbmfLMzEysXLkSYrEYYWFhAIAZM2bA0dERUqkUHh4eWLVqFYYPHy5wpURERERk6sx2prxHjx5IT0+Hvb09bGxscOfOHcjlcqhUKtjY2MDDwwPR0dFCl0lEREREZsBst0S0sLAAAOTk5CAnJwcAoFKpAMCwbzkRERERUUMw25lyIiIiIqLGwmx7yomIiIiIGguGciIiIiIigTGUExEREREJjKGciIiIiEhgDOVERERERAJjKCciIiIiEhhDORERERGRwBjKiYhIMK+++ip69+4tdBlERIIz2xM9iYhM1W+//YbXXnvtic9LJBJcuXKlASsiIqKnYSgnIjJRgwcPRvfu3SuNi8X8JSkRUWPDUE5EZKLatm2LYcOGCV0GERHVAKdLiIjMVGpqKnx9fREZGYl9+/ZhyJAhCAwMRM+ePREZGQmNRlPpnqSkJMyaNQudO3dGYGAgwsPDsXbtWmi12krXZmZm4pNPPkGfPn0QEBCALl26YMqUKTh9+nSla+/fv48FCxagY8eOCA4Oxuuvv44bN27Uy8dNRNQYcaaciMhEFRcX4+HDh5XG5XI5rK2tDY9PnDiBO3fuYOLEiWjSpAlOnDiBlStXIi0tDZ999pnhuvj4eLz66quQSqWGa0+ePIlly5YhKSkJX3zxheHa1NRUjB8/HllZWRg2bBgCAgJQXFyM2NhYnDlzBl27djVcW1RUhFdeeQXBwcGYP38+UlNT8f3332PmzJnYt28fJBJJPf0NERE1HgzlREQmKjIyEpGRkZXGe/bsiW+++cbwOCkpCTt27IC/vz8A4JVXXsHs2bMRFRWFsWPHIiQkBACwePFiqFQqbNmyBX5+foZr582bh3379iEiIgJdunQBAPzjH/9ARkYG1q1bh5dffrnC++t0ugqPs7Oz8frrr2P69OmGMUdHRyxduhRnzpypdD8RkSliKCciMlFjx45FWFhYpXFHR8cKj1966SVDIAcAkUiEadOm4dixYzh69ChCQkKQlZWFmJgY9OvXzxDIy6+dMWMGDh06hKNHj6JLly7IycnBqVOn8PLLL1cZqB9faCoWiyvtFvPiiy8CAG7dusVQTkRmgaGciMhENW/eHC+99NJTr/P29q405uPjAwC4c+cOgLJ2lEfHH9WqVSuIxWLDtbdv34Zer0fbtm1rVKeLiwsUCkWFMXt7ewBATk5OjV6DiMjYcaEnEREJqrqecb1e34CVEBEJh6GciMjMpaSkVBpLTk4GAHh6egIAmjVrVmH8UdevX4dOpzNc6+XlBZFIhMTExPoqmYjI5DCUExGZuTNnzuDy5cuGx3q9HuvWrQMA9O3bFwDg5OSE0NBQnDx5EteuXatw7bfffgsA6NevH4Cy1pPu3bvjl19+wZkzZyq9H2e/iYgqY085EZGJunLlCqKjo6t8rjxsA4Cfnx8mTZqEiRMnwtnZGcePH8eZM2cwbNgwhIaGGq5btGgRXn31VUycOBETJkyAs7MzTp48iV9//RWDBw827LwCAH/7299w5coVTJ8+HcOHD4e/vz9KS0sRGxsLDw8PvB5rLtQAAAD2SURBVPvuu/X3gRMRGSGGciIiE7Vv3z7s27evyueOHDli6OXu3bs3WrZsiW+++QY3btyAk5MTZs6ciZkzZ1a4JzAwEFu2bMFXX32FzZs3o6ioCJ6enli4cCGmTp1a4VpPT0/s3LkTq1atwi+//ILo6GjY2trCz88PY8eOrZ8PmIjIiIn0/D0iEZFZSk1NRZ8+fTB79mzMmTNH6HKIiMwae8qJiIiIiATGUE5EREREJDCGciIiIiIigbGnnIiIiIhIYJwpJyIiIiISGEM5EREREZHAGMqJiIiIiATGUE5EREREJDCGciIiIiIigTGUExEREREJ7P8D83+w3j+JgI0AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}