trainer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "collapsed_sections": [
        "JWngv0NcwCjA",
        "yzCv692FwHao",
        "dzGuQNwQwnnC",
        "LDnwkVx6xHoO",
        "4v264h-vX-hz",
        "VhGouCKy0J81",
        "J4dPO6700UQo",
        "49MBTd-J0g9J",
        "4lYkfDGE6x-D"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b78908d1186f44dbb7d663e0bcd37769": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c731c744daed4a9aac52936ad8772b3a",
              "IPY_MODEL_0ef6a99cbe8b46e88c24cfca437ab443",
              "IPY_MODEL_6a4bf9fc7aa54c5ba2128965d02f9135"
            ],
            "layout": "IPY_MODEL_61084fe2d5bd4851ae77a9a0f38ca3cf"
          }
        },
        "c731c744daed4a9aac52936ad8772b3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e0171cc137834d6bbd2013a6abcd1c8e",
            "placeholder": "​",
            "style": "IPY_MODEL_597c81db94694bd987d62d7f1fba54ae",
            "value": "Map: 100%"
          }
        },
        "0ef6a99cbe8b46e88c24cfca437ab443": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_22a0d1b8076246c491123c801445e3d3",
            "max": 7000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c9622a62881240ab976e248ed8c73673",
            "value": 7000
          }
        },
        "6a4bf9fc7aa54c5ba2128965d02f9135": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5c8b58bd471b4f6eb4593299e58ec5a9",
            "placeholder": "​",
            "style": "IPY_MODEL_bc1d2db5acdc4d21a70227fe03441696",
            "value": " 7000/7000 [00:02&lt;00:00, 3365.03 examples/s]"
          }
        },
        "61084fe2d5bd4851ae77a9a0f38ca3cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e0171cc137834d6bbd2013a6abcd1c8e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "597c81db94694bd987d62d7f1fba54ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "22a0d1b8076246c491123c801445e3d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c9622a62881240ab976e248ed8c73673": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5c8b58bd471b4f6eb4593299e58ec5a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bc1d2db5acdc4d21a70227fe03441696": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b55fb0ac49f146f48068a3e51855b65d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2195c097919e439fa9ed612410336b67",
              "IPY_MODEL_6e8ced64c9d6406a8c0dad7b4cbb462a",
              "IPY_MODEL_6982a2b8d0ff4ca7afc5c8ad887577c6"
            ],
            "layout": "IPY_MODEL_ff4f4c876b784d37b415388c8c91dea9"
          }
        },
        "2195c097919e439fa9ed612410336b67": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f73cc9a23c4a4e91b97b0b63a002bcb7",
            "placeholder": "​",
            "style": "IPY_MODEL_bd847732ee8f49879aace37bc9e17b51",
            "value": "Map: 100%"
          }
        },
        "6e8ced64c9d6406a8c0dad7b4cbb462a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7a0d124331ff43da9c9a7dd285a010d0",
            "max": 62480,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5aa2a64a90444194a24137c07ae711bd",
            "value": 62480
          }
        },
        "6982a2b8d0ff4ca7afc5c8ad887577c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_66824aed874d472485cf7e0f940adb42",
            "placeholder": "​",
            "style": "IPY_MODEL_92990f869eff48e99d19a591a3c27214",
            "value": " 62480/62480 [00:21&lt;00:00, 1918.10 examples/s]"
          }
        },
        "ff4f4c876b784d37b415388c8c91dea9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f73cc9a23c4a4e91b97b0b63a002bcb7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bd847732ee8f49879aace37bc9e17b51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7a0d124331ff43da9c9a7dd285a010d0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5aa2a64a90444194a24137c07ae711bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "66824aed874d472485cf7e0f940adb42": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "92990f869eff48e99d19a591a3c27214": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# **Import & install Important Libraries**"
      ],
      "metadata": {
        "id": "JWngv0NcwCjA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers\n",
        "!pip install datasets\n",
        "!pip install --upgrade accelerate\n",
        "!pip install -U sentence-transformers\n",
        "!pip install hazm"
      ],
      "metadata": {
        "id": "GdmBjxypRfCF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "QxwZdv44n-B5"
      },
      "outputs": [],
      "source": [
        "import hazm\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "from scipy import optimize\n",
        "from sklearn import metrics\n",
        "from sklearn.metrics import confusion_matrix, classification_report, accuracy_score, precision_score, recall_score\n",
        "# from transformers import AutoModelForSequenceClassification, TrainingArguments, Trainer, AutoConfig, AutoTokenizer, AutoModel\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "from transformers import DataCollatorWithPadding,AutoModelForSequenceClassification, Trainer, TrainingArguments,AutoTokenizer,AutoModel,AutoConfig\n",
        "from transformers.modeling_outputs import TokenClassifierOutput\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from tqdm.notebook import tqdm_notebook\n",
        "import copy\n",
        "from datasets import load_metric , Dataset\n",
        "from transformers import TrainingArguments, Trainer\n",
        "from torch.utils.data import DataLoader\n",
        "from transformers import AdamW, get_scheduler\n",
        "import sys\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Import Train Dataset**"
      ],
      "metadata": {
        "id": "yzCv692FwHao"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data = pd.read_csv('/content/drive/MyDrive/Snappfood - Sentiment Analysis.csv',  on_bad_lines='skip' , delimiter='\\t')"
      ],
      "metadata": {
        "id": "pYO3C-pOozQy"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "F7NDfO7apFNb",
        "outputId": "07698c65-e8d3-4094-8d8e-3390d31bc703"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  Unnamed: 0                                            comment  label  \\\n",
              "0        NaN    واقعا حیف وقت که بنویسم سرویس دهیتون شده افتضاح    SAD   \n",
              "1        NaN  قرار بود ۱ ساعته برسه ولی نیم ساعت زودتر از مو...  HAPPY   \n",
              "2        NaN  قیمت این مدل اصلا با کیفیتش سازگاری نداره، فقط...    SAD   \n",
              "3        NaN  عالللی بود همه چه درست و به اندازه و کیفیت خوب...  HAPPY   \n",
              "4        NaN                      شیرینی وانیلی فقط یک مدل بود.  HAPPY   \n",
              "\n",
              "   label_id  \n",
              "0       1.0  \n",
              "1       0.0  \n",
              "2       1.0  \n",
              "3       0.0  \n",
              "4       0.0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-daa135df-894f-4de5-ba6e-7decda04941a\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>comment</th>\n",
              "      <th>label</th>\n",
              "      <th>label_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NaN</td>\n",
              "      <td>واقعا حیف وقت که بنویسم سرویس دهیتون شده افتضاح</td>\n",
              "      <td>SAD</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>قرار بود ۱ ساعته برسه ولی نیم ساعت زودتر از مو...</td>\n",
              "      <td>HAPPY</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NaN</td>\n",
              "      <td>قیمت این مدل اصلا با کیفیتش سازگاری نداره، فقط...</td>\n",
              "      <td>SAD</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NaN</td>\n",
              "      <td>عالللی بود همه چه درست و به اندازه و کیفیت خوب...</td>\n",
              "      <td>HAPPY</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NaN</td>\n",
              "      <td>شیرینی وانیلی فقط یک مدل بود.</td>\n",
              "      <td>HAPPY</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-daa135df-894f-4de5-ba6e-7decda04941a')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-daa135df-894f-4de5-ba6e-7decda04941a button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-daa135df-894f-4de5-ba6e-7decda04941a');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-6a732b26-16b0-4c3b-bd44-e9bbfa8198d1\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-6a732b26-16b0-4c3b-bd44-e9bbfa8198d1')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-6a732b26-16b0-4c3b-bd44-e9bbfa8198d1 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "data",
              "summary": "{\n  \"name\": \"data\",\n  \"rows\": 70000,\n  \"fields\": [\n    {\n      \"column\": \"Unnamed: 0\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 520,\n        \"samples\": [\n          \" \\u0647\\u0632\\u0627\\u0631 \\u062a\\u0648\\u0645\\u0646 \\u0628\\u0633\\u062a\\u0647 \\u0628\\u0646\\u062f\\u06cc \\u0631\\u0648 \\u0645\\u0646 \\u0645\\u062a\\u0648\\u062c\\u0647 \\u0646\\u0634\\u062f\\u0645 \\u0648\\u0627\\u0642\\u0639\\u0627\\u060c \\u0628\\u0631\\u0627\\u06cc \\u06cc\\u06a9 \\u067e\\u0627\\u06a9\\u062a \\u0645\\u0642\\u0648\\u0627\\u06cc\\u06cc \\u06f8 \\u0647\\u0632\\u0627\\u0631 \\u062a\\u0648\\u0645\\u0646 \\u0622\\u062e\\u0647\",\n          \"- \\u0642\\u06cc\\u0645\\u062a \\u067e\\u06cc\\u062a\\u0632\\u0627 \\u06f3\\u06f0 \\u0647\\u0632\\u0627\\u0631 \\u062a\\u0648\\u0645\\u0627\\u0646 \\u0628\\u0648\\u062f \\u0627\\u0645\\u0627 \\u0627\\u0632 \\u0646\\u0638\\u0631 \\u06a9\\u06cc\\u0641\\u06cc\\u062a\\u060c \\u0637\\u0639\\u0645 \\u0648 \\u062d\\u062c\\u0645 \\u062f\\u0631 \\u0648\\u0627\\u0642\\u0639 \\u06cc\\u06a9 \\u067e\\u06cc\\u062a\\u0632\\u0627\\u06cc \\u06a9\\u0627\\u0645\\u0644\\u0627 \\u0645\\u0639\\u0645\\u0648\\u0644\\u06cc \\u0628\\u0647 \\u0627\\u0631\\u0632\\u0634 \\u062a\\u0642\\u0631\\u06cc\\u0628\\u06cc \\u06f1\\u06f5 \\u0647\\u0632\\u0627\\u0631 \\u062a\\u0648\\u0645\\u0627\\u0646 \\u0628\\u0648\\u062f! \\u0627\\u0632 \\u0686\\u0646\\u06cc\\u0646 \\u067e\\u06cc\\u062a\\u0632\\u0627\\u06cc\\u06cc \\u0627\\u0646\\u062a\\u0638\\u0627\\u0631 \\u0646\\u0645\\u06cc\\u200c\\u0631\\u0648\\u062f \\u06a9\\u0647 \\u062f\\u0642\\u06cc\\u0642\\u0627 \\u062a\\u062c\\u0631\\u0628\\u0647 \\u067e\\u06cc\\u062a\\u0632\\u0627\\u0647\\u0627\\u06cc \\u06a9\\u0627\\u0645\\u0644\\u0627 \\u0645\\u0639\\u0645\\u0648\\u0644\\u06cc \\u0631\\u0627 \\u0628\\u0631\\u0627\\u06cc \\u0645\\u0634\\u062a\\u0631\\u06cc \\u062a\\u06a9\\u0631\\u0627\\u0631 \\u06a9\\u0646\\u062f! \\u062f\\u0631 \\u0645\\u0646\\u0648 \\u0646\\u0648\\u0634\\u062a\\u0647 \\u0634\\u062f\\u0647 \\u0628\\u0648\\u062f \\u0647\\u0645\\u0631\\u0627\\u0647 \\u0628\\u0627 \\u0633\\u0633 \\u0645\\u062e\\u0635\\u0648\\u0635\\u060c \\u0627\\u0645\\u0627 \\u063a\\u0630\\u0627 \\u0641\\u0627\\u0642\\u062f \\u0633\\u0633 \\u0645\\u062e\\u0635\\u0648\\u0635 \\u0628\\u0648\\u062f! \\u063a\\u0630\\u0627 \\u0633\\u0631\\u062f \\u062a\\u062d\\u0648\\u06cc\\u0644 \\u062f\\u0627\\u062f\\u0647 \\u0634\\u062f!\",\n          \"\\u0639\\u062f\\u062f \\u0628\\u0633\\u062a\\u0646\\u06cc \\u062f\\u0627\\u062e\\u0644 \\u0633\\u0641\\u0627\\u0631\\u0634\\u0645 \\u0628\\u0648\\u062f \\u0627\\u0645\\u0627 \\u0627\\u0631\\u0633\\u0627\\u0644 \\u0646\\u0634\\u062f \\u062d\\u062a\\u06cc \\u062f\\u0627\\u062e\\u0644 \\u0641\\u0627\\u06a9\\u062a\\u0648\\u0631 \\u0647\\u0645 \\u062a\\u06cc\\u06a9 \\u0646\\u062e\\u0648\\u0631\\u062f\\u0647 \\u0627\\u06af\\u0631 \\u0646\\u062f\\u0627\\u0634\\u062a\\u0646 \\u0628\\u0627\\u06cc\\u062f \\u0627\\u0637\\u0644\\u0627\\u0639 \\u0645\\u06cc\\u062f\\u0627\\u062f\\u0646 \\u0645\\u0645\\u0646\\u0648\\u0646\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"comment\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 69482,\n        \"samples\": [\n          \"\\u062e\\u06cc\\u0644\\u06cc \\u0628\\u062f \\u0628\\u0648\\u062f\\u060c\\u060c\\u060c \\u0627\\u0632 \\u0627\\u06cc\\u0646 \\u0628\\u062f\\u062a\\u0631 \\u0646\\u0645\\u06cc\\u200c\\u0634\\u062f \\u0627\\u06cc\\u0646 \\u0647\\u0645\\u0647 \\u06af\\u0648\\u0634\\u062a \\u0648 \\u0645\\u0631\\u063a \\u0631\\u0648 \\u0628\\u062f \\u0648 \\u0686\\u0631\\u0628 \\u062f\\u0631\\u0633\\u062a \\u06a9\\u0631\\u062f\",\n          \"\\u06cc\\u0647 \\u0646\\u0627\\u0646 \\u0633\\u0646\\u06af\\u06a9 \\u0633\\u06cc\\u0627\\u0647 \\u062f\\u0627\\u0646\\u0647 \\u0648 \\u06cc\\u0647 \\u0646\\u0627\\u0646 \\u0633\\u0646\\u06af\\u06a9 \\u0633\\u0627\\u062f\\u0647 \\u0633\\u0641\\u0627\\u0631\\u0634 \\u062f\\u0627\\u062f\\u0645 \\u0648\\u0644\\u06cc \\u0645\\u062a\\u0627\\u0633\\u0641\\u0627\\u0646\\u0647 \\u062f\\u0648\\u062a\\u0627 \\u0646\\u0627\\u0646 \\u06a9\\u0646\\u062c\\u062f\\u06cc \\u0645\\u0639\\u0645\\u0648\\u0644\\u06cc \\u0622\\u0645\\u062f\",\n          \"\\u0639\\u0644\\u06cc \\u0631\\u063a\\u0645 \\u06a9\\u06cc\\u0641\\u06cc\\u062a \\u062e\\u0648\\u0628 \\u0648\\u0644\\u06cc \\u067e\\u06cc\\u06a9 \\u0645\\u06cc\\u062a\\u0648\\u0646\\u0633\\u062a \\u0632\\u0648\\u062f\\u062a\\u0631 \\u0628\\u0631\\u0633\\u0647\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"label\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          \"HAPPY\",\n          \"0\",\n          \"SAD\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"label_id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.4999971814981656,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0.0,\n          1.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_Hey9_8rzT7H",
        "outputId": "b6706f1b-672b-4664-f697-d537a18b06fc"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 70000 entries, 0 to 69999\n",
            "Data columns (total 4 columns):\n",
            " #   Column      Non-Null Count  Dtype  \n",
            "---  ------      --------------  -----  \n",
            " 0   Unnamed: 0  520 non-null    object \n",
            " 1   comment     70000 non-null  object \n",
            " 2   label       70000 non-null  object \n",
            " 3   label_id    69480 non-null  float64\n",
            "dtypes: float64(1), object(3)\n",
            "memory usage: 2.1+ MB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data = data[['comment' , 'label_id']]\n",
        "data.dropna(inplace=True)"
      ],
      "metadata": {
        "id": "l_pyyc_GvjfO"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data['label_id'] = data['label_id'].astype(int)\n",
        "data = data.rename(columns={\"label_id\": \"label\"})\n",
        "data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "Vvae9YxSwBYd",
        "outputId": "c8ce577d-45a4-4d79-ae35-b88f586ea99d"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                             comment  label\n",
              "0    واقعا حیف وقت که بنویسم سرویس دهیتون شده افتضاح      1\n",
              "1  قرار بود ۱ ساعته برسه ولی نیم ساعت زودتر از مو...      0\n",
              "2  قیمت این مدل اصلا با کیفیتش سازگاری نداره، فقط...      1\n",
              "3  عالللی بود همه چه درست و به اندازه و کیفیت خوب...      0\n",
              "4                      شیرینی وانیلی فقط یک مدل بود.      0"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2bce1201-d0ce-4625-98fb-ddb530407a74\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>comment</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>واقعا حیف وقت که بنویسم سرویس دهیتون شده افتضاح</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>قرار بود ۱ ساعته برسه ولی نیم ساعت زودتر از مو...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>قیمت این مدل اصلا با کیفیتش سازگاری نداره، فقط...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>عالللی بود همه چه درست و به اندازه و کیفیت خوب...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>شیرینی وانیلی فقط یک مدل بود.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2bce1201-d0ce-4625-98fb-ddb530407a74')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-2bce1201-d0ce-4625-98fb-ddb530407a74 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-2bce1201-d0ce-4625-98fb-ddb530407a74');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-7f6535cc-8722-4b22-8d42-482bdb8a6394\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-7f6535cc-8722-4b22-8d42-482bdb8a6394')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-7f6535cc-8722-4b22-8d42-482bdb8a6394 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "data",
              "summary": "{\n  \"name\": \"data\",\n  \"rows\": 69480,\n  \"fields\": [\n    {\n      \"column\": \"comment\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 69480,\n        \"samples\": [\n          \"\\u063a\\u0630\\u0627 \\u062e\\u06cc\\u0644\\u06cc \\u0633\\u0631\\u062f \\u0628\\u0648\\u062f \\u062f\\u0631 \\u0635\\u0648\\u0631\\u062a\\u06cc\\u06a9\\u0647 \\u0641\\u0627\\u0635\\u0644\\u0647 \\u0645\\u0627 \\u062e\\u06cc\\u0644\\u06cc \\u06a9\\u0645 \\u0627\\u0633\\u062a\",\n          \"\\u0628\\u0647\\u062a\\u0631\\u0647 \\u0628\\u062a\\u0648\\u0646\\u06cc\\u0645 \\u0631\\u0627\\u0646 \\u06cc\\u0627 \\u0633\\u06cc\\u0646\\u0647 \\u0631\\u0648 \\u062e\\u0648\\u062f\\u0645\\u0648\\u0646 \\u0627\\u0646\\u062a\\u062e\\u0627\\u0628 \\u06a9\\u0646\\u06cc\\u0645\",\n          \"\\u063a\\u0630\\u0627 \\u0628\\u062f \\u0628\\u0648\\u062f \\u062d\\u0627\\u0644\\u0645 \\u062e\\u06cc\\u06cc\\u06cc\\u06cc\\u06cc\\u0644\\u06cc \\u0628\\u062f\\u0647. \\u062f\\u0644 \\u062f\\u0631\\u062f\\u0648 \\u062f\\u0644 \\u067e\\u06cc\\u0686\\u0647. \\u0645\\u0639\\u0644\\u0648\\u0645 \\u0646\\u06cc\\u0633\\u062a \\u0686\\u06cc \\u062a\\u0648\\u0634 \\u0631\\u06cc\\u062e\\u062a\\u0646. \\u0627\\u0641\\u062a\\u0636\\u0627\\u0627\\u0627\\u0627\\u0627\\u0627\\u0627\\u0627\\u0627\\u062d \\u0628\\u0648\\u062f. \\u062f\\u0648\\u0633\\u0627\\u0639\\u062a\\u0647 \\u06a9 \\u062d\\u0627\\u0644\\u0645 \\u0628\\u062f\\u0647.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"label\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0,\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data['label'].hist()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 447
        },
        "id": "umTqA1a7wte0",
        "outputId": "e6beed44-b41c-4536-8ea9-6c1d60a655ef"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<Axes: >"
            ]
          },
          "metadata": {},
          "execution_count": 40
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAGdCAYAAADwjmIIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA1dklEQVR4nO3de3RU9bn/8U8SMhMiDDebhBwCRqhC5GooYbwVMGSALCqVWrwsGhHhQJOuQs4BTYvhVoulcqtGc6xiPEuogEttS2jIGAoUCSKRHG7CqUgP7ZIJVoFgkMmQ7N8fXZkf03CbmJl0vrxfa82S2fuZ7zzzJHE+a/beSZRlWZYAAAAME93WDQAAAIQCIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYKR2bd1AW2psbNSnn36qjh07Kioqqq3bAQAA18CyLJ09e1bJycmKjr785zXXdcj59NNPlZKS0tZtAACAFvjrX/+qHj16XHb/dR1yOnbsKOkfQ3I4HK22rs/nU3l5ubKyshQbG9tq6yIQcw4fZh0ezDk8mHN4hHLOtbW1SklJ8b+PX851HXKaDlE5HI5WDznx8fFyOBz8AIUQcw4fZh0ezDk8mHN4hGPOVzvVhBOPAQCAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRggo5L774ogYOHOj/DcFOp1N/+MMf/PtHjBihqKiogNuMGTMC1jh+/Liys7MVHx+vhIQEzZkzRxcuXAio2bp1q26//XbZ7Xb16dNHJSUlzXopKirSTTfdpLi4OGVkZGj37t3BvBQAAGC4oEJOjx499Mwzz6iqqkp79uzRqFGjdN999+ngwYP+mmnTpunEiRP+29KlS/37GhoalJ2drfr6eu3cuVOvvfaaSkpKVFhY6K85duyYsrOzNXLkSFVXV2vWrFl6/PHHtXnzZn/NunXrlJ+fr/nz5+vDDz/UoEGD5HK5dPLkya8zCwAAYJCgQs748eM1btw4ffOb39Qtt9yip59+Wh06dNCuXbv8NfHx8UpKSvLfLv6bUOXl5Tp06JBef/11DR48WGPHjtXixYtVVFSk+vp6SVJxcbFSU1O1bNky9evXT3l5efre976nFStW+NdZvny5pk2bpilTpigtLU3FxcWKj4/X6tWrv+48AACAIVr8BzobGhq0YcMG1dXVyel0+revWbNGr7/+upKSkjR+/Hg99dRTio+PlyRVVlZqwIABSkxM9Ne7XC7NnDlTBw8e1JAhQ1RZWanMzMyA53K5XJo1a5Ykqb6+XlVVVSooKPDvj46OVmZmpiorK6/Ys9frldfr9d+vra2V9I8/Iubz+Vo2iEtoWqs110RzzDl8mHV4MOfwYM7hEco5X+uaQYec/fv3y+l06vz58+rQoYPefvttpaWlSZIefvhh9erVS8nJydq3b5+eeOIJHTlyRG+99ZYkyePxBAQcSf77Ho/nijW1tbX66quvdOrUKTU0NFyy5vDhw1fsfcmSJVq4cGGz7eXl5f4g1prcbnerr4nmmHP4MOvwYM7hwZzDIxRzPnfu3DXVBR1ybr31VlVXV+vMmTN68803lZOTo23btiktLU3Tp0/31w0YMEDdu3fXvffeq6NHj6p3797BPlWrKygoUH5+vv9+bW2tUlJSlJWVFXBY7evy+Xxyu916ak+0vI1X/jPw/0oOLHC1dQtBaZrz6NGjFRsb29btGI1ZhwdzDg/mHB6hnHPTkZirCTrk2Gw29enTR5KUnp6uDz74QKtWrdJ//dd/NavNyMiQJH388cfq3bu3kpKSml0FVVNTI0lKSkry/7dp28U1DodD7du3V0xMjGJiYi5Z07TG5djtdtnt9mbbY2NjQ/KN7m2MkrchckJOpP6wh+rrh+aYdXgw5/BgzuERijlf63pf+/fkNDY2BpzncrHq6mpJUvfu3SVJTqdT+/fvD7gKyu12y+Fw+A95OZ1OVVRUBKzjdrv95/3YbDalp6cH1DQ2NqqioiLg3CAAAHB9C+qTnIKCAo0dO1Y9e/bU2bNntXbtWm3dulWbN2/W0aNHtXbtWo0bN07dunXTvn37NHv2bN1zzz0aOHCgJCkrK0tpaWmaPHmyli5dKo/Ho3nz5ik3N9f/CcuMGTP0/PPPa+7cuXrssce0ZcsWrV+/XqWlpf4+8vPzlZOTo6FDh2rYsGFauXKl6urqNGXKlFYcDQAAiGRBhZyTJ0/qBz/4gU6cOKFOnTpp4MCB2rx5s0aPHq2//vWvevfdd/2BIyUlRRMnTtS8efP8j4+JidHGjRs1c+ZMOZ1O3XDDDcrJydGiRYv8NampqSotLdXs2bO1atUq9ejRQy+//LJcrv9/vsikSZP02WefqbCwUB6PR4MHD1ZZWVmzk5EBADDFTU+WXr3oX4g9xtLSYW3bQ1Ah55VXXrnsvpSUFG3btu2qa/Tq1UubNm26Ys2IESO0d+/eK9bk5eUpLy/vqs8HAACuT/ztKgAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIQYWcF198UQMHDpTD4ZDD4ZDT6dQf/vAH//7z588rNzdX3bp1U4cOHTRx4kTV1NQErHH8+HFlZ2crPj5eCQkJmjNnji5cuBBQs3XrVt1+++2y2+3q06ePSkpKmvVSVFSkm266SXFxccrIyNDu3buDeSkAAMBwQYWcHj166JlnnlFVVZX27NmjUaNG6b777tPBgwclSbNnz9bvf/97bdiwQdu2bdOnn36q+++/3//4hoYGZWdnq76+Xjt37tRrr72mkpISFRYW+muOHTum7OxsjRw5UtXV1Zo1a5Yef/xxbd682V+zbt065efna/78+frwww81aNAguVwunTx58uvOAwAAGCKokDN+/HiNGzdO3/zmN3XLLbfo6aefVocOHbRr1y6dOXNGr7zyipYvX65Ro0YpPT1dr776qnbu3Kldu3ZJksrLy3Xo0CG9/vrrGjx4sMaOHavFixerqKhI9fX1kqTi4mKlpqZq2bJl6tevn/Ly8vS9731PK1as8PexfPlyTZs2TVOmTFFaWpqKi4sVHx+v1atXt+JoAABAJGvX0gc2NDRow4YNqqurk9PpVFVVlXw+nzIzM/01ffv2Vc+ePVVZWanhw4ersrJSAwYMUGJior/G5XJp5syZOnjwoIYMGaLKysqANZpqZs2aJUmqr69XVVWVCgoK/Pujo6OVmZmpysrKK/bs9Xrl9Xr992trayVJPp9PPp+vpaNopmkte7TVamuGQ2vOIBya+o20viMRsw4P5hwekTpne0xkvac0vQeGYs7XumbQIWf//v1yOp06f/68OnTooLfffltpaWmqrq6WzWZT586dA+oTExPl8XgkSR6PJyDgNO1v2nelmtraWn311Vc6deqUGhoaLllz+PDhK/a+ZMkSLVy4sNn28vJyxcfHX/3FB2nx0MZWXzOUNm3a1NYttIjb7W7rFq4bzDo8mHN4RNqclw5r6w5aJhRzPnfu3DXVBR1ybr31VlVXV+vMmTN68803lZOTo23btgXdYFsoKChQfn6+/35tba1SUlKUlZUlh8PRas/j8/nkdrv11J5oeRujWm3dUDuwwNXWLQSlac6jR49WbGxsW7djNGYdHsw5PCJ1zv0XbL560b8Qe7SlxUMbQzLnpiMxVxN0yLHZbOrTp48kKT09XR988IFWrVqlSZMmqb6+XqdPnw74NKempkZJSUmSpKSkpGZXQTVdfXVxzT9fkVVTUyOHw6H27dsrJiZGMTExl6xpWuNy7Ha77HZ7s+2xsbEh+Ub3NkbJ2xA5ISeSftgvFqqvH5pj1uHBnMMj0uYcSe8nFwvFnK91va/9e3IaGxvl9XqVnp6u2NhYVVRU+PcdOXJEx48fl9PplCQ5nU7t378/4Coot9sth8OhtLQ0f83FazTVNK1hs9mUnp4eUNPY2KiKigp/DQAAQFCf5BQUFGjs2LHq2bOnzp49q7Vr12rr1q3avHmzOnXqpKlTpyo/P19du3aVw+HQj370IzmdTg0fPlySlJWVpbS0NE2ePFlLly6Vx+PRvHnzlJub6/+EZcaMGXr++ec1d+5cPfbYY9qyZYvWr1+v0tJSfx/5+fnKycnR0KFDNWzYMK1cuVJ1dXWaMmVKK44GAABEsqBCzsmTJ/WDH/xAJ06cUKdOnTRw4EBt3rxZo0ePliStWLFC0dHRmjhxorxer1wul1544QX/42NiYrRx40bNnDlTTqdTN9xwg3JycrRo0SJ/TWpqqkpLSzV79mytWrVKPXr00MsvvyyX6/+fLzJp0iR99tlnKiwslMfj0eDBg1VWVtbsZGQAAHD9CirkvPLKK1fcHxcXp6KiIhUVFV22plevXle9imfEiBHau3fvFWvy8vKUl5d3xRoAAHD94m9XAQAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIwUVMhZsmSJvvWtb6ljx45KSEjQhAkTdOTIkYCaESNGKCoqKuA2Y8aMgJrjx48rOztb8fHxSkhI0Jw5c3ThwoWAmq1bt+r222+X3W5Xnz59VFJS0qyfoqIi3XTTTYqLi1NGRoZ2794dzMsBAAAGCyrkbNu2Tbm5udq1a5fcbrd8Pp+ysrJUV1cXUDdt2jSdOHHCf1u6dKl/X0NDg7Kzs1VfX6+dO3fqtddeU0lJiQoLC/01x44dU3Z2tkaOHKnq6mrNmjVLjz/+uDZv3uyvWbdunfLz8zV//nx9+OGHGjRokFwul06ePNnSWQAAAIO0C6a4rKws4H5JSYkSEhJUVVWle+65x789Pj5eSUlJl1yjvLxchw4d0rvvvqvExEQNHjxYixcv1hNPPKEFCxbIZrOpuLhYqampWrZsmSSpX79+2rFjh1asWCGXyyVJWr58uaZNm6YpU6ZIkoqLi1VaWqrVq1frySefDOZlAQAAAwUVcv7ZmTNnJEldu3YN2L5mzRq9/vrrSkpK0vjx4/XUU08pPj5eklRZWakBAwYoMTHRX+9yuTRz5kwdPHhQQ4YMUWVlpTIzMwPWdLlcmjVrliSpvr5eVVVVKigo8O+Pjo5WZmamKisrL9uv1+uV1+v136+trZUk+Xw++Xy+Fkzg0prWskdbrbZmOLTmDMKhqd9I6zsSMevwYM7hEalztsdE1ntK03tgKOZ8rWu2OOQ0NjZq1qxZuvPOO9W/f3//9ocffli9evVScnKy9u3bpyeeeEJHjhzRW2+9JUnyeDwBAUeS/77H47liTW1trb766iudOnVKDQ0Nl6w5fPjwZXtesmSJFi5c2Gx7eXm5P4S1psVDG1t9zVDatGlTW7fQIm63u61buG4w6/BgzuERaXNeOqytO2iZUMz53Llz11TX4pCTm5urAwcOaMeOHQHbp0+f7v/3gAED1L17d9177706evSoevfu3dKnaxUFBQXKz8/336+trVVKSoqysrLkcDha7Xl8Pp/cbree2hMtb2NUq60bagcWuNq6haA0zXn06NGKjY1t63aMxqzDgzmHR6TOuf+CzVcv+hdij7a0eGhjSObcdCTmaloUcvLy8rRx40Zt375dPXr0uGJtRkaGJOnjjz9W7969lZSU1OwqqJqaGknyn8eTlJTk33ZxjcPhUPv27RUTE6OYmJhL1lzuXCBJstvtstvtzbbHxsaG5Bvd2xglb0PkhJxI+mG/WKi+fmiOWYcHcw6PSJtzJL2fXCwUc77W9YK6usqyLOXl5entt9/Wli1blJqaetXHVFdXS5K6d+8uSXI6ndq/f3/AVVBut1sOh0NpaWn+moqKioB13G63nE6nJMlmsyk9PT2gprGxURUVFf4aAABwfQvqk5zc3FytXbtWv/3tb9WxY0f/OTSdOnVS+/btdfToUa1du1bjxo1Tt27dtG/fPs2ePVv33HOPBg4cKEnKyspSWlqaJk+erKVLl8rj8WjevHnKzc31f8oyY8YMPf/885o7d64ee+wxbdmyRevXr1dpaam/l/z8fOXk5Gjo0KEaNmyYVq5cqbq6Ov/VVgAA4PoWVMh58cUXJf3jF/5d7NVXX9Wjjz4qm82md9991x84UlJSNHHiRM2bN89fGxMTo40bN2rmzJlyOp264YYblJOTo0WLFvlrUlNTVVpaqtmzZ2vVqlXq0aOHXn75Zf/l45I0adIkffbZZyosLJTH49HgwYNVVlbW7GRkAABwfQoq5FjWlS9fS0lJ0bZt2666Tq9eva56Jc+IESO0d+/eK9bk5eUpLy/vqs8HAACuP/ztKgAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRggo5S5Ys0be+9S117NhRCQkJmjBhgo4cORJQc/78eeXm5qpbt27q0KGDJk6cqJqamoCa48ePKzs7W/Hx8UpISNCcOXN04cKFgJqtW7fq9ttvl91uV58+fVRSUtKsn6KiIt10002Ki4tTRkaGdu/eHczLAQAABgsq5Gzbtk25ubnatWuX3G63fD6fsrKyVFdX56+ZPXu2fv/732vDhg3atm2bPv30U91///3+/Q0NDcrOzlZ9fb127typ1157TSUlJSosLPTXHDt2TNnZ2Ro5cqSqq6s1a9YsPf7449q8ebO/Zt26dcrPz9f8+fP14YcfatCgQXK5XDp58uTXmQcAADBEu2CKy8rKAu6XlJQoISFBVVVVuueee3TmzBm98sorWrt2rUaNGiVJevXVV9WvXz/t2rVLw4cPV3l5uQ4dOqR3331XiYmJGjx4sBYvXqwnnnhCCxYskM1mU3FxsVJTU7Vs2TJJUr9+/bRjxw6tWLFCLpdLkrR8+XJNmzZNU6ZMkSQVFxertLRUq1ev1pNPPvm1BwMAACLb1zon58yZM5Kkrl27SpKqqqrk8/mUmZnpr+nbt6969uypyspKSVJlZaUGDBigxMREf43L5VJtba0OHjzor7l4jaaapjXq6+tVVVUVUBMdHa3MzEx/DQAAuL4F9UnOxRobGzVr1izdeeed6t+/vyTJ4/HIZrOpc+fOAbWJiYnyeDz+mosDTtP+pn1XqqmtrdVXX32lU6dOqaGh4ZI1hw8fvmzPXq9XXq/Xf7+2tlaS5PP55PP5rvWlX1XTWvZoq9XWDIfWnEE4NPUbaX1HImYdHsw5PCJ1zvaYyHpPaXoPDMWcr3XNFoec3NxcHThwQDt27GjpEmG3ZMkSLVy4sNn28vJyxcfHt/rzLR7a2OprhtKmTZvauoUWcbvdbd3CdYNZhwdzDo9Im/PSYW3dQcuEYs7nzp27proWhZy8vDxt3LhR27dvV48ePfzbk5KSVF9fr9OnTwd8mlNTU6OkpCR/zT9fBdV09dXFNf98RVZNTY0cDofat2+vmJgYxcTEXLKmaY1LKSgoUH5+vv9+bW2tUlJSlJWVJYfDEcQErszn88ntduupPdHyNka12rqhdmCBq61bCErTnEePHq3Y2Ni2bsdozDo8mHN4ROqc+y/YfPWifyH2aEuLhzaGZM5NR2KuJqiQY1mWfvSjH+ntt9/W1q1blZqaGrA/PT1dsbGxqqio0MSJEyVJR44c0fHjx+V0OiVJTqdTTz/9tE6ePKmEhARJ/0h5DodDaWlp/pp//lTB7Xb717DZbEpPT1dFRYUmTJgg6R+HzyoqKpSXl3fZ/u12u+x2e7PtsbGxIflG9zZGydsQOSEnkn7YLxaqrx+aY9bhwZzDI9LmHEnvJxcLxZyvdb2gQk5ubq7Wrl2r3/72t+rYsaP/HJpOnTqpffv26tSpk6ZOnar8/Hx17dpVDodDP/rRj+R0OjV8+HBJUlZWltLS0jR58mQtXbpUHo9H8+bNU25urj+AzJgxQ88//7zmzp2rxx57TFu2bNH69etVWlrq7yU/P185OTkaOnSohg0bppUrV6qurs5/tRUAALi+BRVyXnzxRUnSiBEjAra/+uqrevTRRyVJK1asUHR0tCZOnCiv1yuXy6UXXnjBXxsTE6ONGzdq5syZcjqduuGGG5STk6NFixb5a1JTU1VaWqrZs2dr1apV6tGjh15++WX/5eOSNGnSJH322WcqLCyUx+PR4MGDVVZW1uxkZAAAcH0K+nDV1cTFxamoqEhFRUWXrenVq9dVT3IdMWKE9u7de8WavLy8Kx6eAgAA1y/+dhUAADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgpKBDzvbt2zV+/HglJycrKipK77zzTsD+Rx99VFFRUQG3MWPGBNR88cUXeuSRR+RwONS5c2dNnTpVX375ZUDNvn37dPfddysuLk4pKSlaunRps142bNigvn37Ki4uTgMGDNCmTZuCfTkAAMBQQYecuro6DRo0SEVFRZetGTNmjE6cOOG//eY3vwnY/8gjj+jgwYNyu93auHGjtm/frunTp/v319bWKisrS7169VJVVZV++ctfasGCBXrppZf8NTt37tRDDz2kqVOnau/evZowYYImTJigAwcOBPuSAACAgdoF+4CxY8dq7NixV6yx2+1KSkq65L6PPvpIZWVl+uCDDzR06FBJ0nPPPadx48bp2WefVXJystasWaP6+nqtXr1aNptNt912m6qrq7V8+XJ/GFq1apXGjBmjOXPmSJIWL14st9ut559/XsXFxcG+LAAAYJigQ8612Lp1qxISEtSlSxeNGjVKP/vZz9StWzdJUmVlpTp37uwPOJKUmZmp6Ohovf/++/rud7+ryspK3XPPPbLZbP4al8ulX/ziFzp16pS6dOmiyspK5efnBzyvy+VqdvjsYl6vV16v13+/trZWkuTz+eTz+VrjpfvXkyR7tNVqa4ZDa84gHJr6jbS+IxGzDg/mHB6ROmd7TGS9pzS9B4Zizte6ZquHnDFjxuj+++9Xamqqjh49qp/85CcaO3asKisrFRMTI4/Ho4SEhMAm2rVT165d5fF4JEkej0epqakBNYmJif59Xbp0kcfj8W+7uKZpjUtZsmSJFi5c2Gx7eXm54uPjW/R6r2Tx0MZWXzOUIvWcJrfb3dYtXDeYdXgw5/CItDkvHdbWHbRMKOZ87ty5a6pr9ZDz4IMP+v89YMAADRw4UL1799bWrVt17733tvbTBaWgoCDg05/a2lqlpKQoKytLDoej1Z7H5/PJ7XbrqT3R8jZGtdq6oXZggautWwhK05xHjx6t2NjYtm7HaMw6PJhzeETqnPsv2NzWLQTFHm1p8dDGkMy56UjM1YTkcNXFbr75Zt144436+OOPde+99yopKUknT54MqLlw4YK++OIL/3k8SUlJqqmpCahpun+1msudCyT941whu93ebHtsbGxIvtG9jVHyNkROyImkH/aLherrh+aYdXgw5/CItDlH0vvJxUIx52tdL+S/J+dvf/ubPv/8c3Xv3l2S5HQ6dfr0aVVVVflrtmzZosbGRmVkZPhrtm/fHnDMze1269Zbb1WXLl38NRUVFQHP5Xa75XQ6Q/2SAABABAg65Hz55Zeqrq5WdXW1JOnYsWOqrq7W8ePH9eWXX2rOnDnatWuX/vKXv6iiokL33Xef+vTpI5frH4dC+vXrpzFjxmjatGnavXu33nvvPeXl5enBBx9UcnKyJOnhhx+WzWbT1KlTdfDgQa1bt06rVq0KONT04x//WGVlZVq2bJkOHz6sBQsWaM+ePcrLy2uFsQAAgEgXdMjZs2ePhgwZoiFDhkiS8vPzNWTIEBUWFiomJkb79u3Td77zHd1yyy2aOnWq0tPT9ac//SngMNGaNWvUt29f3XvvvRo3bpzuuuuugN+B06lTJ5WXl+vYsWNKT0/Xf/zHf6iwsDDgd+nccccdWrt2rV566SUNGjRIb775pt555x3179//68wDAAAYIuhzckaMGCHLuvxlbJs3X/3EqK5du2rt2rVXrBk4cKD+9Kc/XbHmgQce0AMPPHDV5wMAANcf/nYVAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEhBh5zt27dr/PjxSk5OVlRUlN55552A/ZZlqbCwUN27d1f79u2VmZmpP//5zwE1X3zxhR555BE5HA517txZU6dO1ZdffhlQs2/fPt19992Ki4tTSkqKli5d2qyXDRs2qG/fvoqLi9OAAQO0adOmYF8OAAAwVNAhp66uToMGDVJRUdEl9y9dulS/+tWvVFxcrPfff1833HCDXC6Xzp8/76955JFHdPDgQbndbm3cuFHbt2/X9OnT/ftra2uVlZWlXr16qaqqSr/85S+1YMECvfTSS/6anTt36qGHHtLUqVO1d+9eTZgwQRMmTNCBAweCfUkAAMBA7YJ9wNixYzV27NhL7rMsSytXrtS8efN03333SZL++7//W4mJiXrnnXf04IMP6qOPPlJZWZk++OADDR06VJL03HPPady4cXr22WeVnJysNWvWqL6+XqtXr5bNZtNtt92m6upqLV++3B+GVq1apTFjxmjOnDmSpMWLF8vtduv5559XcXFxi4YBAADMEXTIuZJjx47J4/EoMzPTv61Tp07KyMhQZWWlHnzwQVVWVqpz587+gCNJmZmZio6O1vvvv6/vfve7qqys1D333CObzeavcblc+sUvfqFTp06pS5cuqqysVH5+fsDzu1yuZofPLub1euX1ev33a2trJUk+n08+n+/rvny/prXs0VarrRkOrTmDcGjqN9L6jkTMOjyYc3hE6pztMZH1ntL0HhiKOV/rmq0acjwejyQpMTExYHtiYqJ/n8fjUUJCQmAT7dqpa9euATWpqanN1mja16VLF3k8nis+z6UsWbJECxcubLa9vLxc8fHx1/ISg7J4aGOrrxlKkXpOk9vtbusWrhvMOjyYc3hE2pyXDmvrDlomFHM+d+7cNdW1asj5V1dQUBDw6U9tba1SUlKUlZUlh8PRas/j8/nkdrv11J5oeRujWm3dUDuwwNXWLQSlac6jR49WbGxsW7djNGYdHsw5PCJ1zv0XbG7rFoJij7a0eGhjSObcdCTmalo15CQlJUmSampq1L17d//2mpoaDR482F9z8uTJgMdduHBBX3zxhf/xSUlJqqmpCahpun+1mqb9l2K322W325ttj42NDck3urcxSt6GyAk5kfTDfrFQff3QHLMOD+YcHpE250h6P7lYKOZ8reu16u/JSU1NVVJSkioqKvzbamtr9f7778vpdEqSnE6nTp8+raqqKn/Nli1b1NjYqIyMDH/N9u3bA465ud1u3XrrrerSpYu/5uLnaappeh4AAHB9CzrkfPnll6qurlZ1dbWkf5xsXF1drePHjysqKkqzZs3Sz372M/3ud7/T/v379YMf/EDJycmaMGGCJKlfv34aM2aMpk2bpt27d+u9995TXl6eHnzwQSUnJ0uSHn74YdlsNk2dOlUHDx7UunXrtGrVqoBDTT/+8Y9VVlamZcuW6fDhw1qwYIH27NmjvLy8rz8VAAAQ8YI+XLVnzx6NHDnSf78peOTk5KikpERz585VXV2dpk+frtOnT+uuu+5SWVmZ4uLi/I9Zs2aN8vLydO+99yo6OloTJ07Ur371K//+Tp06qby8XLm5uUpPT9eNN96owsLCgN+lc8cdd2jt2rWaN2+efvKTn+ib3/ym3nnnHfXv379FgwAAAGYJOuSMGDFClnX5y9iioqK0aNEiLVq06LI1Xbt21dq1a6/4PAMHDtSf/vSnK9Y88MADeuCBB67cMAAAuC7xt6sAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAI7V6yFmwYIGioqICbn379vXvP3/+vHJzc9WtWzd16NBBEydOVE1NTcAax48fV3Z2tuLj45WQkKA5c+bowoULATVbt27V7bffLrvdrj59+qikpKS1XwoAAIhgIfkk57bbbtOJEyf8tx07dvj3zZ49W7///e+1YcMGbdu2TZ9++qnuv/9+//6GhgZlZ2ervr5eO3fu1GuvvaaSkhIVFhb6a44dO6bs7GyNHDlS1dXVmjVrlh5//HFt3rw5FC8HAABEoHYhWbRdOyUlJTXbfubMGb3yyitau3atRo0aJUl69dVX1a9fP+3atUvDhw9XeXm5Dh06pHfffVeJiYkaPHiwFi9erCeeeEILFiyQzWZTcXGxUlNTtWzZMklSv379tGPHDq1YsUIulysULwkAAESYkIScP//5z0pOTlZcXJycTqeWLFminj17qqqqSj6fT5mZmf7avn37qmfPnqqsrNTw4cNVWVmpAQMGKDEx0V/jcrk0c+ZMHTx4UEOGDFFlZWXAGk01s2bNumJfXq9XXq/Xf7+2tlaS5PP55PP5WuGVy7+eJNmjrVZbMxxacwbh0NRvpPUdiZh1eDDn8IjUOdtjIus9pek9MBRzvtY1Wz3kZGRkqKSkRLfeeqtOnDihhQsX6u6779aBAwfk8Xhks9nUuXPngMckJibK4/FIkjweT0DAadrftO9KNbW1tfrqq6/Uvn37S/a2ZMkSLVy4sNn28vJyxcfHt+j1XsnioY2tvmYobdq0qa1baBG3293WLVw3mHV4MOfwiLQ5Lx3W1h20TCjmfO7cuWuqa/WQM3bsWP+/Bw4cqIyMDPXq1Uvr16+/bPgIl4KCAuXn5/vv19bWKiUlRVlZWXI4HK32PD6fT263W0/tiZa3MarV1g21Awsi61Bf05xHjx6t2NjYtm7HaMw6PJhzeETqnPsviKzzTu3RlhYPbQzJnJuOxFxNSA5XXaxz58665ZZb9PHHH2v06NGqr6/X6dOnAz7Nqamp8Z/Dk5SUpN27dwes0XT11cU1/3xFVk1NjRwOxxWDlN1ul91ub7Y9NjY2JN/o3sYoeRsiJ+RE0g/7xUL19UNzzDo8mHN4RNqcI+n95GKhmPO1rhfy35Pz5Zdf6ujRo+revbvS09MVGxuriooK//4jR47o+PHjcjqdkiSn06n9+/fr5MmT/hq32y2Hw6G0tDR/zcVrNNU0rQEAANDqIec///M/tW3bNv3lL3/Rzp079d3vflcxMTF66KGH1KlTJ02dOlX5+fn64x//qKqqKk2ZMkVOp1PDhw+XJGVlZSktLU2TJ0/W//zP/2jz5s2aN2+ecnNz/Z/CzJgxQ5988onmzp2rw4cP64UXXtD69es1e/bs1n45AAAgQrX64aq//e1veuihh/T555/rG9/4hu666y7t2rVL3/jGNyRJK1asUHR0tCZOnCiv1yuXy6UXXnjB//iYmBht3LhRM2fOlNPp1A033KCcnBwtWrTIX5OamqrS0lLNnj1bq1atUo8ePfTyyy9z+TgAAPBr9ZDzxhtvXHF/XFycioqKVFRUdNmaXr16XfVKnxEjRmjv3r0t6hEAAJiPv10FAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMBIhBwAAGImQAwAAjETIAQAARiLkAAAAIxFyAACAkQg5AADASIQcAABgJEIOAAAwEiEHAAAYiZADAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJEIOQAAwEiEHAAAYCRCDgAAMFLEh5yioiLddNNNiouLU0ZGhnbv3t3WLQEAgH8BER1y1q1bp/z8fM2fP18ffvihBg0aJJfLpZMnT7Z1awAAoI1FdMhZvny5pk2bpilTpigtLU3FxcWKj4/X6tWr27o1AADQxtq1dQMtVV9fr6qqKhUUFPi3RUdHKzMzU5WVlZd8jNfrldfr9d8/c+aMJOmLL76Qz+drtd58Pp/OnTundr5oNTRGtdq6ofb555+3dQtBaZrz559/rtjY2LZux2jMOjyYc3hE6pzbXahr6xaC0q7R0rlzjSGZ89mzZyVJlmVduYdWfdYw+vvf/66GhgYlJiYGbE9MTNThw4cv+ZglS5Zo4cKFzbanpqaGpMdIc+Oytu4AAGCSh0O8/tmzZ9WpU6fL7o/YkNMSBQUFys/P999vbGzUF198oW7duikqqvU+camtrVVKSor++te/yuFwtNq6CMScw4dZhwdzDg/mHB6hnLNlWTp79qySk5OvWBexIefGG29UTEyMampqArbX1NQoKSnpko+x2+2y2+0B2zp37hyqFuVwOPgBCgPmHD7MOjyYc3gw5/AI1Zyv9AlOk4g98dhmsyk9PV0VFRX+bY2NjaqoqJDT6WzDzgAAwL+CiP0kR5Ly8/OVk5OjoUOHatiwYVq5cqXq6uo0ZcqUtm4NAAC0sYgOOZMmTdJnn32mwsJCeTweDR48WGVlZc1ORg43u92u+fPnNzs0htbFnMOHWYcHcw4P5hwe/wpzjrKudv0VAABABIrYc3IAAACuhJADAACMRMgBAABGIuQAAAAjEXJaqKioSDfddJPi4uKUkZGh3bt3X7F+w4YN6tu3r+Li4jRgwABt2rQpTJ1GtmDm/Otf/1p33323unTpoi5duigzM/OqXxf8Q7Dfz03eeOMNRUVFacKECaFt0CDBzvr06dPKzc1V9+7dZbfbdcstt/D/j2sQ7JxXrlypW2+9Ve3bt1dKSopmz56t8+fPh6nbyLR9+3aNHz9eycnJioqK0jvvvHPVx2zdulW333677Ha7+vTpo5KSktA2aSFob7zxhmWz2azVq1dbBw8etKZNm2Z17tzZqqmpuWT9e++9Z8XExFhLly61Dh06ZM2bN8+KjY219u/fH+bOI0uwc3744YetoqIia+/evdZHH31kPfroo1anTp2sv/3tb2HuPLIEO+cmx44ds/7t3/7Nuvvuu6377rsvPM1GuGBn7fV6raFDh1rjxo2zduzYYR07dszaunWrVV1dHebOI0uwc16zZo1lt9utNWvWWMeOHbM2b95sde/e3Zo9e3aYO48smzZtsn76059ab731liXJevvtt69Y/8knn1jx8fFWfn6+dejQIeu5556zYmJirLKyspD1SMhpgWHDhlm5ubn++w0NDVZycrK1ZMmSS9Z///vft7KzswO2ZWRkWP/+7/8e0j4jXbBz/mcXLlywOnbsaL322muhatEILZnzhQsXrDvuuMN6+eWXrZycHELONQp21i+++KJ18803W/X19eFq0QjBzjk3N9caNWpUwLb8/HzrzjvvDGmfJrmWkDN37lzrtttuC9g2adIky+VyhawvDlcFqb6+XlVVVcrMzPRvi46OVmZmpiorKy/5mMrKyoB6SXK5XJetR8vm/M/OnTsnn8+nrl27hqrNiNfSOS9atEgJCQmaOnVqONo0Qktm/bvf/U5Op1O5ublKTExU//799fOf/1wNDQ3hajvitGTOd9xxh6qqqvyHtD755BNt2rRJ48aNC0vP14u2eC+M6N943Bb+/ve/q6GhodlvVU5MTNThw4cv+RiPx3PJeo/HE7I+I11L5vzPnnjiCSUnJzf7ocL/15I579ixQ6+88oqqq6vD0KE5WjLrTz75RFu2bNEjjzyiTZs26eOPP9YPf/hD+Xw+zZ8/PxxtR5yWzPnhhx/W3//+d911112yLEsXLlzQjBkz9JOf/CQcLV83LvdeWFtbq6+++krt27dv9efkkxwY6ZlnntEbb7yht99+W3FxcW3djjHOnj2ryZMn69e//rVuvPHGtm7HeI2NjUpISNBLL72k9PR0TZo0ST/96U9VXFzc1q0ZZevWrfr5z3+uF154QR9++KHeeustlZaWavHixW3dGr4mPskJ0o033qiYmBjV1NQEbK+pqVFSUtIlH5OUlBRUPVo25ybPPvusnnnmGb377rsaOHBgKNuMeMHO+ejRo/rLX/6i8ePH+7c1NjZKktq1a6cjR46od+/eoW06QrXke7p79+6KjY1VTEyMf1u/fv3k8XhUX18vm80W0p4jUUvm/NRTT2ny5Ml6/PHHJUkDBgxQXV2dpk+frp/+9KeKjubzgNZwufdCh8MRkk9xJD7JCZrNZlN6eroqKir82xobG1VRUSGn03nJxzidzoB6SXK73ZetR8vmLElLly7V4sWLVVZWpqFDh4aj1YgW7Jz79u2r/fv3q7q62n/7zne+o5EjR6q6ulopKSnhbD+itOR7+s4779THH3/sD5KS9L//+7/q3r07AecyWjLnc+fONQsyTcHS4s87tpo2eS8M2SnNBnvjjTcsu91ulZSUWIcOHbKmT59ude7c2fJ4PJZlWdbkyZOtJ5980l//3nvvWe3atbOeffZZ66OPPrLmz5/PJeTXINg5P/PMM5bNZrPefPNN68SJE/7b2bNn2+olRIRg5/zPuLrq2gU76+PHj1sdO3a08vLyrCNHjlgbN260EhISrJ/97Gdt9RIiQrBznj9/vtWxY0frN7/5jfXJJ59Y5eXlVu/eva3vf//7bfUSIsLZs2etvXv3Wnv37rUkWcuXL7f27t1r/d///Z9lWZb15JNPWpMnT/bXN11CPmfOHOujjz6yioqKuIT8X9Vzzz1n9ezZ07LZbNawYcOsXbt2+fd9+9vftnJycgLq169fb91yyy2WzWazbrvtNqu0tDTMHUemYObcq1cvS1Kz2/z588PfeIQJ9vv5YoSc4AQ76507d1oZGRmW3W63br75Zuvpp5+2Lly4EOauI08wc/b5fNaCBQus3r17W3FxcVZKSor1wx/+0Dp16lT4G48gf/zjHy/5/9ym2ebk5Fjf/va3mz1m8ODBls1ms26++Wbr1VdfDWmPUZbFZ3EAAMA8nJMDAACMRMgBAABGIuQAAAAjEXIAAICRCDkAAMBIhBwAAGAkQg4AADASIQcAABiJkAMAAIxEyAEAAEYi5AAAACMRcgAAgJH+H8KMZhPYyaO1AAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Data preprocessing**"
      ],
      "metadata": {
        "id": "dzGuQNwQwnnC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "normalizer = hazm.Normalizer()"
      ],
      "metadata": {
        "id": "CTOAEbNmkN8d"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data['normalized_comment'] = data['comment'].apply(lambda c: normalizer.normalize(c.strip()))"
      ],
      "metadata": {
        "id": "7lntEW64kN-s"
      },
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data.iloc[1]['comment']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "0F-iE9_-kOA5",
        "outputId": "a337a945-3962-4fe9-de12-769d8487c349"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'قرار بود ۱ ساعته برسه ولی نیم ساعت زودتر از موقع رسید، شما ببین چقدرررررررررررر پلاک خفنههههه، من سالهاست مشتریشونم و سالهاست مزه بهشت میده غذاشون'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data.iloc[1]['normalized_comment']"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "8kmYUk5Qkwl2",
        "outputId": "cf412dec-e423-41f8-e885-b9122b8c801a"
      },
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'قرار بود ۱ ساعته برسه ولی نیم ساعت زودتر از موقع رسید، شما ببین چقدر پلاک خفنهه، من سالهاست مشتریشونم و سالهاست مزه بهشت میده غذاشون'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Train ParsBert model**"
      ],
      "metadata": {
        "id": "LDnwkVx6xHoO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "checkpointt = \"HooshvareLab/bert-base-parsbert-uncased\"\n",
        "config   = AutoConfig   .from_pretrained(checkpointt)\n",
        "tokenizer = AutoTokenizer.from_pretrained(checkpointt, truncation=True, padding=True, tokenization=\"byte_level\")\n",
        "model    = AutoModelForSequenceClassification.from_pretrained(checkpointt, num_labels=2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0IebdZ-wRyI4",
        "outputId": "334bb5bd-8cb8-4f51-d5a6-7eccdc7119f1"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at HooshvareLab/bert-base-parsbert-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "### split data to train (90%) and valid (10%)\n",
        "\n",
        "train_data = data.iloc[:7000]\n",
        "val_data = data.iloc[7000:]"
      ],
      "metadata": {
        "id": "d04GwByRRyLC"
      },
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataset = Dataset.from_pandas(train_data)\n",
        "valid_dataset = Dataset.from_pandas(val_data)\n",
        "# test_dataset  = Dataset.from_pandas(test_data)\n",
        "\n",
        "def preprocess_function(examples):\n",
        "    return tokenizer(examples[\"normalized_comment\"], truncation=True, padding=True)\n",
        "\n",
        "train_dataset = train_dataset.map(preprocess_function)\n",
        "valid_dataset = valid_dataset.map(preprocess_function)\n",
        "# test_dataset  = test_dataset .map(preprocess_function, batched=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 98,
          "referenced_widgets": [
            "b78908d1186f44dbb7d663e0bcd37769",
            "c731c744daed4a9aac52936ad8772b3a",
            "0ef6a99cbe8b46e88c24cfca437ab443",
            "6a4bf9fc7aa54c5ba2128965d02f9135",
            "61084fe2d5bd4851ae77a9a0f38ca3cf",
            "e0171cc137834d6bbd2013a6abcd1c8e",
            "597c81db94694bd987d62d7f1fba54ae",
            "22a0d1b8076246c491123c801445e3d3",
            "c9622a62881240ab976e248ed8c73673",
            "5c8b58bd471b4f6eb4593299e58ec5a9",
            "bc1d2db5acdc4d21a70227fe03441696",
            "b55fb0ac49f146f48068a3e51855b65d",
            "2195c097919e439fa9ed612410336b67",
            "6e8ced64c9d6406a8c0dad7b4cbb462a",
            "6982a2b8d0ff4ca7afc5c8ad887577c6",
            "ff4f4c876b784d37b415388c8c91dea9",
            "f73cc9a23c4a4e91b97b0b63a002bcb7",
            "bd847732ee8f49879aace37bc9e17b51",
            "7a0d124331ff43da9c9a7dd285a010d0",
            "5aa2a64a90444194a24137c07ae711bd",
            "66824aed874d472485cf7e0f940adb42",
            "92990f869eff48e99d19a591a3c27214"
          ]
        },
        "id": "7AwuDctKRyNE",
        "outputId": "923510a1-85cb-43c9-d70f-9d0a66cc6fc9"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/7000 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b78908d1186f44dbb7d663e0bcd37769"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/62480 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b55fb0ac49f146f48068a3e51855b65d"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def compute_metrics(eval_pred):\n",
        "    logits, labels = eval_pred\n",
        "    predictions = np.argmax(logits, axis=-1)\n",
        "    metric = load_metric(\"accuracy\")\n",
        "    return metric.compute(predictions=predictions, references=labels)"
      ],
      "metadata": {
        "id": "5HjsxC52R1k8"
      },
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import DataCollatorWithPadding\n",
        "\n",
        "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
      ],
      "metadata": {
        "id": "O5YS80FSR1nF"
      },
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import TrainerCallback, Trainer\n",
        "import os\n",
        "\n",
        "class SaveModelCallback(TrainerCallback):\n",
        "    def __init__(self, output_dir, save_every_n_epochs):\n",
        "        self.output_dir = output_dir\n",
        "        self.save_every_n_epochs = save_every_n_epochs\n",
        "\n",
        "    def on_epoch_end(self, args, state, control, model, **kwargs):\n",
        "        if (state.epoch + 1) % self.save_every_n_epochs == 0:\n",
        "            model.save_pretrained(os.path.join(self.output_dir, f\"checkpoint-{state.epoch + 1}\"))\n",
        "\n",
        "# Instantiate the SaveModelCallback\n",
        "save_callback = SaveModelCallback(output_dir=\"./result\", save_every_n_epochs=10)"
      ],
      "metadata": {
        "id": "3GpW486cR1o1"
      },
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "training_args = TrainingArguments(\n",
        "    output_dir                  = \"./result_parts_bert1\",\n",
        "    per_device_train_batch_size = 32,\n",
        "    per_device_eval_batch_size  = 16,\n",
        "    num_train_epochs            = 3,\n",
        "\n",
        "    gradient_accumulation_steps = 1,\n",
        "    overwrite_output_dir        = True,\n",
        "    save_strategy               = \"epoch\",\n",
        "    evaluation_strategy         = \"epoch\",\n",
        "    load_best_model_at_end      = True,\n",
        "    metric_for_best_model       = \"accuracy\",\n",
        "    # report_to=\"wandb\",\n",
        "\n",
        ")\n",
        "\n",
        "\n",
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=train_dataset,\n",
        "    eval_dataset=valid_dataset,\n",
        "    tokenizer=tokenizer,\n",
        "    data_collator=data_collator,\n",
        "    compute_metrics=compute_metrics,\n",
        "    callbacks=[save_callback]\n",
        ")\n",
        "trainer.train()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "Gcejk7UhRyPK",
        "outputId": "b02faa36-3be3-45eb-f28a-238ce6802ffd"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='657' max='657' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [657/657 17:25, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>No log</td>\n",
              "      <td>0.331030</td>\n",
              "      <td>0.856066</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>No log</td>\n",
              "      <td>0.354147</td>\n",
              "      <td>0.858755</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.302300</td>\n",
              "      <td>0.465714</td>\n",
              "      <td>0.851985</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=657, training_loss=0.2626682147950705, metrics={'train_runtime': 1046.0305, 'train_samples_per_second': 20.076, 'train_steps_per_second': 0.628, 'total_flos': 841614155002080.0, 'train_loss': 0.2626682147950705, 'epoch': 3.0})"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pred = trainer.predict(valid_dataset)\n",
        "print( classification_report( pred.label_ids , pred.predictions.argmax(-1) , labels=[0, 1]) )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "id": "4aEnoWTuxo44",
        "outputId": "e1ee718e-eb71-4766-fe74-32cb01e25c71"
      },
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": []
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.89      0.82      0.85     31432\n",
            "           1       0.83      0.90      0.86     31048\n",
            "\n",
            "    accuracy                           0.86     62480\n",
            "   macro avg       0.86      0.86      0.86     62480\n",
            "weighted avg       0.86      0.86      0.86     62480\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Pruning Attention Heads & Encoding layers**"
      ],
      "metadata": {
        "id": "4v264h-vX-hz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "\n",
        "def deleteEncodingLayers(model, num_layers_to_keep):\n",
        "    oldModuleList = model.bert.encoder.layer\n",
        "    newModuleList = nn.ModuleList()\n",
        "\n",
        "    for i in range(0, num_layers_to_keep):\n",
        "        newModuleList.append(oldModuleList[i])\n",
        "\n",
        "    # create a copy of the model, modify it with the new list, and return\n",
        "    copyOfModel = copy.deepcopy(model)\n",
        "    copyOfModel.bert.encoder.layer = newModuleList\n",
        "\n",
        "    return copyOfModel\n",
        "\n",
        "\n",
        "def generate_dictionary_heads(num_values, num_chosen):\n",
        "    random.seed(5)\n",
        "    dictionary = {}\n",
        "    num_chosen = 12 - num_chosen\n",
        "    for key in range(0, num_values):\n",
        "        values = random.sample(range(12), num_chosen)\n",
        "        dictionary[key] = values\n",
        "\n",
        "    return dictionary\n",
        "\n",
        "def RemoveAttentionHeads(model, num_heads_to_keep, num_layers):\n",
        "\n",
        "  drop_heads = generate_dictionary_heads(num_layers,num_heads_to_keep)\n",
        "  copyModel = copy.deepcopy(model)\n",
        "  copyModel.prune_heads(drop_heads)\n",
        "\n",
        "  return copyModel\n",
        "\n",
        "def Pruner(model, num_layers_to_keep= 12, num_heads_to_keep= 12):\n",
        "   new_model = deleteEncodingLayers(model, num_layers_to_keep)\n",
        "   new_model = RemoveAttentionHeads(new_model, num_heads_to_keep, num_layers_to_keep)\n",
        "   return new_model"
      ],
      "metadata": {
        "id": "HgSwEj_KXp9T"
      },
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "new_model = Pruner(model, num_layers_to_keep= 11, num_heads_to_keep=10)"
      ],
      "metadata": {
        "id": "HE931R9bXp_i"
      },
      "execution_count": 56,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "training_args = TrainingArguments(\n",
        "    output_dir                  = \"./result_pruned_model\",\n",
        "    per_device_train_batch_size = 32,\n",
        "    per_device_eval_batch_size  = 16,\n",
        "    num_train_epochs            = 2,\n",
        "\n",
        "    gradient_accumulation_steps = 1,\n",
        "    overwrite_output_dir        = True,\n",
        "    save_strategy               = \"epoch\",\n",
        "    evaluation_strategy         = \"epoch\",\n",
        "    load_best_model_at_end      = True,\n",
        "    metric_for_best_model       = \"accuracy\"\n",
        ")\n",
        "\n",
        "trainer = Trainer(\n",
        "    model           = new_model,\n",
        "    args            = training_args,\n",
        "    train_dataset   = train_dataset,\n",
        "    eval_dataset    = valid_dataset,\n",
        "    tokenizer       = tokenizer,\n",
        "    data_collator=data_collator,\n",
        "    compute_metrics = compute_metrics,\n",
        ")\n",
        "\n",
        "trainer.train()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "id": "Trhk4eKCR4s_",
        "outputId": "12f50514-3fda-4dc1-deeb-748b1168193d"
      },
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='438' max='438' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [438/438 10:04, Epoch 2/2]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>No log</td>\n",
              "      <td>0.428383</td>\n",
              "      <td>0.841357</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>No log</td>\n",
              "      <td>0.581415</td>\n",
              "      <td>0.851104</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=438, training_loss=0.15461147430280572, metrics={'train_runtime': 604.6284, 'train_samples_per_second': 23.155, 'train_steps_per_second': 0.724, 'total_flos': 485639530756032.0, 'train_loss': 0.15461147430280572, 'epoch': 2.0})"
            ]
          },
          "metadata": {},
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Export model to onnxruntime**"
      ],
      "metadata": {
        "id": "VhGouCKy0J81"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Import & install Important Libraries"
      ],
      "metadata": {
        "id": "J4dPO6700UQo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install optimum\n",
        "!pip install onnxruntime\n",
        "!pip install onnx\n",
        "!pip install onnxruntime-extensions"
      ],
      "metadata": {
        "id": "O802MLMfR4uy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import transformers\n",
        "from pathlib import Path\n",
        "from onnxruntime.quantization import quantize_dynamic, QuantType\n",
        "from onnxruntime_extensions.tools import add_pre_post_processing_to_model as add_ppp\n",
        "from contextlib import contextmanager"
      ],
      "metadata": {
        "id": "c928lVSf0bK2"
      },
      "execution_count": 59,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Model Quantization"
      ],
      "metadata": {
        "id": "49MBTd-J0g9J"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def get_model(model_path: str = \"csarron/mobilebert-uncased-squad-v2\"):\n",
        "    \"\"\"\n",
        "    Step 1. Download the model and convert to onnx\n",
        "    \"\"\"\n",
        "    model = AutoModelForSequenceClassification.from_pretrained(model_path, num_labels= 2)\n",
        "    onnx_config = transformers.models.bert.BertOnnxConfig(config, \"sequence-classification\")\n",
        "\n",
        "    model_path = Path('/content/qmodel/parsbert.onnx')\n",
        "    onnx_inputs, onnx_outputs = transformers.onnx.export(tokenizer, model, onnx_config, 16, model_path)\n",
        "    return model_path\n",
        "\n",
        "def quantize_model(model_path: Path):\n",
        "    \"\"\"\n",
        "    Step 2. Quantize the model, so that it can be run on mobile devices with smaller memory footprint\n",
        "    \"\"\"\n",
        "    quantized_model_path = model_path.with_name(model_path.stem+\"_quant\").with_suffix(model_path.suffix)\n",
        "    quantize_dynamic(model_path, quantized_model_path, weight_type=QuantType.QInt8)\n",
        "    model_path.unlink()\n",
        "    return quantized_model_path\n",
        "\n",
        "\n",
        "def add_pre_post_process(quantized_model_path: Path, tokenizer):\n",
        "    \"\"\"\n",
        "    Step 3. Add pre and post processing to the model, for tokenization and post processing\n",
        "    \"\"\"\n",
        "    @contextmanager\n",
        "    def temp_vocab_file():\n",
        "        vocab_file = quantized_model_path.parent / \"vocab.txt\"\n",
        "        yield vocab_file\n",
        "        vocab_file.unlink()\n",
        "\n",
        "    tokenizer_type = 'BertTokenizer'\n",
        "    task_type = 'SequenceClassification'\n",
        "    output_model_path = quantized_model_path.with_name(\n",
        "        quantized_model_path.stem+'_with_pre_post_processing').with_suffix(quantized_model_path.suffix)\n",
        "    with temp_vocab_file() as vocab_file:\n",
        "        import json\n",
        "        with open(str(vocab_file), 'w') as f:\n",
        "            f.write(json.dumps(tokenizer.vocab))\n",
        "        add_ppp.transformers_and_bert(quantized_model_path, output_model_path, vocab_file, tokenizer_type, task_type,add_debug_before_postprocessing=True)\n",
        "    quantized_model_path.unlink()\n",
        "    return output_model_path\n"
      ],
      "metadata": {
        "id": "XT7t2rAX0TMk"
      },
      "execution_count": 76,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_out_path = Path('/content/qmodel/parsbert.onnx')\n",
        "model = get_model('/content/result_parts_bert1/checkpoint-1314')\n",
        "quantized_model = quantize_model(model_path)\n",
        "output_model = add_pre_post_process(quantized_model, tokenizer)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OsiTleH73siA",
        "outputId": "2b3c600b-4c45-4052-be83-28c554304e78"
      },
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:root:Please consider to run pre-processing before quantization. Refer to example: https://github.com/microsoft/onnxruntime-inference-examples/blob/main/quantization/image_classification/cpu/ReadMe.md \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Test quantized model"
      ],
      "metadata": {
        "id": "4lYkfDGE6x-D"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import onnxruntime as _ort\n",
        "from onnxruntime_extensions import get_library_path as _lib_path\n",
        "\n",
        "so = _ort.SessionOptions()\n",
        "so.register_custom_ops_library(_lib_path())\n",
        "\n",
        "# Run the ONNXRuntime Session as per ONNXRuntime docs suggestions.\n",
        "sess = _ort.InferenceSession('/content/qmodel/parsbert_quant_with_pre_post_processing.onnx', so)"
      ],
      "metadata": {
        "id": "QloMZG-w5UZc"
      },
      "execution_count": 78,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_test = ['غذا اصن خوب نبود', 'اعصابم داغونه', 'خیلی خوشمزعه و فوق العاده بود' , 'چه روز قشنگی']\n",
        "labels_test = [1, 1, 0, 0]"
      ],
      "metadata": {
        "id": "TKBijLdD7E0N"
      },
      "execution_count": 79,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "labels = []\n",
        "for item in X_test:\n",
        "  model_output = sess.run([output_name], {input_name: [[item]]})\n",
        "  labels.append(model_output[0].argmax())\n",
        "  # labels.append(model_output[0].max())\n",
        "\n",
        "labels"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MLzvI50e69Ie",
        "outputId": "620b131f-1b08-4f5e-84a5-05d830e97025"
      },
      "execution_count": 82,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1, 1, 0, 0]"
            ]
          },
          "metadata": {},
          "execution_count": 82
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Run inference\n",
        "input_name = sess.get_inputs()[0].name\n",
        "output_name = sess.get_outputs()[0].name\n",
        "model_output = sess.run([output_name], {input_name: [['واقعا بد مزه بود. این چی بود اخهههههههه']]})\n",
        "model_output[0].argmax()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lH8xcvvh657t",
        "outputId": "fccd2a3c-453c-4454-dae2-4c6b1ad57a91"
      },
      "execution_count": 83,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {},
          "execution_count": 83
        }
      ]
    }
  ]
}