torch_train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.functional as F\n",
    "import torch.optim as optim\n",
    "import torchvision\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from torch.utils.data import DataLoader,Dataset, random_split\n",
    "import matplotlib.pyplot as plt\n",
    "import random \n",
    "import time\n",
    "from tqdm import tqdm_notebook as tq\n",
    "import warnings\n",
    "import pickle as pkl\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import string\n",
    "import sys\n",
    "from nltk.corpus import stopwords\n",
    "from torch.autograd import Variable\n",
    "plt.ion()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "epochs = 10\n",
    "num_class = 2\n",
    "embed_dim = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"gpu\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class YelpReviewsSentimentAnalysis(nn.Module):\n",
    "    \n",
    "    def __init__(self,vocab_size,embed_dim,num_class):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.EmbeddingBag(vocab_size, embed_dim, sparse=True)\n",
    "        self.fc = nn.Linear(embed_dim, num_class)\n",
    "        self.init_weights()\n",
    "        \n",
    "    def init_weights(self):\n",
    "        initrange = 0.5\n",
    "        self.embedding.weight.data.uniform_(-initrange, initrange)\n",
    "        self.fc.weight.data.uniform_(-initrange, initrange)\n",
    "        self.fc.bias.data.zero_()\n",
    "        \n",
    "    def forward(self, text, offsets):\n",
    "        embedded = self.embedding(text, offsets)\n",
    "        return self.fc(embedded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class YelpDataset(Dataset):\n",
    "    \n",
    "    def __init__(self,json_file,threshold=3):\n",
    "        \n",
    "        self.raw_data = pd.read_json(json_file,lines=True)\n",
    "    \n",
    "        self.raw_data['label'] = self.raw_data.stars.apply(lambda x : 1 if x>=3 else 0)\n",
    "        \n",
    "        self.raw_data = self.raw_data[[\"label\",\"text\"]].iloc[:100000,]\n",
    "        \n",
    "        self.word2idx = {}\n",
    "        \n",
    "        self.idx2word = {}\n",
    "        \n",
    "        self.word2freq = {}\n",
    "        self.word_count = 0\n",
    "        \n",
    "        self.maxLen = 0\n",
    "        self.__init__preprocess()\n",
    "        \n",
    "        self.data = self.raw_data.to_numpy()\n",
    "   \n",
    "        \n",
    "       \n",
    "        \n",
    "    def __len__(self):\n",
    "        return self.data.shape[0]\n",
    "    \n",
    "    \n",
    "    def __getitem__(self,idx):\n",
    "        \n",
    "        \n",
    "        sample = self.data[idx,:]\n",
    "            \n",
    "        return sample\n",
    "    \n",
    "    def __init__preprocess(self):\n",
    "        \n",
    "        \n",
    "        def clean(text):\n",
    "            text = text.lower()\n",
    "        \n",
    "            text = [ch for ch in text if ch not in string.punctuation]\n",
    "\n",
    "\n",
    "            text = \"\".join(text)\n",
    "\n",
    "            text = [c for c in text if c == \" \" or c.isalnum()]\n",
    "\n",
    "            text = \"\".join(text)\n",
    "\n",
    "            stop_words = set(stopwords.words(\"english\"))\n",
    "\n",
    "            text = text.split(\" \")\n",
    "\n",
    "            text = [word for word in text if word not in stop_words]\n",
    "        \n",
    "        \n",
    "            text = \" \".join(text)\n",
    "            \n",
    "            return text\n",
    "          \n",
    "            \n",
    "        def build_vocab(text):\n",
    "            \n",
    "            \n",
    "            text = text.split(\" \")\n",
    "            \n",
    "            text_token = []\n",
    "            \n",
    "            for word in text:\n",
    "                \n",
    "                if word not in self.word2idx:\n",
    "                    \n",
    "                    self.word2idx[word] = self.word_count\n",
    "                    \n",
    "                    self.idx2word[self.word_count] = word\n",
    "                    \n",
    "                    self.word_count+=1\n",
    "                    \n",
    "                    \n",
    "                text_token.append(self.word2idx[word])\n",
    "             \n",
    "            self.maxLen = max(self.maxLen,len(text_token))\n",
    "            return text_token\n",
    "        \n",
    "        self.raw_data['text'] = self.raw_data.text.apply(lambda x : clean(x))\n",
    "        self.raw_data['text'] = self.raw_data.text.apply(lambda x : build_vocab(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def collate_offset(batch):\n",
    "    \n",
    "    text = [torch.tensor(x[1]) for x in batch]\n",
    "    \n",
    "    label = torch.tensor([x[0] for x in batch])\n",
    "    offsets = [0] + [len(x) for x in text]\n",
    "    offsets = torch.tensor(offsets[:-1]).cumsum(dim=0)\n",
    "    \n",
    "    text = torch.cat(text)\n",
    "    \n",
    "    return text, offsets, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_func(train):\n",
    "    \n",
    "    train_data = DataLoader(train,batch_size=BATCH_SIZE,shuffle=True,collate_fn=collate_offset,num_workers=4)\n",
    "    \n",
    "    train_loss = 0\n",
    "    train_acc = 0\n",
    "    for i, (text, offsets, label) in enumerate(train_data):\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        text, offsets, label = text.to(device), offsets.to(device), label.to(device)\n",
    "        \n",
    "    \n",
    "        output = model(text, offsets)\n",
    "        \n",
    "        loss = criterion(output, label)\n",
    "        \n",
    "        #loss = Variable(loss,requires_grad=True)\n",
    "        \n",
    "        train_loss += loss.item()\n",
    "        \n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        train_acc += (output.argmax(1) == label).sum().item()\n",
    "\n",
    "    \n",
    "    scheduler.step()\n",
    "    \n",
    "    return train_loss / len(train), train_acc / len(train)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_func(test):\n",
    "    \n",
    "    test_data = DataLoader(test,batch_size=BATCH_SIZE,shuffle=True,collate_fn=collate_offset,num_workers=4)\n",
    "    \n",
    "    test_loss = 0\n",
    "    test_acc = 0\n",
    "    \n",
    "    for i, (text, offsets, label) in enumerate(test_data):\n",
    "        \n",
    "        text, offsets, label = text.to(device), offsets.to(device), label.to(device)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            \n",
    "            output = model(text, offsets)\n",
    "\n",
    "            loss = criterion(output, label)\n",
    "            \n",
    "            #loss = Variable(loss,requires_grad=True)\n",
    "            \n",
    "            test_loss += loss.item()\n",
    "            \n",
    "           \n",
    "            test_acc += (output.argmax(1) == label).sum().item()   \n",
    "    \n",
    "    return test_loss / len(test), test_acc / len(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time Taken to for dataset preprocessing is 3.930955235163371 minutes...\n",
      "Length of training dataset is 80000\n",
      "Length of validation dataset is 10000\n",
      "model_0.pkl saved..\n",
      "Time Taken to complete epoch is 0.36666666666666664 minutes...\n",
      "model_1.pkl saved..\n",
      "Time Taken to complete epoch is 0.36666666666666664 minutes...\n",
      "model_2.pkl saved..\n",
      "Time Taken to complete epoch is 0.38333333333333336 minutes...\n",
      "model_3.pkl saved..\n",
      "Time Taken to complete epoch is 0.4 minutes...\n",
      "model_4.pkl saved..\n",
      "Time Taken to complete epoch is 0.36666666666666664 minutes...\n",
      "model_5.pkl saved..\n",
      "Time Taken to complete epoch is 0.36666666666666664 minutes...\n",
      "model_6.pkl saved..\n",
      "Time Taken to complete epoch is 0.38333333333333336 minutes...\n",
      "model_7.pkl saved..\n",
      "Time Taken to complete epoch is 0.36666666666666664 minutes...\n",
      "model_8.pkl saved..\n",
      "Time Taken to complete epoch is 0.38333333333333336 minutes...\n",
      "model_9.pkl saved..\n",
      "Time Taken to complete epoch is 0.38333333333333336 minutes...\n"
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    \n",
    "    \n",
    "    \n",
    "    start_time = time.time()\n",
    "    yelp_dataset = YelpDataset(json_file = \"~/data/yelp/review.json\")\n",
    "    \n",
    "    print(\"Time Taken to for dataset preprocessing is {} minutes...\".format((time.time()-start_time)/60))\n",
    "    \n",
    "    \n",
    "    train_len = int(len(yelp_dataset)*0.8)\n",
    "    \n",
    "    valid_len = int(len(yelp_dataset)*0.1)\n",
    "    \n",
    "    test_len = len(yelp_dataset) - train_len -valid_len\n",
    "    \n",
    "    train,valid,test = random_split(yelp_dataset,[train_len,valid_len,test_len])\n",
    "    \n",
    "    \n",
    "    \n",
    "    print(\"Length of training dataset is {}\".format(train_len))\n",
    "          \n",
    "    print(\"Length of validation dataset is {}\".format(valid_len))\n",
    "          \n",
    "      \n",
    "    model = YelpReviewsSentimentAnalysis(yelp_dataset.word_count,embed_dim, num_class)\n",
    "    \n",
    "    \n",
    "    criterion = torch.nn.CrossEntropyLoss().to(device)\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=4.0)\n",
    "    \n",
    "    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, gamma=0.9)\n",
    "     \n",
    "    tl_,ta_ = [],[]\n",
    "    vl_,va_ = [],[]\n",
    "    \n",
    "    for epch in range(epochs):\n",
    "          \n",
    "        start_time = time.time()\n",
    "        train_loss, train_acc = train_func(train)\n",
    "        valid_loss, valid_acc = test_func(valid)\n",
    "\n",
    "        tl_.append(train_loss)\n",
    "        ta_.append(train_acc)\n",
    "        \n",
    "        \n",
    "        vl_.append(valid_loss)\n",
    "        va_.append(valid_acc)\n",
    "        \n",
    "        \n",
    "        secs = int(time.time() - start_time)\n",
    "        mins = secs / 60\n",
    "        \n",
    "        \n",
    "        \n",
    "        name = \"model_\"+str(epch)+\".pkl\"\n",
    "        with open(name,\"wb\") as file:\n",
    "            pkl.dump(model,file)\n",
    "            \n",
    "        print(\"{} saved..\".format(name))\n",
    "        print(\"Time Taken to complete epoch is {} minutes...\".format(mins))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f5325862250>]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmkAAAHVCAYAAACqkWxBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3Xl0XdV99//31uxRnqTrebaxZAPGCJvRYCzHdprgkDAYkoaktGSAtkl+fVro71nP6pP+2idJm9AMQEIDlKQxQ0hI3TyJjQ2GMBgbmSF4tjyPsjzPliXt3x+6JorxINuS79XV+7UWi3vP3efoe9bKWvmwz9nfHWKMSJIkKb1kpboASZIkfZghTZIkKQ0Z0iRJktKQIU2SJCkNGdIkSZLSkCFNkiQpDRnSJEmS0pAhTZIkKQ0Z0iRJktJQTqoLaA49evSIAwcOTHUZkiRJZ7Ro0aIdMcaiM43LiJA2cOBAKioqUl2GJEnSGYUQ1jdlnI87JUmS0pAhTZIkKQ0Z0iRJktKQIU2SJCkNGdIkSZLSkCFNkiQpDRnSJEmS0pAhTZIkKQ0Z0iRJktKQIU2SJCkNGdIkSZLSkCFNkiQpDRnSJEmS0pAhTZIkKQ0Z0iRJktKQIU2SJCkNGdKaIMbIlj2HU12GJElqQwxpTfDDV9Yw/lvz2Hv4WKpLkSRJbYQhrQnGDupGbX3klZXVqS5FkiS1EYa0Jhjdrws9OuYxd2lVqkuRJElthCGtCbKzAjeOKGbeiu0cq6tPdTmSJKkNMKQ1UXlJgv1Hanlr7a5UlyJJktoAQ1oTXTusB/k5WcxZ5iNPSZLU8gxpTdQ+L4drh/Zg7rIqYoypLkeSJGU4Q9pZKC9NsHHXYVZWHUh1KZIkKcMZ0s7CxBHFAMz1kackSWphhrSzUNy5gEv7FjLHVhySJKmFGdLOUnlJgnc37mH7/iOpLkWSJGUwQ9pZKi9NAPDSsu0prkSSJGUyQ9pZGtGzE326tGOuIU2SJLUgQ9pZCiEwqTTBa5XVHK6pS3U5kiQpQxnSzkF5SYIjx+p5vXJHqkuRJEkZypB2DsYO6kan/BxbcUiSpBZjSDsHeTlZXH9REXOXbae+3t0HJElS8zOknaNJpQl2HDjKe5v2pLoUSZKUgQxp5+iG4cVkZwUfeUqSpBbRpJAWQpgSQlgRQqgMIdx/kt/zQwjPJH9fEEIY2Oi3B5LHV4QQJjc6/tUQwpIQwuIQwlMhhILk8UHJa6xKXjPv/G+z+RW2z2XswG7MXWorDkmS1PzOGNJCCNnAQ8BUoBS4I4RQesKwu4HdMcahwIPAN5PnlgLTgZHAFODhEEJ2CKEP8FdAWYxxFJCdHEfy3AdjjMOA3clrp6Xy0gQrqvazYeehVJciSZIyTFNm0sYClTHGNTHGGuBpYNoJY6YBTyY/PwdMDCGE5PGnY4xHY4xrgcrk9QBygHYhhBygPbAlec6NyWuQvOYnzu3WWl55iRuuS5KkltGUkNYH2Njo+6bksZOOiTHWAnuB7qc6N8a4GfhXYAOwFdgbY3whec6e5DVO9bcACCHcE0KoCCFUVFdXN+E2mt+A7h0YnuhoSJMkSc2uKSEtnOTYiX0nTjXmpMdDCF1pmGUbBPQGOoQQPtPEv9VwMMZHY4xlMcayoqKiUxbf0spLEixYu4u9h46lrAZJkpR5mhLSNgH9Gn3vC2w51Zjk48tCYNdpzi0H1sYYq2OMx4BfAlcDO4AuyWuc6m+llfLSBHX1kZdXuoBAkiQ1n6aEtLeAYclVl3k0vOA/84QxM4G7kp9vAV6KMcbk8enJ1Z+DgGHAQhoec14ZQmiffA9tIrAsec685DVIXvO/zv32Wt7ovl3o0THPDdclSVKzOmNIS74fdh8wG1gGPBtjXBJC+HoI4abksMeA7iGESuBrwP3Jc5cAzwJLgVnAvTHGuhjjAhoWB7wNvJ+s49Hktf4O+FryWt2T105bWVmBiSMSvLxiOzW19akuR5IkZYjQMHnVupWVlcWKioqU/f05S6v4i59U8LM/H8c1Q3ukrA5JkpT+QgiLYoxlZxrnjgPN4NqhPcjPyWLOUld5SpKk5mFIawbt8rK5blgP5i6rIhNmJiVJUuoZ0ppJeUmCTbsPs6Jqf6pLkSRJGcCQ1kxuHJHcfcBHnpIkqRkY0ppJcecCLu3XhTm24pAkSc3AkNaMJpUU897GPWzfdyTVpUiSpFbOkNaMyksTALy03Nk0SZJ0fgxpzeiiRCf6dm3nhuuSJOm8GdKaUQiB8pIEr67aweGaulSXI0mSWjFDWjObVJrgaG09r1XuSHUpkiSpFTOkNbOxg7rRqSDHVhySJOm8GNKaWW52FjdcVMyLy6uor3f3AUmSdG4MaS2gvKSYHQdqeHfTnlSXIkmSWilDWgu4YXgxOVnBR56SJOmcGdJaQGH7XMYO6mYrDkmSdM4MaS2kvCTByqoDrN95MNWlSJKkVsiQ1kLKSxp2H5jrXp6SJOkcGNJaSP/u7bko0cn30iRJ0jkxpLWg8tJiFq7bxd5Dx1JdiiRJamUMaS2ovCRBXX3k5ZU+8pQkSWfHkNaCLu3bhR4d85njI09JknSWDGktKCsrUF5SzCsrqqmprU91OZIkqRUxpLWw8pIE+4/WsnDtrlSXIkmSWhFDWgu7ZmgPCnKzbGwrSZLOiiGthbXLy+baoUXMWVpFjG64LkmSmsaQdgFMKi1m857DLN+2P9WlSJKkVsKQdgFMGFEMYGNbSZLUZIa0C6C4UwGj+3XxvTRJktRkhrQLZFJpgvc27aVq35FUlyJJkloBQ9oFcnzD9ZeWu/uAJEk6M0PaBTI80ZF+3dr5XpokSWoSQ9oFEkKgvCTBa5U7OFRTm+pyJElSmjOkXUCTShIcra3ntVU7Ul2KJElKc4a0C+iKQd3oVJDjKk9JknRGhrQLKDc7iwkXFfPisu3U1bv7gCRJOjVD2gVWXppg58Ea3t24J9WlSJKkNGZIu8CuH15ETlbwkackSTotQ9oFVtgul3GDu9mKQ5IknZYhLQXKSxKs2n6AdTsOproUSZKUpgxpKXB89wEfeUqSpFNpUkgLIUwJIawIIVSGEO4/ye/5IYRnkr8vCCEMbPTbA8njK0IIk5PHLgohvNvon30hhK8kf/uHEMLmRr99tHluNX3069aeET07GdIkSdIpnTGkhRCygYeAqUApcEcIofSEYXcDu2OMQ4EHgW8mzy0FpgMjgSnAwyGE7Bjjihjj6BjjaOBy4BDwfKPrPXj89xjjb87vFtNTeUmCt9btZs+hmlSXIkmS0lBTZtLGApUxxjUxxhrgaWDaCWOmAU8mPz8HTAwhhOTxp2OMR2OMa4HK5PUamwisjjGuP9ebaI3KSxPU1UdeXlGd6lIkSVIaakpI6wNsbPR9U/LYScfEGGuBvUD3Jp47HXjqhGP3hRB+H0J4PITQ9WRFhRDuCSFUhBAqqqtbX9C5pE8hRZ3ymeMjT0mSdBJNCWnhJMdObJd/qjGnPTeEkAfcBPy80e+PAEOA0cBW4NsnKyrG+GiMsSzGWFZUVHTq6tNUVlagvKSYV1ZUU1Nbn+pyJElSmmlKSNsE9Gv0vS+w5VRjQgg5QCGwqwnnTgXejjF+MJ0UY6yKMdbFGOuBf+fDj0czRnlJggNHa1mwdmeqS5EkSWmmKSHtLWBYCGFQcuZrOjDzhDEzgbuSn28BXooxxuTx6cnVn4OAYcDCRufdwQmPOkMIvRp9vRlY3NSbaW2uGdqDgtwsG9tKkqQPOWNIS75jdh8wG1gGPBtjXBJC+HoI4abksMeA7iGESuBrwP3Jc5cAzwJLgVnAvTHGOoAQQntgEvDLE/7kt0II74cQfg9MAL56nveYtgpys7luWBFzl22nIdNKkiQ1CJkQDsrKymJFRUWqyzgnz761kb/9xe/5zV9dR2nvzqkuR5IktbAQwqIYY9mZxrnjQIpNGFFMCO4+IEmS/pghLcWKOuUzul8XQ5okSfojhrQ0UF6S4Peb9lK170iqS5EkSWnCkJYGJpU2bLj+4rLtKa5EkiSlC0NaGhhW3JH+3dr7yFOSJH3AkJYGQgiUlyR4rXIHh2pqU12OJElKA4a0NFFeWkxNbT2vrtqR6lIkSVIaMKSliSsGdqNzQY67D0iSJMCQljZys7OYMKKYl5Zvp66+9TcYliRJ58eQlkbKSxLsPFjDuxt3p7oUSZKUYoa0NHL9RUXkZAXmLLUVhyRJbZ0hLY10LsjlysHdbcUhSZIMaemmvKSYyu0HWLvjYKpLkSRJKWRISzMTS47vPuBsmiRJbZkhLc3069aeET07McdWHJIktWmGtDQ0qTRBxfrd7D5Yk+pSJElSihjS0lB5SYK6+sjLK13lKUlSW2VIS0MX9ymkuFM+c23FIUlSm2VIS0NZWYGJJQleWVnN0dq6VJcjSZJSwJCWpiaVFnPgaC0L1uxKdSmSJCkFDGlp6uohPWiXm21jW0mS2ihDWpoqyM3mumE9mLu0ihjdcF2SpLbGkJbGyksTbNl7hKVb96W6FEmSdIEZ0tLYjSOKCQFXeUqS1AYZ0tJYj475XNavi++lSZLUBhnS0lx5aYL3N+9l294jqS5FkiRdQIa0NDfp+Ibry51NkySpLTGkpbmhxR0Z0L09c91wXZKkNsWQluZCCJSXJHh99U4OHq1NdTmSJOkCMaS1AuUlCWpq63l11Y5UlyJJki4QQ1orUDawK4Xtcl3lKUlSG2JIawVys7OYcFERLy3fTl29uw9IktQWGNJaifLSBLsO1vDOht2pLkWSJF0AhrRWYvzwInKzA3N85ClJUptgSGslOhfkcuXg7rbikCSpjTCktSLlJQlWVx9kTfWBVJciSZJamCGtFZlYUgzAi8vccF2SpExnSGtF+nZtT0mvzr6XJklSG2BIa2UmlRRTsW4Xuw/WpLoUSZLUggxprUx5aYL6CPNW+MhTkqRMZkhrZUb1LiTROd/dByRJynBNCmkhhCkhhBUhhMoQwv0n+T0/hPBM8vcFIYSBjX57IHl8RQhhcvLYRSGEdxv9sy+E8JXkb91CCHNCCKuS/+7aPLeaGbKyAhNLEryyopqjtXWpLkeSJLWQM4a0EEI28BAwFSgF7gghlJ4w7G5gd4xxKPAg8M3kuaXAdGAkMAV4OISQHWNcEWMcHWMcDVwOHAKeT17rfuDFGOMw4MXkdzUyqSTBwZo63lyzK9WlSJKkFtKUmbSxQGWMcU2MsQZ4Gph2wphpwJPJz88BE0MIIXn86Rjj0RjjWqAyeb3GJgKrY4zrT3KtJ4FPnM0NtQVXDelOu9xsG9tKkpTBmhLS+gAbG33flDx20jExxlpgL9C9iedOB55q9D0RY9yavNZWoPhkRYUQ7gkhVIQQKqqrq5twG5mjIDeb8cN7MHdZFTG64bokSZmoKSEtnOTYicngVGNOe24IIQ+4Cfh5E+r444vE+GiMsSzGWFZUVHS2p7d65SUJtu49wpIt+1JdiiRJagFNCWmbgH6NvvcFtpxqTAghBygEdjXh3KnA2zHGxs/tqkIIvZLX6gXYa+IkJowoJgRc5SlJUoZqSkh7CxgWQhiUnPmaDsw8YcxM4K7k51uAl2LDc7iZwPTk6s9BwDBgYaPz7uCPH3WeeK27gP9q6s20JT065jOmf1e3iJIkKUOdMaQl3zG7D5gNLAOejTEuCSF8PYRwU3LYY0D3EEIl8DWSKzJjjEuAZ4GlwCzg3hhjHUAIoT0wCfjlCX/yG8CkEMKq5O/fOL9bzFzlJQne37yXrXsPp7oUSZLUzEImvHheVlYWKyoqUl3GBVe5fT/l3/kd/98nRvGZKwekuhxJktQEIYRFMcayM41zx4FWbEhRRwZ2b+97aZIkZSBDWisWQqC8JMEblTs5eLQ21eVIkqRmZEhr5cpLE9TU1fPqqrbVK06SpExnSGvlygZ0pbBdLnOWuspTkqRMYkhr5XKys7hxRDEvLa+irr71LwKRJEkNDGkZoLwkwe5Dx3h7w+5UlyJJkpqJIS0DjB/eg9zs4IbrkiRlEENaBuhUkMuVg7szx1YckiRlDENahphUmmBN9UFWVx9IdSmSJKkZGNIyxMSSBAAvOpsmSVJGMKRliD5d2lHaqzNzbcUhSVJGMKRlkPLSBBXrd7HrYE2qS5EkSefJkJZBJpUkqI8wb7mzaZIktXaGtAwyqk9nEp3z3XBdkqQMYEjLIMc3XH9lZTVHjtWluhxJknQeDGkZprw0waGaOt5cszPVpUiSpPNgSMswVw3uTvu8bB95SpLUyhnSMkxBbjbjhxUxd+l2YnTDdUmSWitDWgYqL02wbd8RlmzZl+pSJEnSOTKkZaAJFxURAsxxw3VJklotQ1oG6t4xn8v7d+XF5YY0SZJaK0NahiovTbB48z627j2c6lIkSdI5MKRlqPLkhutzl7n7gCRJrZEhLUMNKerAoB4dmOt7aZIktUqGtAzVsPtAMfNX7+TA0dpUlyNJks6SIS2DlZckqKmr59WV1akuRZIknSVDWga7fEBXurTPZY67D0iS1OoY0jJYTnYWN15UzLzl26mtq091OZIk6SwY0jJceWmC3YeO8faGPakuRZIknQVDWoYbP7yIvOwsN1yXJKmVMaRluI75OVw5pLutOCRJamUMaW3ApJJi1uw4yOrqA6kuRZIkNZEhrQ2YeHz3AWfTJElqNQxpbUDvLu0Y2buz76VJktSKGNLaiPKSBIvW72bngaOpLkWSJDWBIa2NmFSaoD7CvBXuPiBJUmtgSGsjRvbuTM/OBb6XJklSK2FIayNCCJSXFvO7VdUcOVaX6nIkSdIZGNLakPKSBIdq6pi/ZmeqS5EkSWdgSGtDrhrSnQ552T7ylCSpFWhSSAshTAkhrAghVIYQ7j/J7/khhGeSvy8IIQxs9NsDyeMrQgiTGx3vEkJ4LoSwPISwLIRwVfL4P4QQNocQ3k3+89Hzv00B5OdkM354EXOXVRFjTHU5kiTpNM4Y0kII2cBDwFSgFLgjhFB6wrC7gd0xxqHAg8A3k+eWAtOBkcAU4OHk9QC+C8yKMY4ALgWWNbregzHG0cl/fnPOd6cPKS9JULXvKIs370t1KZIk6TSaMpM2FqiMMa6JMdYATwPTThgzDXgy+fk5YGIIISSPPx1jPBpjXAtUAmNDCJ2B8cBjADHGmhjjnvO/HZ3JhBHFZAVsbCtJUpprSkjrA2xs9H1T8thJx8QYa4G9QPfTnDsYqAaeCCG8E0L4cQihQ6Nx94UQfh9CeDyE0PVkRYUQ7gkhVIQQKqqr7f3VVN065HH5gK6GNEmS0lxTQlo4ybETX2g61ZhTHc8BxgCPxBgvAw4Cx991ewQYAowGtgLfPllRMcZHY4xlMcayoqKiM96E/qC8JMGSLfvYsudwqkuRJEmn0JSQtgno1+h7X2DLqcaEEHKAQmDXac7dBGyKMS5IHn+OhtBGjLEqxlgXY6wH/p2Gx61qRuWlDRuuv+hsmiRJaaspIe0tYFgIYVAIIY+GhQAzTxgzE7gr+fkW4KXYsHxwJjA9ufpzEDAMWBhj3AZsDCFclDxnIrAUIITQq9F1bwYWn8N96TSGFHVkcI8OzFm2PdWlSJKkU8g504AYY20I4T5gNpANPB5jXBJC+DpQEWOcScMCgJ+GECppmEGbnjx3SQjhWRoCWC1wb4zxeLv7vwR+lgx+a4DPJ49/K4QwmobHouuALzTPraqx8tIET7y+lv1HjtGpIDfV5UiSpBOETOiXVVZWFisqKlJdRquycO0ubvvRfB7+9Bg+enGvM58gSZKaRQhhUYyx7Ezj3HGgjRrTvwtd2+e6+4AkSWnKkNZG5WRnMWFEMS+t2E5tXX2qy5EkSScwpLVhk0oS7Dl0jEXrd6e6FEmSdAJDWht23fAi8rKzbGwrSVIaMqS1YR3zc7hqSHfmLHXDdUmS0o0hrY0rL02wbuchVlcfTHUpkiSpEUNaG1deUgy44bokSenGkNbG9Spsx6g+nW3FIUlSmjGkifKSBIs27GbngaOpLkWSJCUZ0kR5SYIY4ZGXV7uAQJKkNGFIEyN7d+b2sn78+LW1/I/nfs8xm9tKkpRyZ9xgXZkvhMA3PnUxvboU8G9zV1G17wgPf3qMG69LkpRCzqQJaAhqXykfzr/ccgnzV+/k1h/OZ+vew6kuS5KkNsuQpj9ya1k/nvj8FWzafZibH3qDZVv3pbokSZLaJEOaPuS6YUX8/ItXAXDbD+fz2qodKa5IkqS2x5Cmkyrp1Znn772aPl3b8bknFvLzio2pLkmSpDbFkKZT6lXYjme/eBVXDu7O/3ju93x37ipbdEiSdIEY0nRanQtyefxzV/CpMX15cO5K/tYWHZIkXRC24NAZ5eVk8a+3XkLfru347our2GaLDkmSWpwzaWqSEAJfnTScbzVq0bFt75FUlyVJUsYypOms3FbWj8c/dwUbdx3i5odfZ/k2W3RIktQSDGk6a+OHF/HsF6+iPkZufWQ+r1faokOSpOZmSNM5Gdm7kOe/fA29u7TjrscX8tyiTakuSZKkjGJI0znr3aUdP//SVYwb3I2/+fl7fO9FW3RIktRcDGk6L50Lcnnic2P55Jg+fGfOSv7uF7bokCSpOdiCQ+ctLyeLb996KX27tud7L65i615bdEiSdL6cSVOzCCHwtUnD+danLuGN1Tu57UdvUrXPFh2SJJ0rQ5qa1W1XNLTo2LDzIDc/9Dortu1PdUmSJLVKhjQ1u+uTLTpq6yO3PPIGb9iiQ5Kks2ZIU4sY2buQ5++9hl5dCrjriYX88m1bdEiSdDYMaWoxfbq04+dfvJorBnbja8++x/dt0SFJUpMZ0tSiCtvl8h+fH8snL+vDt+es5IFfvm+LDkmSmsAWHGpxeTlZfPu2S+nbtR3fe6mSLckWHR3z/Z+fJEmn4kyaLogQAl/7yEV845MX83rlDm774XxbdEiSdBqGNF1Q08f257G7ylifbNGxssoWHZIknYwhTRfcDRcV88wXGlp0fMoWHZIknZQhTSkxqk+yRUdhQ4uO59+xRYckSY0Z0pQyx1t0lA3oxlefeY8fvGSLDkmSjjOkKaUK2+Xy5J+N5ebL+vCvL9iiQ5Kk4+yBoJTLy8niO7ddSp8u7fjBvEq27j3CQ7bokCS1cU2aSQshTAkhrAghVIYQ7j/J7/khhGeSvy8IIQxs9NsDyeMrQgiTGx3vEkJ4LoSwPISwLIRwVfJ4txDCnBDCquS/u57/bSrdhRD4m8kX8X8+eTGvVe7g9h/ZokOS1LadMaSFELKBh4CpQClwRwih9IRhdwO7Y4xDgQeBbybPLQWmAyOBKcDDyesBfBeYFWMcAVwKLEsevx94McY4DHgx+V1txB1j+/Pju8pYt+Mgn3z4DVt0SJLarKbMpI0FKmOMa2KMNcDTwLQTxkwDnkx+fg6YGEIIyeNPxxiPxhjXApXA2BBCZ2A88BhAjLEmxrjnJNd6EvjEud2aWqsJyRYdNXX1DS06VtuiQ5LU9jQlpPUBNjb6vil57KRjYoy1wF6g+2nOHQxUA0+EEN4JIfw4hNAhOSYRY9yavNZWoPis7kgZYVSfQp7/8tX07FzAXY8v5FfvbE51SZIkXVBNCWnhJMdO7JNwqjGnOp4DjAEeiTFeBhzkLB9rhhDuCSFUhBAqqqurz+ZUtRJ9u7bnuS9dzeUDuvKVZ97loXmVtuiQJLUZTQlpm4B+jb73BbacakwIIQcoBHad5txNwKYY44Lk8edoCG0AVSGEXslr9QK2n6yoGOOjMcayGGNZUVFRE25DrdHxFh2fGN2bf5m9gr9//n1qbdEhSWoDmhLS3gKGhRAGhRDyaFgIMPOEMTOBu5KfbwFeig1THjOB6cnVn4OAYcDCGOM2YGMI4aLkOROBpSe51l3Af53DfSmD5Odk8+Dto7lvwlCeWriRP/9JBQeP1qa6LEmSWtQZQ1ryHbP7gNk0rMB8Nsa4JITw9RDCTclhjwHdQwiVwNdIPrqMMS4BnqUhgM0C7o0x1iXP+UvgZyGE3wOjgX9OHv8GMCmEsAqYlPyuNu54i45/vvliXl21g9sfnc92W3RIkjJYyIR3fMrKymJFRUWqy9AFMm/5du6d8TZd2+fxH5+/gmGJTqkuSZKkJgshLIoxlp1pnNtCqdWZMKKYZ5MtOj75yBvMX70z1SVJktTsDGlqlY636Eh0LuCzjy/gv961RYckKbMY0tRq9e3anl98saFFx18/bYsOSVJmMaSpVSts39CiY1qyRcf/+6vFtuiQJGWEnFQXIJ2v/JxsHrxtNH26tOPhl1ezdc9hfnDnGDrk+z9vSVLr5UyaMkJWVuBvp4zgn24exSsrqxtadOy3RYckqfUypCmjfHrcAB676wrWVB/k5ofeoHL7/lSXJEnSOTGkKeNMGFHMM/ckW3Q8/AZvrrFFhySp9TGkKSNd3LeQX37paoo7F/DZxxbaokOS1OoY0pSx+nVraNFxWf8u/PXT7/Lduatc+SlJajUMacpohe1z+cndY7n5sj48OHcln3rkDVZW+Z6aJCn9GdKU8fJzsvnObZfygzsvY+Puw3zse6/x0LxKZ9UkSWnNkKY2IYTAxy7pzZyvjmfSyAT/MnsFNz/8Bsu37Ut1aZIknZQhTW1K9475PHTnGB7+9Bi27DnMx7//Gt9/cRXHnFWTJKUZQ5rapI9e3IsXvjqeKaN68e05K/nEQ6+zbKuzapKk9GFIU5vVvWM+37/jMn74mTFU7TvCx7//Gt+d66yaJCk9GNLU5k0Z1Ys5X72eP7mkFw/OXcm0H7zOki17U12WJKmNM6RJQNcOeXx3+mX86E8vZ/v+o0z7wet8Z85KamqdVZMkpYYhTWpk8siezP3aeD5+aW++9+IqbvrBayze7KyaJOnCM6RJJ+jSPo8Hbx/Nv3+2jJ0Ha5j20Ot8+4UVHK2tS3VpkqQ2xJAmncKk0gRzvjqeaaN78/2XKrnp+6/z+017Ul2WJKmNMKRJp9GlfR7fuW00j3+ujD2Ha7j54Tf4l9nLnVWTJLU4Q5rUBDeOSPDCV6/n5sv68NC81Xz8+6/x3kZn1SRJLceQJjVRYbtc/vXWS3ni81ew73AtNz/8Ot+ctZwjx5xVkyQ1P0OadJYmXFQYqerOAAAfU0lEQVTMC18bz62X9+ORl1fzse+/xjsbdqe6LElShjGkSeegc0Eu37zlEp78s7EcPFrLpx55g//z22XOqkmSmo0hTToP1w8vYvZXx3P7Ff340Str+JPvvcrbzqpJkpqBIU06T50Lcvk/n7yEn/zZWI4cq+eWR97gn3/jrJok6fwY0qRmMn54EbO+ch3Tx/bn0d+t4aPffZVF63eluixJUitlSJOaUaeCXP755ov5z7vHcbS2nlt+OJ9//PVSDtc4qyZJOjuGNKkFXDusB7O/Op7PjBvAY6+t5aPfe5W31jmrJklqOkOa1EI65ufwj58YxYy/GMexunpu+9F8/vd/L+FQTW2qS5MktQKGNKmFXT2kB7O/Mp4/vXIAT7y+jqnffZUFa3amuixJUpozpEkXQIf8HL4+bRRP/cWVxAi3P/om/zDTWTVJ0qkZ0qQL6Koh3Zn1lev43NUD+Y831jHl315l/mpn1SRJH2ZIky6w9nk5/MNNI3nmnisJAe749zf5X/+1mINHnVWTJP2BIU1KkXGDuzPrr8fzZ9cM4qdvrmfyv/2ONyp3pLosSVKaMKRJKdQuL5v/9fFSnv3CVeRmZ3HnjxfwP3/1PgecVZOkNs+QJqWBKwZ24zd/dR1/fu0gfrZgA5Mf/B2vO6smSW2aIU1KE+3ysvmfHyvluS9eRX5OFp/+8QL+/vn32X/kWKpLkySlgCFNSjOXD+jGb/76Ou4ZP5inFzbMqv1uZXWqy5IkXWBNCmkhhCkhhBUhhMoQwv0n+T0/hPBM8vcFIYSBjX57IHl8RQhhcqPj60II74cQ3g0hVDQ6/g8hhM3J4++GED56frcotT4Fudn8/UdLeO5LV9MuL5vPPr6Q+3/xe/Y5qyZJbcYZQ1oIIRt4CJgKlAJ3hBBKTxh2N7A7xjgUeBD4ZvLcUmA6MBKYAjycvN5xE2KMo2OMZSdc78Hk8dExxt+cy41JmWBM/67837+6ji9cP5hnKzYy+cHf8fKK7akuS5J0ATRlJm0sUBljXBNjrAGeBqadMGYa8GTy83PAxBBCSB5/OsZ4NMa4FqhMXk9SExXkZvPA1BJ+8aWr6ZCfw+eeeIu/fe499h52Vk2SMllTQlofYGOj75uSx046JsZYC+wFup/h3Ai8EEJYFEK454Tr3RdC+H0I4fEQQteTFRVCuCeEUBFCqKiu9n0dZb7L+nfl1395LV++YQjPLdrE5Ad/x7zlzqpJUqZqSkgLJzkWmzjmdOdeE2McQ8Nj1HtDCOOTxx8BhgCjga3At09WVIzx0RhjWYyxrKio6Ay3IGWGgtxs/nbKCJ7/8jV0bpfD5//jLf7m5++x95CzapKUaZoS0jYB/Rp97wtsOdWYEEIOUAjsOt25Mcbj/94OPE/yMWiMsSrGWBdjrAf+HR+PSh9yab8u/PdfXst9E4by/Dub+ci/vcLcpVXEeOJ/P0mSWqumhLS3gGEhhEEhhDwaFgLMPGHMTOCu5OdbgJdiw/9bzASmJ1d/DgKGAQtDCB1CCJ0AQggdgI8Ai5PfezW67s3Hj0v6Y/k52fzN5Iv41ZevoUu7PP78JxXc+sP5zFux3bAmSRkg50wDYoy1IYT7gNlANvB4jHFJCOHrQEWMcSbwGPDTEEIlDTNo05PnLgkhPAssBWqBe2OMdSGEBPB8w9oCcoAZMcZZyT/5rRDCaBoei64DvtB8tytlnov7FvLff3ktTy3cwI9eWc3nn3iLkb07c++EoUwZ2ZOsrJO9dSBJSnchE/6Lu6ysLFZUVJx5oJThamrr+dU7m3nkldWs3XGQIUUd+PINQ7lpdG9ys+1dLUnpIISw6CTtxz48zpAmZZ66+shv3t/KQ/MqWb5tP327tuML1w/h1sv7UpCbfeYLSJJajCFNEjFGXly2nR/Mq+TdjXso7pTPX1w3mDvH9adD/hnfdpAktQBDmqQPxBiZv3onP5hXyRurd9KlfS6fv3oQn7t6IIXtc1NdniS1KYY0SSf19obdPDyvkrnLttMxP4fPXDmAu68dRFGn/FSXJkltgiFN0mkt27qPh+ZV8n/f30pedhbTr+jHPdcPoU+XdqkuTZIymiFNUpOsqT7AD19ZzS/f3gzAJ8f04Us3DGVQjw4prkySMpMhTdJZ2bznMI++spqn39rIsbp6PnpxL+6dMJSSXp1TXZokZRRDmqRzUr3/KI+9tpb/fHM9B47WUl5SzJcnDGVM/66pLk2SMoIhTdJ52XvoGE/OX8fjr69lz6FjXD2kO/dNGMpVQ7qT3C1EknQODGmSmsXBo7XMWLCBR19dQ/X+o1zWvwv33jCUiSXFhjVJOgeGNEnN6sixOp5btIkfvrKaTbsPM6JnJ+6dMJSPXtyLbPcHlaQmM6RJahHH6uqZ+e4WHn65ktXVBxnUowNfun4In7isD3k57g8qSWdiSJPUourrI7OXbOMH8ypZsmUfvQsL+ML1Q7j9in7uDypJp2FIk3RBxBh5eWU1D71UScX63fTomMfd1w7mM1f2p1OBW05J0okMaZIuuAVrGvYHfXXVDjoX5PC5awbx+asH0rVDXqpLk6S0YUiTlDLvbdzDwy9XMntJFe3zsvn0uP78xXWDKe5ckOrSJCnlDGmSUm5l1X4enlfJzPe2kJOdxW1lffnC+CH069Y+1aVJUsoY0iSljfU7D/LDV9bwi0WbqIuRaaN78+UbhjK0uGOqS5OkC86QJintbNt7hEd/t4YZC9dztLaeqaN68uUbhjKqT2GqS5OkC8aQJilt7TxwlCdeX8eTb6xj/9FabrioiPsmDKVsYLdUlyZJLc6QJint7TtyjJ/OX89jr61l18Eaxg3qxn03DuXaoT3cckpSxjKkSWo1DtfU8dTCDTz6uzVs23eES/sW8uUJQ5lUkiDLLackZRhDmqRW52htHb98ezOPvLyaDbsOMTzRkS/fMJSPXdKLnGy3nJKUGQxpklqt2rp6/u/7W3loXiUrqw4woHt7vnj9ED45pg/5OW45Jal1M6RJavXq6yNzl1Xx0LxK3tu0l+4d8rj9in7cOa4/fbvaa01S62RIk5QxYoy8sXonT76xjrnLqgC4cUSCz141gGuH9vC9NUmtSlNDWs6FKEaSzkcIgWuG9uCaoT3YvOcwTy3YwNNvbWDusioG9ejAp8f159bL+1HY3g3dJWUOZ9IktUpHa+uYtXgbP5m/nkXrd1OQm8W0S/vwp1cNsDmupLTm405JbcaSLXv5zzfX86t3tnD4WB1j+nfhs1cNZOrFPV1oICntGNIktTl7Dx/juUWb+M8317N2x8EPFhp8+soB9OnSLtXlSRJgSJPUhtXXR15fvYOfzF/Pi8mFBhNLEvzplS40kJR6LhyQ1GZlZQWuG1bEdcOK2LznMDMWrOfphRuZs7RhocFnrhzALZf3pbCdCw0kpS9n0iS1CUdr6/jt+9v46Zt/WGjwidENCw1G9nahgaQLx8edknQKizcnFxq8u5kjx+q5fEBX/vTKAS40kHRBGNIk6Qz2HjrGc2//YaFBj47HdzRwoYGklmNIk6Qmqq+PvFa5g5+++ccLDT571QCuGeJCA0nNy4UDktREWVmB8cOLGD+8iE27DzFjwQaeeathocHgHh34tAsNJKWAM2mSdBLHFxr8ZP463t6wh3a52Xzist585koXGkg6Pz7ulKRmcrKFBp+9agBTRrnQQNLZM6RJUjPbe+gYP1+0kf98cz3rdh5yoYGkc9KsIS2EMAX4LpAN/DjG+I0Tfs8HfgJcDuwEbo8xrkv+9gBwN1AH/FWMcXby+Dpgf/J47fFiQwjdgGeAgcA64LYY4+7T1WdIk3QhHV9o8JP563lpecNCg/KSBJ+9aiDXDO1OCC40kHRqzRbSQgjZwEpgErAJeAu4I8a4tNGYLwOXxBi/GEKYDtwcY7w9hFAKPAWMBXoDc4HhMca6ZEgrizHuOOHvfQvYFWP8RgjhfqBrjPHvTlejIU1SqmzafYifJRca7DpYw+DkjgafcqGBpFNoakjLasK1xgKVMcY1McYa4Glg2gljpgFPJj8/B0wMDf8pOQ14OsZ4NMa4FqhMXu90Gl/rSeATTahRklKib9f2/N2UEcx/4EYevP1SCtvn8vVfL+XKf36RB375Pku37Et1iZJaqaa04OgDbGz0fRMw7lRjYoy1IYS9QPfk8TdPOLdP8nMEXgghROBHMcZHk8cTMcatyWttDSEUn6yoEMI9wD0A/fv3b8JtSFLLyc/J5ubL+nLzZX1ZvHkvP52/nuff2cRTCzdQNqArf3rVAKaO6kVeTlP+21iSmjaTdrKXK058RnqqMac795oY4xhgKnBvCGF8E2r5w0VifDTGWBZjLCsqKjqbUyWpRY3qU8g3b7mEBQ+U8z//pIQdB47y10+/y9XfeJF/nb2CLXsOp7pESa1AU0LaJqBfo+99gS2nGhNCyAEKgV2nOzfGePzf24Hn+cNj0KoQQq/ktXoB25t+O5KUPgrb5/Ln1w3mpf/nBp78s7GM7teFh16u5NpvvsQXflrBa6t2kAkr7CW1jKY87nwLGBZCGARsBqYDd54wZiZwFzAfuAV4KcYYQwgzgRkhhO/QsHBgGLAwhNAByIox7k9+/gjw9ROu9Y3kv//rfG5QklItKytw/fAirh9exMZdh5ixsGGhwewlVQwu6sBnxrnQQNKHNbUFx0eBf6OhBcfjMcZ/CiF8HaiIMc4MIRQAPwUuo2EGbXqMcU3y3P8X+DOgFvhKjPG3IYTBNMyeQUNQnBFj/Kfk+O7As0B/YANwa4xx1+nqc3WnpNbmyLE6fvP+Vn4yfz3vbjy+o0EfPj2uP6P6uKOBlMlsZitJrcT7m/by0zfXMfO9LRw5Vs8lfQu5c2x/bhrdm/Z5brEsZRpDmiS1MnsPH+P5tzcxY+EGVlYdoFN+Dp+4rA93jutPSa/OqS5PUjMxpElSKxVjZNH63cxYsIFfv7+Vmtp6LuvfhTvH9udjl/SmXZ77hUqtmSFNkjLAnkM1/OLtzfxswXrWVB+kc0EOnxzTl0+P68+wRKdUlyfpHBjSJCmDxBhZsHYXMxZsYNbibdTU1XPFwK7cOa4/U0f1oiDX2TWptTCkSVKG2nngKL94exMzFmxg3c5DdGmfyy1j+nLHuP4MKeqY6vIknYEhTZIyXH19ZP6ancxYsIHZS7ZRWx+5cnA37hw3gMkjE+TnOLsmpSNDmiS1IdX7j/LzRRt5auEGNu46TLcOedx6eV/uGNufgT06pLo8SY0Y0iSpDaqvj7xauYMZC9Yzd9l26uoj1w7twZ3j+jOpNEFuthu8S6lmSJOkNq5q3xGefathdm3L3iP06JjPbWUNs2v9urVPdXlSm2VIkyQBUFcfeWXldmYs2MBLy7cTgfHDirhzXH8mjigmx9k16YIypEmSPmTLnsM8/dZGnnlrA1X7jpLonM/tZf24fWx/+nRpl+rypDbBkCZJOqXaunpeWr6dGQs38MrKagIw4aJi7hzXnxsuKiY7K6S6RCljGdIkSU2ycdchnnlrI89UbKR6/1F6FxYwfWx/br+iH4nOBakuT8o4hjRJ0lk5VlfP3KVVzFi4gVdX7SA7KzBxRMPs2vhhRWQ5uyY1i6aGtJwLUYwkKf3lZmcx9eJeTL24F+t3HuSphRv5ecVGXlhaRd+u7bhjbH9uLetLcSdn16QLwZk0SdIp1dTWM3vJNmYs2MD8NTvJyQp8ZGSCO8cO4Ooh3Z1dk86BM2mSpPOWl5PFxy/tzccv7c2a6gM8tXADzy3axG/e38bA7u2ZPrY/t17el+4d81NdqpRxnEmTJJ2VI8fqmLW4YXZt4bpd5GYHpozqxZ1j+3Pl4G6E4OyadDouHJAktbhVVfuZsXADv1i0iX1Hahlc1IE7x/bnU2P60rVDXqrLk9KSIU2SdMEcOVbHr3+/lRkL1vP2hj3k5WTxJxf34s5x/Skb0NXZNakRQ5okKSWWb9vHjAUbeP7tzew/Wsuw4o7cOa4/n7ysL4Xtc1NdnpRyhjRJUkodqqnlv9/bwowFG3hv017yc7K4fngRUy/uyY0jEhS2M7CpbTKkSZLSxuLNe/l5xUZmLdlG1b6j5GYHrh7Sg6mjejKpNOHqULUphjRJUtqpr4+8u2kPsxZv47eLt7Jx12GyAowd1I2po3oxeWRPehbaLFeZzZAmSUprMUaWbt2XDGzbqNx+AIDL+ndh6qieTB3Vi37d2qe4Sqn5GdIkSa1K5fb9zFq8jVlLtrF48z4ARvbuzJSRPZl6cU+GFndKcYVS8zCkSZJarY27Dn3wSPTtDXsAGFLUgamjejFlVE9G9u5sWw+1WoY0SVJGqNp3hNlLtvHb97exYO1O6iP069aOKSN7MmVULy7r18U9RNWqGNIkSRln18Ea5ixteIft9codHKuLJDrnM3lkT6aM6snYgd3Iyc5KdZnSaRnSJEkZbe/hY8xbvp3fLt7KKyurOXKsnm4d8phUkmDKxT25ZkgP8nIMbEo/hjRJUptxqKaWV1ZU89vF23hp+XYOHK2lU34OE0uKmTKqF9cPL6JdXnaqy5QAQ5okqY06cqyON1bv4Lfvb2POsir2HDpGu9xsbrioiCmjenLjiGI6FbjbgVKnqSEt50IUI0nShVKQm82NIxLcOCLBsbp6Fq7dxW8Xb2X2kip+u3gbedlZXDusB1NG9WRSSYKuHfJSXbJ0Us6kSZLahLr6yNsbdjf0Ylu8jc17DpOdFbhycDemjOrF5JEJiju524Fano87JUk6hRgjizfv47eLtzJr8TbW7DhICFA2oOsHK0X7dnW3A7UMQ5okSU0QY2Rl1YEPmucu37YfgIv7FDJlVE+mjurJ4KKOKa5SmcSQJknSOVi34yCzljT0YntvY8NuB8MTHZkyqhdTR/VkRM9O7nag82JIkyTpPG3Zc7hht4PF23hr3S5ihIHd2zM5uQH8pX0LDWw6a4Y0SZKaUfX+o8xZWsVvF29l/uqd1NZHehUWMHlkwyPRsoHdyHZ7KjWBIU2SpBay51ANc5dtZ9bibfxuVTU1tfX06JjHpNKGRQdXDe7ubgc6JUOaJEkXwIGjtcxb3hDY5q3YzqGaOjoV5FBekmDyyJ7udqAPadaQFkKYAnwXyAZ+HGP8xgm/5wM/AS4HdgK3xxjXJX97ALgbqAP+KsY4u9F52UAFsDnG+LHksf8Argf2Jod9Lsb47unqM6RJktLBkWN1vLZqB7OWbGNucreDgtwsbhhezJRRPZkwopjCdu520NY1244DySD1EDAJ2AS8FUKYGWNc2mjY3cDuGOPQEMJ04JvA7SGEUmA6MBLoDcwNIQyPMdYlz/trYBnQ+YQ/+z9ijM+dqTZJktJJQW425aUJykv/sNvBrMXbmL1kG7OWbCM3O3D1kORuB6UJenTMT3XJSmNN2RZqLFAZY1wDEEJ4GpgGNA5p04B/SH5+DvhBaFjuMg14OsZ4FFgbQqhMXm9+CKEv8CfAPwFfa4Z7kSQpbeRmZ3HN0B5cM7QH//umkbyzcQ8vJFeKPvDL9/n759/nioHdmDyyJ5NHJmyeqw9pSkjrA2xs9H0TMO5UY2KMtSGEvUD35PE3Tzi3T/LzvwF/C3Q6yd/8pxDC/wJeBO5Phrw/EkK4B7gHoH///k24DUmSUiMrK3D5gK5cPqAr908dwfJt+z+YYfvHXy/lH3+99IPmuZNH9mRosc1z1bSQdrL1xCe+yHaqMSc9HkL4GLA9xrgohHDDCb8/AGwD8oBHgb8Dvv6hi8T4aPJ3ysrKWv/qB0lSmxBCoKRXZ0p6deark4azdsfBhsehi7fxL7NX8C+zVzC0uCNTkttTjezd2V5sbVRTQtomoF+j732BLacYsymEkAMUArtOc+5NwE0hhI8CBUDnEMJ/xhg/E2Pcmhx7NITwBPA3Z3lPkiS1GoN6dOCL1w/hi9cPYevew7ywpIpZi7fx8MuV/GBeJX26tGPKqIbANqZ/V3uxtSFnXN2ZDF0rgYnAZuAt4M4Y45JGY+4FLo4xfjG5cOCTMcbbQggjgRk0vIfWm4bHl8MaLRwgOZP2N41Wd/aKMW5NvtP2IHAkxnj/6Wp0dackKdPsOljD3KVVzFqyjddW7aCmrp4eHfP5yMgEU0b25Ep7sbVazba6M/mO2X3AbBpacDweY1wSQvg6UBFjnAk8Bvw0uTBgFw0rOkmOe5aGRQa1wL2NA9op/CyEUETDo9J3gS+eqUZJkjJNtw553HZFP267oh/7jxxj3opqZi/Zxq/e2cyMBRvonOzF9hF7sWUsm9lKktSKNO7FNmdpFXsP24uttWm2mTRJkpQ+7MXWdjiTJklSBqivj7yzcc8HK0U37DpEVoCygd2YMrInk0f1pE+XdqkuU7h3pyRJbVaMkWVb9zNryTZeWLKN5dv2A3BJ30ImJ1t7DCmyF1uqGNIkSRLAH/Vie3fjHgCGFXf8oHmuvdguLEOaJEn6kMa92Bas3Ul9xF5sF5ghTZIknZa92FLDkCZJkpqscS+2ecu3c6im7oNebJNH9WT8MHuxNRdDmiRJOicn68XWLjeba4f14Ooh3Rk3qDsjenYiy8ei58Q+aZIk6ZycqhfbvBXbmbO0CoAu7XMZO7Ab4wZ358rB3Sjp2dnQ1swMaZIk6ZRys7O4ZmgPrhnaA4BNuw+xYM0uFqzdyZtrdvFCMrR1Lshh7KCGwHbl4O6U9OrsAoTzZEiTJElN1rdre/pe3p5PXd4XgC17DjcEttUNwW3usobQ1qkgh7EDGwLblYO7U9rb0Ha2fCdNkiQ1m217jyRn2Rpm2tbuOAhAp/wcrhjUjXGDGoLbyN6dyclumytHfSdNkiRdcD0LC5g2ug/TRvcBoGrfkQ8C24K1O3lp+XYAOubnUDawK1cO7s64Qd24uE9hmw1tp2JIkyRJLSbR+Y9D2/Z9R1iwdlcyuO3k5RXVAHTIy6ZsYDfGJd9pu7hPIbltPLT5uFOSJKXM9v1HWLh2FwvWNAS3VdsPANA+L5vLB3RNvtPWjYv7dMmYxrr2SZMkSa3OjgNHWZicaVuwZhcrqho2h2+Xm03ZwK4fvNN2Sd/WG9oMaZIkqdXbmQxtxx+RLt/WENoKcrMaZtoGdWfc4O5c2q+Q/JzWsSOCIU2SJGWcXQdr/jDTtnYXy7buAyA/J4sx/f/wePTSfl0oyE3P0GZIkyRJGW/PoeOhrSG4Ldu2jxghLyeLMf27MG5QQ5+2y/qnT2gzpEmSpDZn76FjLFx3fKZtJ0u2/CG0je7XpWGmbVA3xgzomrLQZkiTJElt3t7Dx6hYt+uDXm1LtuylPkJedkNoO97yY0z/rrTLuzChzZAmSZJ0gn1Hjoe2XSxYs5P3NzeEttzswKV9u/C1jwzn6iE9WrQGdxyQJEk6QeeCXG4ckeDGEQkA9h85RsX63R+0/EinBrqGNEmS1GZ1KshlwkXFTLioONWlfEj6xEVJkiR94P9v735+5ZrjMI6/n/QSWhESNtpGKxE0Eqk0UppYqAUhbElYWPtRIpHyN4iwEIkUGw2L6kJEsGDdoJVQl6Qp2ktFLfyITTU+FjOSLiRzz63b7/eM92s1czLJPMknc84z53zPjCVNkiSpQ5Y0SZKkDlnSJEmSOmRJkyRJ6pAlTZIkqUOWNEmSpA5Z0iRJkjpkSZMkSeqQJU2SJKlDljRJkqQOWdIkSZI6ZEmTJEnqkCVNkiSpQ5Y0SZKkDlnSJEmSOmRJkyRJ6pAlTZIkqUOpqtYZzlqSk8B3q/w2lwE/r/J7aHU5w/FzhuPm/MbPGf43rqyqy2e9aC5K2rmQ5JOq2tY6h1bOGY6fMxw35zd+zvDc8nKnJElShyxpkiRJHbKkLd/LrQPorDnD8XOG4+b8xs8ZnkOuSZMkSeqQZ9IkSZI6ZEmTJEnqkCVtGZLckeTrJEeS7G6dR8Mk2ZjkoySLSQ4n2dU6k4ZLsibJoSTvtM6i4ZJckmRfkq+mn8WbW2fSMEmemO5Dv0jyRpILWmead5a0GZKsAV4E7gS2APcn2dI2lQY6DTxZVdcB24GHneEo7QIWW4fQir0AvFdV1wI34CxHJcl64DFgW1VdD6wB7mubav5Z0ma7CThSVUer6hTwJnBv40waoKpOVNXB6ePfmRwc1rdNpSGSbADuAva0zqLhklwM3Aq8AlBVp6rql7aptAILwIVJFoC1wA+N88w9S9ps64HjZzxfwgP8aCXZBGwFDrRNooGeB54C/modRCtyFXASeG16yXpPknWtQ2n5qup74FngGHAC+LWqPmibav5Z0mbLv2zzd0tGKMlFwFvA41X1W+s8Wp4kdwM/VdWnrbNoxRaAG4GXqmor8Afg+t4RSXIpk6tIm4ErgHVJHmibav5Z0mZbAjae8XwDnuIdnSTnMSloe6tqf+s8GmQHcE+Sb5ksN7gtyettI2mgJWCpqv45g72PSWnTeNwOfFNVJ6vqT2A/cEvjTHPPkjbbx8DVSTYnOZ/JQsm3G2fSAEnCZC3MYlU91zqPhqmqp6tqQ1VtYvL5+7Cq/AY/IlX1I3A8yTXTTTuBLxtG0nDHgO1J1k73qTvx5o9Vt9A6QO+q6nSSR4D3mdzN8mpVHW4cS8PsAB4EPk/y2XTbM1X1bsNM0v/No8De6Zfdo8BDjfNogKo6kGQfcJDJHfOH8C+iVp1/CyVJktQhL3dKkiR1yJImSZLUIUuaJElShyxpkiRJHbKkSZIkdciSJkmS1CFLmiRJUof+BgtT3sWjzSusAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,8))\n",
    "plt.plot(tl_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.8899875,\n",
       " 0.9192875,\n",
       " 0.92635,\n",
       " 0.93205,\n",
       " 0.93545,\n",
       " 0.938625,\n",
       " 0.942275,\n",
       " 0.9442,\n",
       " 0.947225,\n",
       " 0.948375]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ta_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.9057, 0.914, 0.9124, 0.9149, 0.914, 0.9148, 0.9138, 0.9124, 0.9134, 0.9128]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "va_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"model_1.pkl\",\"rb\") as file:\n",
    "    model_1 = pkl.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"model_9.pkl\",\"rb\") as file:\n",
    "    model_9 = pkl.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model_1.fc.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model_9.fc.weight.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#w1 = model_1.fc.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#w2 = model_9.fc.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#w9 = model_9.fc.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "#torch.all(torch.eq(w1, w2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#yelp_dataset.data[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_loss, test_acc = test_func(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9184"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}