Employee  SVM KNN DT RF (1).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Employee Attrition prediction using KNN, RF, DT and SVM with up-sampling to the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.gridspec import GridSpec\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.metrics import (accuracy_score,\n",
    "                             f1_score,\n",
    "                             roc_auc_score,\n",
    "                             roc_curve,\n",
    "                             confusion_matrix)\n",
    "from sklearn.model_selection import (cross_val_score,\n",
    "                                     GridSearchCV,\n",
    "                                     RandomizedSearchCV,\n",
    "                                     learning_curve,\n",
    "                                     validation_curve,\n",
    "                                     train_test_split)\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.utils import resample\n",
    "from warnings import filterwarnings\n",
    "\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[94mData types:\n",
      "-----------\n",
      "\u001b[30msatisfaction_level       float64\n",
      "last_evaluation          float64\n",
      "number_project             int64\n",
      "average_montly_hours       int64\n",
      "time_spend_company         int64\n",
      "Work_accident              int64\n",
      "left                       int64\n",
      "promotion_last_5years      int64\n",
      "sales                     object\n",
      "salary                    object\n",
      "dtype: object\n",
      "\n",
      "\u001b[1m\u001b[94mSum of null values in each column:\n",
      "-----------------------------------\n",
      "\u001b[30msatisfaction_level       0\n",
      "last_evaluation          0\n",
      "number_project           0\n",
      "average_montly_hours     0\n",
      "time_spend_company       0\n",
      "Work_accident            0\n",
      "left                     0\n",
      "promotion_last_5years    0\n",
      "sales                    0\n",
      "salary                   0\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>satisfaction_level</th>\n",
       "      <th>last_evaluation</th>\n",
       "      <th>number_project</th>\n",
       "      <th>average_montly_hours</th>\n",
       "      <th>time_spend_company</th>\n",
       "      <th>Work_accident</th>\n",
       "      <th>left</th>\n",
       "      <th>promotion_last_5years</th>\n",
       "      <th>sales</th>\n",
       "      <th>salary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.38</td>\n",
       "      <td>0.53</td>\n",
       "      <td>2</td>\n",
       "      <td>157</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>sales</td>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.80</td>\n",
       "      <td>0.86</td>\n",
       "      <td>5</td>\n",
       "      <td>262</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>sales</td>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.11</td>\n",
       "      <td>0.88</td>\n",
       "      <td>7</td>\n",
       "      <td>272</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>sales</td>\n",
       "      <td>medium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.72</td>\n",
       "      <td>0.87</td>\n",
       "      <td>5</td>\n",
       "      <td>223</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>sales</td>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.37</td>\n",
       "      <td>0.52</td>\n",
       "      <td>2</td>\n",
       "      <td>159</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>sales</td>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   satisfaction_level  last_evaluation  number_project  average_montly_hours  \\\n",
       "0                0.38             0.53               2                   157   \n",
       "1                0.80             0.86               5                   262   \n",
       "2                0.11             0.88               7                   272   \n",
       "3                0.72             0.87               5                   223   \n",
       "4                0.37             0.52               2                   159   \n",
       "\n",
       "   time_spend_company  Work_accident  left  promotion_last_5years  sales  \\\n",
       "0                   3              0     1                      0  sales   \n",
       "1                   6              0     1                      0  sales   \n",
       "2                   4              0     1                      0  sales   \n",
       "3                   5              0     1                      0  sales   \n",
       "4                   3              0     1                      0  sales   \n",
       "\n",
       "   salary  \n",
       "0     low  \n",
       "1  medium  \n",
       "2  medium  \n",
       "3     low  \n",
       "4     low  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "import pandas as pd # data processing, CSV file I/O\n",
    "import numpy as np # linear algebra\n",
    "import sklearn\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from pandas.plotting import scatter_matrix\n",
    "import seaborn as sns\n",
    "\n",
    "# Load the data\n",
    "df = pd.read_csv('Downloads/HR_comma_sep.csv')\n",
    "# Check both the datatypes and if there is missing values\n",
    "print('\\033[1m' + '\\033[94m' + 'Data types:\\n' + 11 * '-')\n",
    "print('\\033[30m' + '{}\\n'.format(df.dtypes))\n",
    "print('\\033[1m' + '\\033[94m' + 'Sum of null values in each column:\\n' + 35 * '-')\n",
    "print('\\033[30m' + '{}'.format(df.isnull().sum()))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* There are no missing values, no imputation required. However, data preprocessing is needed:\n",
    "* Changing sales feature name to department.\n",
    "* Convert salary into ordinal categorical feature since there is intrinsic order between: low, medium and high.\n",
    "* Create dummy features from department feature and drop the first one to avoid linear dependency where some learning algorithms may struggle.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename sales feature into department\n",
    "df = df.rename(columns={\"sales\": \"department\"})\n",
    "# Map salary into integers\n",
    "salary_map = {\"low\": 0, \"medium\": 1, \"high\": 2}\n",
    "df[\"salary\"] = df[\"salary\"].map(salary_map)\n",
    "# Create dummy variables for department feature\n",
    "df = pd.get_dummies(df, columns=[\"department\"], drop_first=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modeling\n",
    "Let’s look at the ratio of each class to see if there are balanced or imbalanced data, since each one has its own set of tools to be used when fitting classifiers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Positive examples = 3571\n",
      "Negative examples = 11428\n",
      "Proportion of positive to negative examples = 31.25%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\seaborn\\_decorators.py:36: FutureWarning: Pass the following variable as a keyword arg: x. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWPUlEQVR4nO3dfbTdVX3n8ffHIE8FHDAXigkYkEyV0AUdUgTtuLS0Fa1tmI5orArOYjVTi0/FsQXr+LSaLq1Oh9IKnbQ6gHWJ0aE16qBiFLVTBAOiGChDFIFIhGh9CHVEw3znj7MzHm7uTW6yc+/JzX2/1vqt8zvfvffv7JN14JPfY1JVSJK0ux4z6glIkmY3g0SS1MUgkSR1MUgkSV0MEklSF4NEktTFIJEmkOTNSf521POQZgODRHNWkt9Osi7JQ0k2Jbk2yS+Nel4zJUklOWHU89DsZ5BoTkpyIXAJ8CfAUcCxwGXAshFOS5qVDBLNOUkeB7wVuKCqrqmqf6mqn1TVR6rqdZOM+WCSbyX5fpLPJVky1PbcJLcn2ZLkm0n+U6vPT/LRJN9L8s9JPp9kwv/mkixJcl3r90CS17f6AUkuSXJ/Wy5JckBre1mSfxi3nf+/l5HkiiTvSvKxNrcbkzyptX2uDfly2yN74a7MVxrmj0Rz0RnAgcDf7cKYa4HFwJHALcD7htreDfzHqjoUOAn4dKu/FtgIjDHY63k9sN0ziZIcCnwK+DjwBOAEYG1r/iPgdOAU4GTgNOANuzDvFwFvAQ4HNgArAarqGa395Ko6pKo+MNX5SuMZJJqLHg98u6q2TnVAVb2nqrZU1cPAm4GT254NwE+AE5McVlXfrapbhupHA09sezyfr4kfbvc84FtV9V+q6kftc25sbS8G3lpVD1bVZgah8NJd+K7XVNVN7bu+j0EgTWaq85UexSDRXPQdYH6S/abSOcm8JG9L8rUkPwC+0Zrmt9d/DzwXuCfJZ5Oc0ervYLAX8MkkX09y0SQfcQzwtUnangDcM/T+nlabqm8Nrf8QOGQHfac6X+lRDBLNRTcAPwLOnmL/32ZwEv5XgMcBi1o9AFX1xapaxuCw198Dq1t9S1W9tqqOB34DuDDJmRNs/z7gSZN89v3AE4feH9tqAP8CHLytIcnPTvH7TGgX5is9ikGiOaeqvg+8EXhXkrOTHJzksUmek+RPJxhyKPAwgz2Zgxlc6QVAkv2TvDjJ46rqJ8APgEda2/OSnJAkQ/VHJtj+R4GfTfKadnL90CRPbW3vB96QZCzJ/Dbvbfe3fBlYkuSUJAcyOOS2Kx4Ajh/6LlOdr/QoBonmpKr6M+BCBieuNzPYK3gFgz2K8a5icEjpm8DtwBfGtb8U+EY77PW7wEtafTGDk+gPMdgLuqyqrp9gLluAX2WwF/At4C7gWa35j4F1wFeA2xic6P/jNu5/M7j67FNtzKOu4JqCNwNXtqu0XjDV+UrjxXNpkqQe7pFIkroYJJKkLgaJJKmLQSJJ6jKlG7L2JfPnz69FixaNehqSNKvcfPPN366qsYna5lyQLFq0iHXr1o16GpI0qyS5Z7I2D21JkroYJJKkLgaJJKmLQSJJ6mKQSJK6GCSSpC4GiSSpi0EiSepikEiSusy5O9v3hFNfd9Wop6C90M3vOHfUU5BGwj0SSVIXg0SS1MUgkSR1MUgkSV0MEklSF4NEktTFIJEkdTFIJEldDBJJUheDRJLUxSCRJHUxSCRJXQwSSVIXg0SS1MUgkSR1MUgkSV0MEklSF4NEktTFIJEkdTFIJEldDBJJUpdpC5Ik70nyYJKvDtWOSHJdkrva6+FDbRcn2ZDkziTPHqqfmuS21nZpkrT6AUk+0Oo3Jlk0Xd9FkjS56dwjuQI4a1ztImBtVS0G1rb3JDkRWA4saWMuSzKvjbkcWAEsbsu2bZ4PfLeqTgD+K/D2afsmkqRJTVuQVNXngH8eV14GXNnWrwTOHqpfXVUPV9XdwAbgtCRHA4dV1Q1VVcBV48Zs29aHgDO37a1IkmbOTJ8jOaqqNgG01yNbfQFw31C/ja22oK2Prz9qTFVtBb4PPH7aZi5JmtDecrJ9oj2J2kF9R2O233iyIsm6JOs2b968m1OUJE1kpoPkgXa4ivb6YKtvBI4Z6rcQuL/VF05Qf9SYJPsBj2P7Q2kAVNWqqlpaVUvHxsb20FeRJMHMB8ka4Ly2fh7w4aH68nYl1nEMTqrf1A5/bUlyejv/ce64Mdu29Xzg0+08iiRpBu03XRtO8n7gmcD8JBuBNwFvA1YnOR+4FzgHoKrWJ1kN3A5sBS6oqkfapl7O4Aqwg4Br2wLwbuC9STYw2BNZPl3fRZI0uWkLkqp60SRNZ07SfyWwcoL6OuCkCeo/ogWRJGl09paT7ZKkWcogkSR1MUgkSV0MEklSF4NEktTFIJEkdTFIJEldDBJJUheDRJLUxSCRJHUxSCRJXQwSSVIXg0SS1MUgkSR1MUgkSV0MEklSF4NEktTFIJEkdTFIJEldDBJJUheDRJLUxSCRJHUxSCRJXQwSSVIXg0SS1MUgkSR1MUgkSV0MEklSl5EESZLfT7I+yVeTvD/JgUmOSHJdkrva6+FD/S9OsiHJnUmePVQ/Ncltre3SJBnF95GkuWzGgyTJAuBVwNKqOgmYBywHLgLWVtViYG17T5ITW/sS4CzgsiTz2uYuB1YAi9ty1gx+FUkSozu0tR9wUJL9gIOB+4FlwJWt/Urg7La+DLi6qh6uqruBDcBpSY4GDquqG6qqgKuGxkiSZsiMB0lVfRN4J3AvsAn4flV9Ejiqqja1PpuAI9uQBcB9Q5vY2GoL2vr4+naSrEiyLsm6zZs378mvI0lz3igObR3OYC/jOOAJwM8kecmOhkxQqx3Uty9WraqqpVW1dGxsbFenLEnagVEc2voV4O6q2lxVPwGuAZ4GPNAOV9FeH2z9NwLHDI1fyOBQ2Ma2Pr4uSZpBowiSe4HTkxzcrrI6E7gDWAOc1/qcB3y4ra8Blic5IMlxDE6q39QOf21JcnrbzrlDYyRJM2S/mf7AqroxyYeAW4CtwJeAVcAhwOok5zMIm3Na//VJVgO3t/4XVNUjbXMvB64ADgKubYskaQbNeJAAVNWbgDeNKz/MYO9kov4rgZUT1NcBJ+3xCUqSpsw72yVJXQwSSVIXg0SS1MUgkSR1MUgkSV0MEklSF4NEktTFIJEkdTFIJEldDBJJUheDRJLUxSCRJHUxSCRJXQwSSVIXg0SS1MUgkSR1MUgkSV0MEklSlykFSZKnT6UmSZp7prpH8hdTrEmS5pj9dtSY5AzgacBYkguHmg4D5k3nxCRJs8MOgwTYHzik9Tt0qP4D4PnTNSlJ0uyxwyCpqs8Cn01yRVXdM0NzkiTNIjvbI9nmgCSrgEXDY6rql6djUpKk2WOqQfJB4K+AvwEemb7pSJJmm6kGydaqunxaZyJJmpWmevnvR5L8XpKjkxyxbZnWmUmSZoWpBsl5wOuAfwRubsu63f3QJP8qyYeS/FOSO5Kc0cLpuiR3tdfDh/pfnGRDkjuTPHuofmqS21rbpUmyu3OSJO2eKQVJVR03wXJ8x+f+OfDxqnoycDJwB3ARsLaqFgNr23uSnAgsB5YAZwGXJdl2D8vlwApgcVvO6piTJGk3TOkcSZJzJ6pX1VW7+oFJDgOeAbysbePHwI+TLAOe2bpdCVwP/CGwDLi6qh4G7k6yATgtyTeAw6rqhrbdq4CzgWt3dU6SpN031ZPtvzi0fiBwJnALsMtBAhwPbAb+e5KTGRwmezVwVFVtAqiqTUmObP0XAF8YGr+x1X7S1sfXt5NkBYM9F4499tjdmLIkaTJTCpKqeuXw+ySPA97b8Zn/BnhlVd2Y5M9ph7EmMdF5j9pBffti1SpgFcDSpUsn7CNJ2j27+xj5HzI4J7E7NgIbq+rG9v5DDILlgSRHA7TXB4f6HzM0fiFwf6svnKAuSZpBU32M/EeSrGnLx4A7gQ/vzgdW1beA+5L8XCudCdwOrGFwdRjtddv21wDLkxyQ5DgGAXZTOwy2Jcnp7Wqtc3d3TpKk3TfVcyTvHFrfCtxTVRsn6zwFrwTel2R/4OvAf2AQaquTnA/cC5wDUFXrk6xmEDZbgQuqatvd9S8HrgAOYnCS3RPtkjTDpnqO5LNJjuKnJ93v6vnQqroVWDpB05mT9F8JrJygvg44qWcukqQ+Uz209QLgJgZ7CS8AbkziY+QlSVM+tPVHwC9W1YMAScaATzE4US5JmsOmetXWY7aFSPOdXRgrSdqHTXWP5ONJPgG8v71/IfA/p2dKkqTZZGf/ZvsJDO44f12S3wJ+icGNgDcA75uB+UmS9nI7Ozx1CbAFoKquqaoLq+r3GeyNXDK9U5MkzQY7C5JFVfWV8cV22e2iaZmRJGlW2VmQHLiDtoP25EQkSbPTzoLki0l+Z3yx3X1+8/RMSZI0m+zsqq3XAH+X5MX8NDiWAvsD/24a5yVJmiV2GCRV9QDwtCTP4qePIvlYVX162mcmSZoVpvqsrc8An5nmuUiSZiHvTpckdTFIJEldDBJJUheDRJLUxSCRJHUxSCRJXQwSSVIXg0SS1MUgkSR1MUgkSV0MEklSF4NEktTFIJEkdTFIJEldDBJJUpeRBUmSeUm+lOSj7f0RSa5Lcld7PXyo78VJNiS5M8mzh+qnJrmttV2aJKP4LpI0l41yj+TVwB1D7y8C1lbVYmBte0+SE4HlwBLgLOCyJPPamMuBFcDitpw1M1OXJG0zkiBJshD4deBvhsrLgCvb+pXA2UP1q6vq4aq6G9gAnJbkaOCwqrqhqgq4amiMJGmGjGqP5BLgD4D/O1Q7qqo2AbTXI1t9AXDfUL+NrbagrY+vbyfJiiTrkqzbvHnzHvkCkqSBGQ+SJM8DHqyqm6c6ZIJa7aC+fbFqVVUtraqlY2NjU/xYSdJU7DeCz3w68JtJngscCByW5G+BB5IcXVWb2mGrB1v/jcAxQ+MXAve3+sIJ6pKkGTTjeyRVdXFVLayqRQxOon+6ql4CrAHOa93OAz7c1tcAy5MckOQ4BifVb2qHv7YkOb1drXXu0BhJ0gwZxR7JZN4GrE5yPnAvcA5AVa1Pshq4HdgKXFBVj7QxLweuAA4Crm2LJGkGjTRIqup64Pq2/h3gzEn6rQRWTlBfB5w0fTOUJO2Md7ZLkroYJJKkLnvTORJJne5968+PegraCx37xtumdfvukUiSuhgkkqQuBokkqYtBIknqYpBIkroYJJKkLgaJJKmLQSJJ6mKQSJK6GCSSpC4GiSSpi0EiSepikEiSuhgkkqQuBokkqYtBIknqYpBIkroYJJKkLgaJJKmLQSJJ6mKQSJK6GCSSpC4GiSSpi0EiSeoy40GS5Jgkn0lyR5L1SV7d6kckuS7JXe318KExFyfZkOTOJM8eqp+a5LbWdmmSzPT3kaS5bhR7JFuB11bVU4DTgQuSnAhcBKytqsXA2vae1rYcWAKcBVyWZF7b1uXACmBxW86ayS8iSRpBkFTVpqq6pa1vAe4AFgDLgCtbtyuBs9v6MuDqqnq4qu4GNgCnJTkaOKyqbqiqAq4aGiNJmiEjPUeSZBHwC8CNwFFVtQkGYQMc2botAO4bGrax1Ra09fF1SdIMGlmQJDkE+B/Aa6rqBzvqOkGtdlCf6LNWJFmXZN3mzZt3fbKSpEmNJEiSPJZBiLyvqq5p5Qfa4Sra64OtvhE4Zmj4QuD+Vl84QX07VbWqqpZW1dKxsbE990UkSSO5aivAu4E7qurPhprWAOe19fOADw/Vlyc5IMlxDE6q39QOf21Jcnrb5rlDYyRJM2S/EXzm04GXArclubXVXg+8DVid5HzgXuAcgKpan2Q1cDuDK74uqKpH2riXA1cABwHXtkWSNINmPEiq6h+Y+PwGwJmTjFkJrJygvg44ac/NTpK0q7yzXZLUxSCRJHUxSCRJXQwSSVIXg0SS1MUgkSR1MUgkSV0MEklSF4NEktTFIJEkdTFIJEldDBJJUheDRJLUxSCRJHUxSCRJXQwSSVIXg0SS1MUgkSR1MUgkSV0MEklSF4NEktTFIJEkdTFIJEldDBJJUheDRJLUxSCRJHUxSCRJXQwSSVKXWR8kSc5KcmeSDUkuGvV8JGmumdVBkmQe8C7gOcCJwIuSnDjaWUnS3DKrgwQ4DdhQVV+vqh8DVwPLRjwnSZpT9hv1BDotAO4ber8ReOr4TklWACva24eS3DkDc5sr5gPfHvUk9gZ553mjnoIezd/mNm/KntjKEydrmO1BMtGfTm1XqFoFrJr+6cw9SdZV1dJRz0Maz9/mzJnth7Y2AscMvV8I3D+iuUjSnDTbg+SLwOIkxyXZH1gOrBnxnCRpTpnVh7aqamuSVwCfAOYB76mq9SOe1lzjIUPtrfxtzpBUbXdKQZKkKZvth7YkSSNmkEiSuhgk+5AkjyS5Ncn6JF9OcmGSx7S2pUkunWTcN5LM38m2r0jyzLb+miQHT9Lv+iRecqlpl+ShXeg7luTGJF9K8m+T/N50zm2uMUj2Lf+nqk6pqiXArwLPBd4EUFXrqupVe+hzXgNMGCTSXupM4J+q6hcY3MRskOxBBsk+qqoeZHA3/ysy8MwkHwVI8vgkn2x/O/tvtBs7kyxKckeSv257NZ9MclDb5PeBHyd5FfAE4DNJPrOjOST5tSQ3JLklyQeTHNLqb0zyxSRfTbKqze8pSW4aGrsoyVfa+qlJPpvk5iSfSHL0Hv7j0j4iyZOSfLz9Vj6f5MlJTgH+FHhukluBtwNPanvv7xjlfPcZVeWyjyzAQxPUvgscBTwT+GirXQq8sa3/OoOnAcwHFgFbgVNa22rgJRNs8xvA/EnmcD2wtG3vc8DPtPofDn3mEUP93wv8Rlu/FTh+qP8bgMcC/wiMtfoLGVzmPfI/b5fRLpP83tcCi9v6U4FPt/WXAX/Z1hcBXx31/PelZVbfR6IpmegxMs8Afgugqj6W5LtDbXdX1a1t/WYG/9HtjtMZPJH5fyUB2B+4obU9K8kfMDg8dgSwHvgIg+B6AfA2BoHxQuDngJOA69p25gGbdnNO2oe1Pd6nAR9svxWAA0Y3o7nDINmHJTkeeAR4EHjKuObJbiB6eGj9EeCgSfrt9OOB66rqRePmdCBwGbC0qu5L8mbgwNb8AQb/E7gGqKq6K8nPA+ur6ozdnIfmjscA36uqU0Y9kbnGcyT7qCRjwF8x2J0fHxqfA17c+j0HOHwXN78FOHQnfb4APD3JCe1zDk7yr/lpaHy7/Q3y+dsGVNXXGITXf2YQKgB3AmNJzmjbeWySJbs4X80BVfUD4O4k5wC0c28nT9B1Kr9f7QKDZN9y0LbLf4FPAZ8E3jJBv7cAz0hyC/BrwL27+DmrgGt3dLK9qjYzOC79/nbS/AvAk6vqe8BfA7cBf8/geWnDPgC8hMFhLmrw78w8H3h7ki8zOI/ytF2cr/ZNByfZOLRcyOAvSOe338p6Jvj3iarqOwwOuX7Vk+17ho9IkSR1cY9EktTFIJEkdTFIJEldDBJJUheDRJLUxSCRZohPq9W+yiCR9k4+rVazhveRSDMkyUNVdci42pOAdwFjwA+B32Fw9/8aBo+n+SaDu/t/s71eV1Wvm8l5Szvjs7ak0VoF/G57rthTgcuq6peTvJHB88hekWQRsMRnSGlvZZBII+LTarWvMEik0fFptdoneLJdGhGfVqt9hUEizRyfVqt9kldtSZK6uEciSepikEiSuhgkkqQuBokkqYtBIknqYpBIkroYJJKkLv8PqHwr+Y7tSdIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get number of positve and negative examples\n",
    "pos = df[df[\"left\"] == 1].shape[0]\n",
    "neg = df[df[\"left\"] == 0].shape[0]\n",
    "print(\"Positive examples = {}\".format(pos))\n",
    "print(\"Negative examples = {}\".format(neg))\n",
    "print(\"Proportion of positive to negative examples = {:.2f}%\".format((pos / neg) * 100))\n",
    "sns.countplot(df[\"left\"])\n",
    "plt.xticks((0, 1), [\"Didn't leave\", \"Left\"])\n",
    "plt.xlabel(\"Left\")\n",
    "plt.ylabel(\"Count\")\n",
    "plt.title(\"Class counts\");"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The above graph shows, we have an imbalanced dataset. As a result, when we fit classifiers on such datasets, we should use metrics other than accuracy when comparing models such as f1-score or AUC (area under ROC curve). Moreover, class imbalance influences a learning algorithm during training by making the decision rule biased towards the majority class by implicitly learns a model that optimizes the predictions based on the majority class in the dataset. There are three ways to deal with this issue:\n",
    "* 1- Assign a larger penalty to wrong predictions from the minority class.\n",
    "* 2- Upsampling the minority class or downsampling the majority class.\n",
    "* 3- Generate synthetic training examples."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: resample in c:\\programdata\\anaconda3\\lib\\site-packages (1.0.1)\n",
      "Requirement already satisfied: scipy>=1.5 in c:\\programdata\\anaconda3\\lib\\site-packages (from resample) (1.5.2)\n",
      "Requirement already satisfied: numpy>=1.17 in c:\\programdata\\anaconda3\\lib\\site-packages (from resample) (1.19.2)\n"
     ]
    }
   ],
   "source": [
    "! pip install resample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original shape: (11999, 17) (11999,)\n",
      "Upsampled shape: (18284, 17) (18284,)\n",
      "Downsampled shape: (5714, 17) (5714,)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from pandas.tseries.frequencies import to_offset\n",
    "from sklearn.utils import resample\n",
    "\n",
    "# Convert dataframe into numpy objects and split them into\n",
    "# train and test sets: 80/20\n",
    "X = df.loc[:, df.columns != \"left\"].values\n",
    "y = df.loc[:, df.columns == \"left\"].values.flatten()\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.2, stratify=y, random_state=1)\n",
    "# Upsample minority class\n",
    "X_train_u, y_train_u = resample(X_train[y_train == 1],\n",
    "                                y_train[y_train == 1],\n",
    "                                replace=True,\n",
    "                                n_samples=X_train[y_train == 0].shape[0],\n",
    "                                random_state=1)\n",
    "X_train_u = np.concatenate((X_train[y_train == 0], X_train_u))\n",
    "y_train_u = np.concatenate((y_train[y_train == 0], y_train_u))\n",
    "# Downsample majority class\n",
    "X_train_d, y_train_d = resample(X_train[y_train == 0],\n",
    "                                y_train[y_train == 0],\n",
    "                                replace=True,\n",
    "                                n_samples=X_train[y_train == 1].shape[0],\n",
    "                                random_state=1)\n",
    "X_train_d = np.concatenate((X_train[y_train == 1], X_train_d))\n",
    "y_train_d = np.concatenate((y_train[y_train == 1], y_train_d))\n",
    "print(\"Original shape:\", X_train.shape, y_train.shape)\n",
    "print(\"Upsampled shape:\", X_train_u.shape, y_train_u.shape)\n",
    "print(\"Downsampled shape:\", X_train_d.shape, y_train_d.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lets see how many principal components needed to explain 90%, 95% and 99% of the variation in the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtQAAAGJCAYAAACq1aREAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABCU0lEQVR4nO3deZzd493/8ddH0CCiIdF7hAgay4QIIpJb7LUUd6wtWr1pb/tStG66qFjvX1u0mrakoYTaopYGRSOtpdVEQ6SWISgRiSBBRcSW5Pr98f3OOJnMciZn5pwzyev5eMwj57u/z3fG+Mx1ru91RUoJSZIkSctmpUoHkCRJkjozC2pJkiSpBBbUkiRJUgksqCVJkqQSWFBLkiRJJbCgliRJkkpgQS2pU4mIoyMiFXy9HxH/jIhTImLlRvuuEhEnRcSjEfHviPg4Il6JiGsiYttmzv9ift7h5XlHxYmIAyPiOx107jERMX0Zj50eEWPaN9Gyy7935y3DcUdHxLc6IJKkFcDKre8iSVXpK8BMoHv++pfAusC5ABGxBnAfsD0wCvg/YD7wReBI4M9Aj8ITRsSO+XaAo4C7OvpNtMGBwJeAn3XAuS8EfrGMxx4EzGvHLJVyNNn/E6+pcA5JnZAFtaTOampK6aX89fiI+CJwOnlBTVYg7gDsmlKaWHDcw8BvI+KgJs55FLAQ+Auwf0SsnVJ6p0PSd6CI+FxK6eNi908p/WtZr5VSenJZj5Wk5YVdPiQtLyYDa0bEuhFRQ9bieFWjYrpBSunOwuWI6Ap8FRgPXAKsChxe7MUj4si868lHETE3In6X5yjcZ3pE3BARh0fEcxHxQUQ8HhHDWjn3GLJiv3dBV5fp+bZd8+WDI+KqiJgDvJlv+2Ke45WI+DAiXo6IKyOiccv8El0+IqJvfs7jI+KCiJidd5m5OyLWb+I9jSlYru+SMyQiboyIeRHxekSMzO9x4bEbR8S9EbEgIt6KiMsi4rj8+L6t3JMuEXFRnm1BRDwUEf2b2K/VexARDwG7ADsW3N+H8m29IuI3EfFCfp3XIuKmiOjdUj5JKxZbqCUtLzYCFpF16zgQ6ELbumwcCKwFXE/WQj2TrIi9orUDI+I44DfAWOD7wHpkXUx2iIhtU0rzC3bfCdgM+BHwEVl3i3siom9K6d/NXOJCoBdZ95X6vt2NW6B/SdbF5RtAfeG6Xv4+TgfeBTYGfgDcCwxt7X3l7+XvwLfIutNcBtxIVny25nfAzcDB+bXOyzOMAIiIVYEH8qwnAW8BxwCHFnFu8vP9gKwLzHhgEE1/v4u5BycBN5D9zByfr6vvxrI22ffp+8Cc/HzfBR6NiM1TSh8VmVfScsyCWlJn1SV/CHFNspblg4G7U0oLImKDfJ9X23C+o4D3gHEppcURcQPwvbxoer65gyKiC1nB+1BK6fCC9c8DfyUrRkcWHNIdGJhSejff7w2y1vV9gZuaukZK6V95y/MnKaVJzUT5R0rpmEbHPQI8UpDp78BLwF8jYpsiumu8mlL6WsHxvYBLImK9lNLrrRx7U0ppRP56QkTsABxBXlCTfYKwMbBDSukf+fnvA6YCfVo6cd66fAYwOqV0Zr56fEQsAn5cuG8x9yClVBcR84CVG9/flNI04LSC47sAjwIzgC8DS3zSIWnFZJcPSZ3V88CnwDtkrcg3khWvbZZ3zdgT+H1Bi+N1+b//3crhm5G13t5YuDKl9Deygr5xa+7E+mI693T+b4tFZBGWKuwiYtWI+EFEPB8RH5Ldr78W5G7NHxsttyVrU8cWHjcEmFFfTAOklBJwexHn3gpYA7i10fpbGu/YDveAiDgx784zn6yP/Yy2HC9p+WdBLamzOoisC8TmwBoppf8ueIDwtfzfDYs815FkH/ePi4jPR8TngTfIWku/EREt/a5cO/93dhPb3ijYXm+JhxwLHh7sSmmauv7/I+sacQOwHzCYrCW/2Os1fiCzLVmbOvZzBcs1ZN08GnuziHPX901vvG9Tx5Z0DyLiVLI/2Cbkxw0m+2OgqOMlrRjs8iGps3qmYJSPxh4i60/9X2T9a1tT3wp9dzPbdycrqJpSXzj+RxPb/gN4vIjrt4fUxLrDgetTShfVr4iIbmXK05rZQG0T679Q5LH1+z7byrGl3oPDgT+nlL5bcPxGbThe0grAFmpJy528f+8Y4LiIaPLhu4g4MP93O2BLsocKd2v0tTdZy2pL3T6mkbWMLjEiSET8J1kL+cPL/k6W8DGwWhuPWZ2si0Ohb7ZPnJJNAvpExOD6FRERwCFFHPsU8AFZ3/lCTY3KUuw9aO7+VvM9lFQlbKGWtLw6HdgU+HNEjCJrYZ5P9iDc18lGhfgD2cOICfhJSumVxieJiD8AB0fESY1G6wAgpbQoIs4FfpM/yHgD0Bu4GHgRuLad3k8dsHZEnEjW6v1RSunpVo65HzgqIp4mexDvYOA/2ylPqcYAZwN3RMQPyUbQOIbPJttZ3NyBKaV/R8TPgR9GxPtkn0JsD/xPE7sXew/qgJMi4jDgX8D7+QOJ9wNnR8QPgH+QfVpR7EgkklYQFtSSlksppfkRsQdwHFkBfQxZn9dZZLMkfjciViEbeeLBporp3G+Bw8iKqDHNXGt0RCwA/hcYR1a43wuc1VQRvoyuJuu7+3/A58keeOzbyjGnAkFW3JNnOoKsMKyolNInEbEX2XB/o8ju2U3AY2QjdbzXyinOI3tvxwCn5Mf9F0t2AYHi78FPyB4yvBroRvbJwq7ABWT3+wyyn5+HyT65eLm4dyppRRDZQ9WSJFVeRNwDbJFS2qTSWSSpWLZQS5IqIiK+Q9Yy/SLZeOJfIRuJ48RK5pKktrKgliRVysdkXSn6kA1bOA04JqX024qmkqQ2ssuHJEmSVAKHzZMkSZJKYEEtSZIklaDT96Hu2bNn6tu3b6VjSJIkaTn3xBNPzE0p9Wq8vtMX1H379uXxx8s1s68kSZJWVBHxalPr7fIhSZIklcCCWpIkSSqBBbUkSZJUgk7fh7opn376KTNnzuSjjz6qdBSpbLp27cr666/PKqusUukokiStUJbLgnrmzJmsueaa9O3bl4iodBypw6WUePvtt5k5cyYbbbRRpeNIkrRCWS67fHz00Uess846FtNaYUQE66yzjp/KSJJUActlQQ1YTGuF48+8JEmVsdwW1JX2xhtvcPjhh7PJJptQW1vLvvvuywsvvNCh19x1111bHZP78ssvZ8GCBQ3L++67L//+9787NFdbFPMejjnmGOrq6trlen379mXu3Lntcq5C7ZlRkiRVt+WyD3WlpZQ46KCDOOqoo7jlllsAmDp1Km+++SabbrppRbNdfvnlHHnkkay++uoA3HvvvRXNsyyuvvrqSkdo0aJFi6o+oyRJaj+2UHeABx98kFVWWYUTTjihYd3AgQPZaaedeOihh9h///0b1p9yyimMGTMGyFpLf/CDHzB06FAGDRrElClT2Hvvvdlkk00YNWoUQIvHFzrxxBMZNGgQ/fv3Z8SIEQCMHDmS119/nd12243ddtut4Zpz587l7LPP5oorrmg4/rzzzuOyyy4D4JJLLmH77bdnwIABDedqbPz48QwdOpRtt92Wr3zlK8yfP59XX32Vfv36MXfuXBYvXsxOO+3E+PHjmT59OptvvjlHHXUUAwYM4NBDD12i1byl9wBLtmJ369aNH/7wh2y99dYMGTKEN998E4A5c+ZwyCGHsP3227P99tvz6KOPAvD222+z1157sc0223D88ceTUlrquldeeSVnnXVWw/KYMWM49dRTATjwwAPZbrvt6N+/P6NHj27Yp1u3bpx77rnssMMOTJw4cYmMzb2Pvn37MmLECLbddlu22mornn/+eQDmz5/PN7/5TbbaaisGDBjA7bff3uw9liRJlVe2FuqIuAbYH3grpbRlE9sD+AWwL7AAODqlNKXU655/97PUvT6v1NMsoXa97oz4r/7Nbn/mmWfYbrvtluncG2ywARMnTuSMM87g6KOP5tFHH+Wjjz6if//+SxTorbn44otZe+21WbRoEXvssQdPPfUU3/72t/nZz37Ggw8+SM+ePZfY//DDD+f000/npJNOAuDWW2/l/vvvZ/z48bz44ov84x//IKXE8OHDeeSRR9h5550bjp07dy4XXXQREyZMYI011uAnP/kJP/vZzzj33HM5++yzOeGEE9hhhx2ora1lr732Yvr06UybNo3f/va37LjjjnzrW9/iiiuu4Mwzz2z1PQwYMGCJfT744AOGDBnCxRdfzFlnncVVV13FOeecw2mnncYZZ5zBsGHDmDFjBnvvvTfPPfcc559/PsOGDePcc8/lj3/84xJFcb1DDz2UoUOH8tOf/hSAsWPH8sMf/hCAa665hrXXXpsPP/yQ7bffnkMOOYR11lmHDz74gC233JILLrigqO9F/fvo2bMnU6ZM4YorruDSSy/l6quv5sILL2Sttdbi6aefBuDdd99t8R5LkqTKKmeXjzHAr4Drm9n+ZaBf/rUDcGX+7wpl+PDhAGy11VbMnz+fNddckzXXXJOuXbu2qa/zrbfeyujRo1m4cCGzZ8+mrq5uqWK00DbbbMNbb73F66+/zpw5c+jRowd9+vRh5MiRjB8/nm222QbIWk9ffPHFJQrqSZMmUVdXx4477gjAJ598wtChQ4GsL/Hvf/97Ro0axdSpUxuO2WCDDRr2P/LIIxk5cuRSBXUx72HVVVdtaLHfbrvteOCBBwCYMGHCEn2Y582bx/vvv88jjzzCHXfcAcB+++1Hjx49lroXvXr1YuONN2bSpEn069ePadOmNWQdOXIkd955JwCvvfYaL774Iuussw5dunThkEMOafP34uCDD27IXp9rwoQJDV2FAHr06ME999zT7D2WJEmVVbaCOqX0SET0bWGXA4DrU/YZ/KSI+HxE1KSUZpdy3ZZakjtK//79ue2225rctvLKK7N48eKG5cbDnH3uc58DYKWVVmp4Xb+8cOHCVo8HeOWVV7j00kuZPHkyPXr04Oijjy5qOLVDDz2U2267reGBSsj6g3//+9/n+OOPb/a4lBJ77rknN99881LbFixYwMyZMwEa/kCApUekaLxc7HtYZZVVGo7t0qULCxcuBGDx4sVMnDiR1VZbbaljihkN47DDDuPWW29l880356CDDiIieOihh5gwYQITJ05k9dVXZ9ddd23I1LVrV7p06bLUeVp7H/Xf48LsKaWlMrZ0jyVJKpebHpvBuKmz2uVcrX3i35lUUx/q3sBrBcsz83VLiYjjIuLxiHh8zpw5ZQnXFrvvvjsff/wxV111VcO6yZMn8/DDD7PhhhtSV1fHxx9/zHvvvcef//znNp27mOPnzZvHGmuswVprrcWbb77Jfffd17BtzTXX5P3332/y3Icffji33HILt912G4ceeigAe++9N9dcc01Df91Zs2bx1ltvLXHckCFDePTRR3nppZeArIiuH9Hk7LPP5utf/zoXXHABxx57bMMxM2bMYOLEiQDcfPPNDBs2rOj3UIy99tqLX/3qVw3L9a3jO++8MzfeeCMA9913H++++26Txx988MH84Q9/4Oabb+awww4D4L333qNHjx6svvrqPP/880yaNKnVHMvyPhpnf/fdd1u8x5Iklcu4qbOom92+XWmXB9U0ykdTzYZLPzEGpJRGA6MBBg0a1OQ+lRQR3HnnnZx++un8+Mc/pmvXrvTt25fLL7+cDTbYgK9+9asMGDCAfv36NXSlKFYxx2+99dZss8029O/fn4033rihmwDAcccdx5e//GVqamp48MEHlziuf//+vP/++/Tu3ZuamhogK+6ee+65hu4F3bp144YbbmDddddtOK5Xr16MGTOGI444go8//hiAiy66iNmzZzN58mQeffRRunTpwu233861117LbrvtxhZbbMF1113H8ccfT79+/TjxxBOLfg/FGDlyJCeffDIDBgxg4cKF7LzzzowaNYoRI0ZwxBFHsO2227LLLrvQp0+fJo/v0aMHtbW11NXVMXjwYAD22WcfRo0axYABA9hss80YMmRIqzmW5X2cc845nHzyyWy55ZZ06dKFESNGcPDBBzd5jys9aowkqWO1Z4twe6ibPY/amu6MPd5uh4WiqVEOOuxiWZePe5p5KPE3wEMppZvz5WnArq11+Rg0aFBqPG7xc889xxZbbNFuudW+pk+fzv77788zzzxT6SjLHX/2JWn5cthvJjYUsdXigIG9+doOTTdILe8i4omU0qDG66uphfou4JSIuIXsYcT3Su0/LUmS1NnZIlz9yjls3s3ArkDPiJgJjABWAUgpjQLuJRsy7yWyYfO+Wa5sKq++ffvaOi1JkpYb5Rzl44hWtifg5DLFkSRJktpFNY3yIUmSJHU6FtSSJElSCSyoJUmSpBJYUHeQbt26tWn/hx56qGEK7bvuuosf//jHLe5/7rnnMmHChBbPsyz69u3L3Llzl/n41uy66640HuawsWOOOWaJacNL0VHvpz0zSpKkzq2ahs3rOEVMNd0mHTx29/Dhwxk+fHiL+1xwwQUdmqGSrr766kpHaNGiRYuqPqMkSSqfFaOgrqCHHnqI8847j549e/LMM8+w3XbbccMNNxAR3H///Zx++un07NmTbbfdtuGYMWPG8Pjjj3PxxRez9dZb8/LLL7PSSiuxYMECNttsM15++WWOPfZY9t9/fw499NBmz3PeeefRrVs3zjzzTAC23HJL7rnnHvr27cuBBx7Ia6+9xkcffcRpp53Gcccd1+L7GD9+PCNGjODjjz9mk0024dprr+Xtt9/mS1/6EhMnTmTttddml1124Uc/+hGbbrop++yzDzvssANPPvkkm266Kddffz2rr776Euc88cQTmTx5Mh9++CGHHnoo559/PpC1Yl966aUMGjSIbt26cdppp3HPPfew2mqrMW7cOL7whS8wZ84cTjjhBGbMmAHA5Zdfzo477sjbb7/NEUccwZw5cxg8eDBNTVx05ZVX8sorr/DTn/604X4/8cQT/PKXv2z2vnTr1o3vfOc7/OlPf+Kyyy7jnHPOacjY3Pvo27cvRx11FHfffTeffvopv//979l8882ZP38+p556Ko8//jgRwYgRIzjkkEOavMdt/aRDkrTsqm1WQqDqJnVR0+zyUQZPPvkkl19+OXV1dbz88ss8+uijfPTRRxx77LHcfffd/PWvf+WNN95Y6ri11lqLrbfemocffhiAu+++m7333ptVVlmlYZ9iztOUa665hieeeILHH3+ckSNH8vbbbze779y5c7nooouYMGECU6ZMYdCgQfzsZz9jww035Oyzz+aEE07gsssuo7a2lr322guAadOmcdxxx/HUU0/RvXt3rrjiiqXOe/HFF/P444/z1FNP8fDDD/PUU08ttc8HH3zAkCFD+Oc//8nOO+/MVVddBcBpp53GGWecweTJk7n99ts55phjADj//PMZNmwYTz75JMOHD28ouAsdeuih3HHHHQ3LY8eO5bDDDmvxvnzwwQdsueWWPPbYYwwbNqzo99GzZ0+mTJnCiSeeyKWXXgrAhRdeyFprrcXTTz/NU089xe67797sPZYklc+4qbOomz2v0jGWUFvTnQMG9q50DLXCFuoyGDx4MOuvvz4AAwcOZPr06XTr1o2NNtqIfv36AXDkkUcyevTopY497LDDGDt2LLvtthu33HILJ5100hLbn3/++aLO09jIkSO58847AXjttdd48cUXWWeddZrcd9KkSdTV1bHjjjsC8MknnzB0aDZj0zHHHMPvf/97Ro0axdSpUxuO2WCDDRr2P/LIIxk5cmRDS3m9W2+9ldGjR7Nw4UJmz55NXV0dAwYMWGKfVVddtaFP+HbbbccDDzwAwIQJE5bowzxv3jzef/99HnnkkYZieb/99qNHjx5LvZ9evXqx8cYbM2nSJPr168e0adMasjZ3X7p06cIhhxzS5P1p6X0cfPDBDdnrc02YMIFbbrml4fgePXpwzz33NHuPJUnl46yEWhYW1GXwuc99ruF1ly5dWLhwIQBRRN/u4cOH8/3vf5933nmHJ554gt13332pfZo7z8orr8zixYsblj/66CMg64YyYcIEJk6cyOqrr86uu+7asK0pKSX23HNPbr755qW2LViwgJkzZwIwf/581lxzzSYzNV5+5ZVXuPTSS5k8eTI9evTg6KOPbjLDKqus0nBs4b1bvHgxEydOZLXVVlvqmGLu62GHHcatt97K5ptvzkEHHUREtHhfunbtSpcuXZY6T2vvo/57X5g9pbRUxpbusSRJqm52+aiQzTffnFdeeYV//etfAM0WUt26dWPw4MGcdtpp7L///ksVdS2dp2/fvkyZMgWAKVOm8MorrwDw3nvv0aNHD1ZffXWef/55Jk2a1GLWIUOG8Oijj/LSSy8BWRH9wgsvAHD22Wfz9a9/nQsuuIBjjz224ZgZM2YwceLEhkyNu0nMmzePNdZYg7XWWos333yT++67r8UMje2111786le/aliubx3feeedufHGGwG47777ePfdd5s8/uCDD+YPf/gDN998c0N3j7bel2V9H42zv/vuuy3eY0mSVN0sqCuka9eujB49mv32249hw4ax4YYbNrvvYYcdxg033NBQ+BV7nkMOOYR33nmHgQMHcuWVV7LpppsCsM8++7Bw4UIGDBjAj370I4YMGdJi1l69ejFmzBiOOOIIBgwYwJAhQ3j++ed5+OGHmTx5ckNRveqqq3LttdcCsMUWW3DdddcxYMAA3nnnHU488cQlzrn11luzzTbb0L9/f771rW81dHUo1siRI3n88ccZMGAAtbW1jBo1CoARI0bwyCOPsO222zJ+/Hj69OnT5PE9evSgtraWV199lcGDBy/TfVnW93HOOefw7rvvsuWWW7L11lvz4IMPNnuPJUlS9YumRkHoTAYNGpQaj2v83HPPscUWW1QokaZPn87+++/PM888U+koKxx/9iVp2R32m+yTVftQqzkR8URKaVDj9bZQS5IkSSWwoFa769u3r63TkiRphWFBLUmSJJVguS2oO3vfcKmt/JmXJKkylstxqLt27crbb7/NOuusU9SYxFJnl1Li7bffpmvXrpWOIklFq7apvp3mW8tquSyo119/fWbOnMmcOXMqHUUqm65duzbMyClJnUH9VN/VUsQ6zbeW1XJZUK+yyipstNFGlY4hSZJa4VTfWh4st32oJUmSpHKwoJYkSZJKYEEtSZIklcCCWpIkSSqBBbUkSZJUAgtqSZIkqQQW1JIkSVIJLKglSZKkElhQS5IkSSWwoJYkSZJKYEEtSZIklcCCWpIkSSrBypUOIEmSyuOmx2YwbuqsSsdoUDd7HrU13SsdQyqZLdSSJK0gxk2dRd3seZWO0aC2pjsHDOxd6RhSyWyhliRpBVJb052xxw+tdAxpuWILtSRJklQCC2pJkiSpBBbUkiRJUgksqCVJkqQSWFBLkiRJJbCgliRJkkpgQS1JkiSVwIJakiRJKoEFtSRJklQCC2pJkiSpBBbUkiRJUgksqCVJkqQSWFBLkiRJJVi50gEkSVpe3fTYDMZNnVXpGA3qZs+jtqZ7pWNIyx1bqCVJ6iDjps6ibva8SsdoUFvTnQMG9q50DGm5Ywu1JEkdqLamO2OPH1rpGJI6kC3UkiRJUgksqCVJkqQSWFBLkiRJJShrQR0R+0TEtIh4KSK+18T2tSLi7oj4Z0Q8GxHfLGc+SZIkqa3KVlBHRBfg18CXgVrgiIiobbTbyUBdSmlrYFfgsohYtVwZJUmSpLYqZwv1YOCllNLLKaVPgFuAAxrtk4A1IyKAbsA7wMIyZpQkSZLapJwFdW/gtYLlmfm6Qr8CtgBeB54GTkspLW58oog4LiIej4jH58yZ01F5JUmSpFaVs6COJtalRst7A1OB9YCBwK8iYqkpnVJKo1NKg1JKg3r16tXeOSVJkqSilbOgnglsULC8PllLdKFvAnekzEvAK8DmZconSZIktVnRBXVEDIiI6/OuFpMj4rqI2KoN15oM9IuIjfIHDQ8H7mq0zwxgj/x6XwA2A15uwzUkSZKksiqqoI6I4cAUshbm+4D7gT7AlIj4r2LOkVJaCJwC/Al4Drg1pfRsRJwQESfku10I/GdEPA38GTg7pTS3LW9IkiRJKqeVi9zvIuDilNKIwpURcUG+7e5iTpJSuhe4t9G6UQWvXwf2KjKTJEmSVHHFdvnYFPhdE+t/R9YtQ5IkSVohFdtC/RawHfBSo/XbAW+2ayJJkpbBTY/NYNzUWZWOsYS62fOorVlqsCpJy5liC+qrgN9ExBeBv5MNdzcMOBO4pIOySZJUtHFTZ1VdAVtb050DBjaeckHS8qYtfajnA98le3AQsiHvRgAjOyCXJEltVlvTnbHHD610DEkrmKIK6pRSAn4O/Dwi1szXvd+RwSRJkqTOoNgW6gYW0pIkSdJnmi2oI+IpYJeU0rv5uNCNpwlvkFIa0BHhJEmSpGrXUgv17cDHBa+bLaglSZKkFVWzBXVK6fyC1+eVJY0kSZLUyRQ79fhfIuLzTazvHhF/afdUkiRJUidR7EyJuwKrNrG+K7BTu6WRJEmSOpkWR/mIiG0LFgdExDsFy12AvYHqmpZKkiRJKqPWhs17nOxhxASMb2L7h8Cp7R1KkiRJ6ixaK6g3AgJ4GRgMzCnY9gnwVkppUQdlkyRJkqpeiwV1SunV/GWxfa0lSZKkFUrRMyVGxMpkrdR9aPSAYkrp+nbOJUmSJHUKRRXUEbE5cDefdQFZlB/7KdnkLxbUkiRJWiEV25XjcuAJYC1gAbAFMAiYChzSEcEkSZKkzqDYLh/bA7uklD6IiMXAyimlKRFxFvBLYECHJZQkSZKqWLEFdZC1TEM20kdvYBowE/hiB+SSJFW5mx6bwbip1TMVQd3sedTWdK90DEkroGK7fDwDbJ2//gdwdkTsApwPvNQRwSRJ1W3c1FnUzZ5X6RgNamu6c8DA3pWOIWkFVGwL9cXAGvnrc4B7gAeBucBXOyCXJKkTqK3pztjjh1Y6hiRVVLEF9V/JRvMgpfQyUBsRawPvppRSR4WTJEmSql2rXT4iogvwHrBZ4fqU0jsW05IkSVrRtVpQ51OLv0qjyVwkSZIkFf9Q4oXAjyOiZ0eGkSRJkjqbYvtQn0k2S+KsiJgJfFC4MaXkONSSJElaIRVbUN/WoSkkSZKkTqqogjqldH5HB5EkSZI6o2L7UEuSJElqggW1JEmSVAILakmSJKkEFtSSJElSCSyoJUmSpBIUXVBHxEkR8WxELIiIjfN134uIr3ZcPEmSJKm6FVVQR8TpwDnAaCAKNs0CTmn/WJIkSVLnUGwL9QnAsSmlXwALC9ZPAfq3eypJkiSpkyi2oN4QeKaJ9Z8Cq7VfHEmSJKlzKXbq8ZeBbYFXG63fF6hr10SSpCbd9NgMxk2dVekYDepmz6O2pnulY0hSxRVbUF8K/CoiVifrQz00Ir4BnAV8q6PCSZI+M27qrKoqYmtrunPAwN6VjiFJFVdUQZ1SujYiVgb+D1gd+B3ZA4nfTimN7cB8kqQCtTXdGXv80ErHkCQVKLaFmpTSVcBVEdETWCml9FbHxZIkSZI6h6IK6ojoD3RJKT2VUppbsH4AsDClZD9qSZIkrZCKHeVjNLBlE+tr822SJEnSCqnYgnoA8I8m1k8Gtmq/OJIkSVLnUmxBvQhYq4n1PVhy5kRJkiRphVJsQf0w8MOI6FK/Ih/144fAIx0RTJIkSeoMih3l4yzgb8BLEfG3fN0woBuwc0cEkyRJkjqDolqoU0rTyPpR3wSsDawD3AhsnVJ6ruPiSZIkSdWtLeNQzybr4iFJkiQpV3RBnU87PhBYl0Yt2ymlO9o3liRJktQ5FDuxy5eAm8m6ejSWgC5NrJckSZKWe8WO8vEL4I/A+imllRp9WUxLkiRphVVsQd0XuDCl9HopF4uIfSJiWkS8FBHfa2afXSNiakQ8GxEPl3I9SZIkqaMV24f6UWAz4F/LeqF8DOtfA3sCM4HJEXFXSqmuYJ/PA1cA+6SUZkTEust6PUmSJKkcii2oRwGXRsR6wNPAp4UbU0pTijjHYOCllNLLABFxC3AAUFewz9eAO1JKM/LzvlVkPkmSJKkiii2ob8v/Hd3EtmIfSuwNvFawPBPYodE+mwKrRMRDwJrAL1JK1zc+UUQcBxwH0KdPnyIuLUmSJHWMYgvqjdrhWtHEutRoeWVgO2APYDVgYkRMSim9sMRBKY0mL+4HDRrU+ByS1C5uemwG46bOqnSMBnWz51Fb073SMSRJjRRVUKeUXm2Ha80ENihYXh9o/JDjTGBuSukD4IOIeATYGngBSSqzcVNnVVURW1vTnQMG9q50DElSI22Z2GVlsn7QfYBVC7c11S2jCZOBfhGxETALOJysz3ShccCv8mutStYl5OfFZpSk9lZb052xxw+tdAxJUhUrdmKXzYG7ybp+BLAoP/ZT4GOg1YI6pbQwIk4B/kTW5/qalNKzEXFCvn1USum5iLgfeApYDFydUnqm7W9LkiRJKo9iW6gvB54gm3r8jfzftYArgXOKvVhK6V7g3kbrRjVavgS4pNhzSpIkSZVUbEG9PbBLSumDiFgMrJxSmhIRZwG/BAZ0WEJJkiSpihU7U2IAC/LXc8iGwIPsIcIvtncoSZIkqbMotoX6GbLRNl4G/gGcHRGLgGOBlzoomyRJklT1ii2oLwbWyF+fA9wDPAjMBb7aAbkkSZKkTqHYcaj/VPD6ZaA2ItYG3k0pObGKJEmSVlhFj0PdWErpnfYMIkmSJHVGzRbUEXEXcGRKaV7+ulkppeHtnkySJEnqBFpqoX4bqO/O8U7Ba0mSJEm5ZgvqlNI3CxZPAj5OKS3q+EiSJElS59HqONQR0QV4D9is4+NIkiRJnUurBXXeKv0qsGrHx5EkSZI6l2JnSrwQ+HFE9OzIMJIkSVJnU+yweWcCGwGzImIm8EHhxpTSgPYOJkmSJHUGxRbUt3VoCkmSJKmTKnamxPM7OogkSZLUGS3zTImS1J5uemwG46bOqnSMJdTNnkdtTfdKx5AkVbmiHkqMiFUj4vyIeCEiPoqIRYVfHR1S0vJv3NRZ1M2eV+kYS6it6c4BA3tXOoYkqcoV20J9IXAY8P+AnwP/C/QFDgd+1CHJJK1wamu6M/b4oZWOIUlSmxQ7bN5XgRNSSr8BFgHjUkrfBkYAe3ZUOEmSJKnaFVtQfwGoy1/PBz6fv74f2KudM0mSJEmdRrEF9Qxgvfz1S8De+euhwIftHUqSJEnqLIotqO8E9shf/wI4PyJeAcYAV3dALkmSJKlTKHYc6u8XvL4tIl4DdgReSCnd01HhJEmSpGpXVEEdET1TSnPrl1NKjwGPdVgqSZIkqZMotsvH6xFxd0R8NSK6dmgiSZIkqRMptqDeH3gbuAp4MyKujYjdIyI6LpokSZJU/YoqqFNK41NKR5MNn3ccsDZwH/BaRPy04+JJkiRJ1a3YFmoAUkofpZTGppQOAAYCc4DvdkQwSZIkqTNoU0EdEWtExJERcR/wT2BN4KIOSSZJkiR1AsWO8rEf8HVgONlELr8Hdk0p/b0Ds0mSJElVr6iCmqyAvgf4GnBvSmlhx0WSJEmSOo9iC+r/SCnN69AkkiRJUidU7CgfFtOSJElSE9r0UKIkSZKkJRXb5UPScuamx2YwbuqsSsdoUDd7HrU13SsdQ5KkNrOFWlpBjZs6i7rZ1dObq7amOwcM7F3pGJIktZkt1NIKrLamO2OPH1rpGJIkdWrNFtQRcU2xJ0kpfat94kiSJEmdS0st1L0aLe8MLAaezpe3JOsy8kgH5JIkSZI6hWYL6pTSf9W/jojvk82Q+M2U0gf5ujWA3/JZgS1JkiStcIp9KPHbwHn1xTRA/vpC4NSOCCZJkiR1BsUW1N2A9ZpYXwOs3n5xJEmSpM6l2IL6duDaiDg8IvrmX4eTdfm4o+PiSZIkSdWt2GHzTgQuA8YAq+TrFpIV1Ge2fyxJkiSpcyiqoE4pfQicFBH/C2wCBPBSYZ9qSZIkaUXU1pkSV8u/nreYliRJkoosqCNizYj4PfAW8Hegd75+VESc13HxJEmSpOpWbAv1T8hG+diWbDzqevcAB7V3KEmSJKmzKPahxOHAQSmlqRGRCtY/B2zc/rEkSZKkzqHYFuoewNtNrF8TWNR+cSRJkqTOpdiCejJZK3W9+lbq48n6VEuSJEkrpGK7fPwA+FNE9M+P+U7+ejCwc0eFkyRJkqpdUS3UKaW/A/8JrAr8C9gDeB0YmlKa0nHxJEmSpOpW9DjUKaWnU0pHpZS2TCnVppSOTCk93ZaLRcQ+ETEtIl6KiO+1sN/2EbEoIg5ty/klSZKkciu2ywcAEbEesC6NCvFiWqkjogvwa2BPYCYwOSLuSinVNbHfT4A/tSWbVO1uemwG46bOqnSMBnWz51Fb073SMSRJ6vSKKqgjYhvgBmBzsmnHCyWgSxGnGUw2XfnL+TlvAQ4A6hrtdypwO7B9MdmkzmLc1FlVVcTW1nTngIG9Kx1DkqROr9gW6tHAa8CxZH2nU8u7N6l3fo56M4EdCneIiN5kE8XsjgW1lkO1Nd0Ze/zQSseQJEntqNiCuhbYJqX0QgnXatyyDUsX5pcDZ6eUFkU0tXt+oojjgOMA+vTpU0IkSZIkqTTFPpT4NPAfJV5rJrBBwfL6ZK3dhQYBt0TEdOBQ4IqIOLDxiVJKo1NKg1JKg3r16lViLEmSJGnZtWUc6p9GxDlkxfWnhRtTSu8UcY7JQL+I2AiYBRwOfK3ReTaqfx0RY4B7Ukp/KDKjJEmSVHbFFtQT8n/Hs2Q3jaDIhxJTSgsj4hSy0Tu6ANeklJ6NiBPy7aOKTi1JkiRViWIL6t3a42IppXuBexuta7KQTikd3R7XlCRJkjpSUQV1Sunhjg4iSZIkdUbNFtQRsS0wNaW0OH/dLKcflyRJ0oqqpRbqx8lG9ngrf51ofui7YiZ2kSRJkpY7LRXUGwFzCl5LkiRJaqTZgjql9GpTryVJkiR9pthRPgCIiPWAPsCqhetTSo+0ZyhJkiSpsyiqoM4L6ZuAnfmsL3XheNT2oZYkSdIKqdipxy8HFgG1wAJgJ+ArwHPAPh2STJIkSeoEiu3ysQuwX0rp+YhIwJyU0qMR8TFwIfBAhyWUJEmSqlixLdSrAXPz1+8A6+av64AB7R1KkiRJ6iyKLaifBzbPX08FToiIDYGTgVkdkEuSJEnqFIrt8vELskleAC4A7geOAD4GjuqAXJIkSVKnUFRBnVK6seD1lIjoS9ZiPSOlNLfZAyVJkqTlXJvGoa6XUloATGnnLFK7uumxGYybWj09kupmz6O2pnulY0iSpHbWbEEdESOLPUlK6dvtE0dqP+OmzqqqIra2pjsHDOxd6RiSJKmdtdRCvVWR50it7yJVRm1Nd8YeP7TSMSRJ0nKs2YI6pbRbOYNIkiRJnVGxw+Y1iIhuEdGtI8JIkiRJnU3RBXVEnB4RM4D3gPci4rWIOCMiouPiSZIkSdWtqFE+IuKnwHHAJcDEfPVQ4FygBjirQ9JJkiRJVa7YYfOOAY5JKd1WsO4vETEN+A0W1JIkSVpBtaUP9VPNrGtzP2xJkiRpeVFsMXw9cHIT608Eftd+cSRJkqTOpdguH58DvhYRewOT8nU7AOsBNxZOAuMkL5IkSVqRFFtQb85nU41vmP/7Rv61RcF+TvIiSZKkFUpRBbWTvEiSJElNK6oPdUT0bWHbf7ZbGkmSJKmTKfahxH9GxJGFKyJipYi4AHiw/WNJkiRJnUOxBfXZwKiIuCkiukfEJmQTvHwL2K/D0kmSJElVrqiCOqU0ChgEbAY8AzwJzAQGpJQmdFw8SZIkqbq1ZVKWN4DpwBeA1YD7U0rvdEQoSZIkqbMo9qHEnclmRVwf6E/W1eOSiLgzItbpwHySJElSVSu2hXoC2WyJO6aUXkop/Q4YCPQCnu6gbJIkSVLVK3Zily+llB4pXJFSmp63XP+g/WNJkiRJnUOxE7s80sz6xcBF7ZpIkiRJ6kRaLKgj4u/Avimlf+fL/w+4pP5hxIjoCUxJKfXp6KCqbjc9NoNxU2dVOsYS6mbPo7ame6VjSJKk5VxrfaiHAKsWLJ8MfL5guQvZg4pawY2bOou62fMqHWMJtTXdOWBg70rHkCRJy7li+1DXiybWpfYIos6vtqY7Y48fWukYkiRJZdWWcaglSZIkNdJaQZ1YugXaFmlJkiQp11qXjwBuiIiP8+WuwFURsSBf/lyHJZMkSZI6gdYK6usaLd/QxD7Xt1MWSZIkqdNpsaBOKX2zXEEkSZKkzsiHEiVJkqQSWFBLkiRJJbCgliRJkkpgQS1JkiSVwIJakiRJKoEFtSRJklQCC2pJkiSpBBbUkiRJUgksqCVJkqQSWFBLkiRJJShrQR0R+0TEtIh4KSK+18T2r0fEU/nX3yNi63LmkyRJktqqbAV1RHQBfg18GagFjoiI2ka7vQLsklIaAFwIjC5XPkmSJGlZlLOFejDwUkrp5ZTSJ8AtwAGFO6SU/p5SejdfnASsX8Z8kiRJUpuVs6DuDbxWsDwzX9ec/wHu69BEkiRJUolWLuO1ool1qckdI3YjK6iHNbP9OOA4gD59+rRXPkmSJKnNytlCPRPYoGB5feD1xjtFxADgauCAlNLbTZ0opTQ6pTQopTSoV69eHRJWkiRJKkY5C+rJQL+I2CgiVgUOB+4q3CEi+gB3AN9IKb1QxmySJEnSMilbl4+U0sKIOAX4E9AFuCal9GxEnJBvHwWcC6wDXBERAAtTSoPKlbEzuemxGYybOqvSMRrUzZ5HbU33SseQJEkqu3L2oSaldC9wb6N1owpeHwMcU85MndW4qbOqqoitrenOAQNbesZUkiRp+VTWglrtq7amO2OPH1rpGJIkSSs0px6XJEmSSmBBLUmSJJXAglqSJEkqgQW1JEmSVAILakmSJKkEFtSSJElSCSyoJUmSpBJYUEuSJEklsKCWJEmSSmBBLUmSJJXAglqSJEkqgQW1JEmSVAILakmSJKkEFtSSJElSCSyoJUmSpBJYUEuSJEklsKCWJEmSSmBBLUmSJJXAglqSJEkqgQW1JEmSVAILakmSJKkEK1c6wIrm/Lufpe71eSWfp272PGprurdDIkmSJJXCgroUEW0/Zo9jYd2Nl/2au+wCQG1Ndw4Y2HvZzyNJkqR2YUFdZiP+fFVpJ7gptU8QSZIktQv7UEuSJEklsKCWJEmSSmBBLUmSJJXAglqSJEkqgQW1JEmSVAILakmSJKkEFtSSJElSCRyHenmyLBPNlCo5LrYkSVqxWVCr41jgS5KkFYBdPiRJkqQS2EKtFYct5pIkqQNYUEuVZJEvSVKnZ0Et6TPVVuCbxz+AJKkTsKCWpM6s3EV+awW+f3RIWgFZUEuSll8W+JLKwFE+JEmSpBLYQi1JUrlUW4t5teWROikLakmSVD0s8tUJ2eVDkiRJKoEFtSRJklQCu3xIkiQ1xy4oKoIt1JIkSVIJbKGWJEnqLGwxr0q2UEuSJEklsKCWJEmSSmBBLUmSJJXAglqSJEkqgQW1JEmSVIKyFtQRsU9ETIuIlyLie01sj4gYmW9/KiK2LWc+SZIkqa3KVlBHRBfg18CXgVrgiIiobbTbl4F++ddxwJXlyidJkiQti3K2UA8GXkopvZxS+gS4BTig0T4HANenzCTg8xFRU8aMkiRJKlZE+b+qUDkL6t7AawXLM/N1bd1HkiRJqhrlnCmxqT8pGk+9U8w+RMRxZF1CAOZHxLQSs5VbT2DuMh3ZMX+ZVVseWNZM5mmdP0MtM0/Lqi0P+DPdGvO0zp+hlq04eYqxYVMry1lQzwQ2KFheH3h9GfYhpTQaGN3eAcslIh5PKQ2qdI561ZYHqi+TeVpWbXmg+jKZp2XmaV21ZTJPy6otD1RfpmrLU4pydvmYDPSLiI0iYlXgcOCuRvvcBfx3PtrHEOC9lNLsMmaUJEmS2qRsLdQppYURcQrwJ6ALcE1K6dmIOCHfPgq4F9gXeAlYAHyzXPkkSZKkZVHOLh+klO4lK5oL140qeJ2Ak8uZqUKqrbtKteWB6stknpZVWx6ovkzmaZl5WldtmczTsmrLA9WXqdryLLPIalhJkiRJy8KpxyVJkqQSWFCXUURcExFvRcQzlc4CEBEbRMSDEfFcRDwbEadVOE/XiPhHRPwzz3N+JfPUi4guEfFkRNxT6SwAETE9Ip6OiKkR8XgV5Pl8RNwWEc/nP0tDK5hls/y+1H/Ni4jTK5Unz3RG/vP8TETcHBFdK5zntDzLs5W6N039LoyItSPigYh4Mf+3R4XzfCW/R4sjoqyjEDST55L8v7GnIuLOiPh8FWS6MM8zNSLGR8R6lcxTsO3MiEgR0bOSeSLivIiYVfD7aN9K5snXnxoR0/Kf7Z+WK09zmSJibMH9mR4RU8uZqT1ZUJfXGGCfSocosBD4bkppC2AIcHIsPR18OX0M7J5S2hoYCOyTj/ZSaacBz1U6RCO7pZQGVslwQ78A7k8pbQ5sTQXvVUppWn5fBgLbkT3cfGel8kREb+DbwKCU0pZkD2QfXsE8WwLHks1cuzWwf0T0q0CUMSz9u/B7wJ9TSv2AP+fLlczzDHAw8EgZc9Qbw9J5HgC2TCkNAF4Avl8FmS5JKQ3I/3u7Bzi3wnmIiA2APYEZZczSbB7g5/W/k/LnyCqWJyJ2I5uRekBKqT9waRnzNJkppXRYwe/s24E7ypyp3VhQl1FK6RHgnUrnqJdSmp1SmpK/fp+sEKrYzJT5lPPz88VV8q+KdvKPiPWB/YCrK5mjWkVEd2Bn4LcAKaVPUkr/rmioz+wB/Cul9GqFc6wMrBYRKwOr08TY+mW0BTAppbQgpbQQeBg4qNwhmvldeABwXf76OuDASuZJKT2XUqrIpGHN5Bmff88AJpHN01DpTPMKFtegjL+vW/j/6c+Bs8qZpZU8FdFMnhOBH6eUPs73easKMgEQEQF8Fbi5nJnakwW1AIiIvsA2wGMVztEl/8jnLeCBlFJF8wCXk/1yXlzhHIUSMD4inohs1tBK2hiYA1ybd4u5OiLWqHCmeodT4V/OKaVZZK1AM4DZZGPrj69gpGeAnSNinYhYnWyY0g1aOaZcvlA/70D+77oVzlPNvgXcV+kQABFxcUS8Bnyd8rZQN5VlODArpfTPSuZo5JS8W8w15ezG1IxNgZ0i4rGIeDgitq9wnkI7AW+mlF6sdJBlZUEtIqIb2UctpzdqcSi7lNKi/KOf9YHB+UfUFRER+wNvpZSeqFSGZuyYUtoW+DJZN52dK5hlZWBb4MqU0jbAB5T3o/omRTZ51HDg9xXO0YOs5XUjYD1gjYg4slJ5UkrPAT8h6z5wP/BPsq5f6iQi4odk37MbK50FIKX0w5TSBmR5TqlUjvwPxB9S4aK+kSuBTci6MM4GLqtomuz3dQ+yLp7/C9yatwxXgyPoxK3TYEG9wouIVciK6RtTSlXTdynvNvAQle1zviMwPCKmA7cAu0fEDRXMA0BK6fX837fI+gcPrmCcmcDMgk8SbiMrsCvty8CUlNKbFc7xJeCVlNKclNKnZP0D/7OSgVJKv00pbZtS2pns49dqaRF6MyJqAPJ/y/pxdGcQEUcB+wNfT9U35u1NwCEVvP4mZH+4/jP/nb0+MCUi/qNSgVJKb+aNRIuBq6js72rIfl/fkXev/AfZJ69le3CzOXl3uIOBsZXOUgoL6hVY/pfpb4HnUko/q4I8veqfXI+I1ciKkecrlSel9P2U0voppb5k3Qf+klKqWOsiQESsERFr1r8G9iL7GL8iUkpvAK9FxGb5qj2AukrlKVAtrR0zgCERsXr+39seVPgB14hYN/+3D9n/xKrhPgHcBRyVvz4KGFfBLFUnIvYBzgaGp5QWVDoPQKMHWodT2d/XT6eU1k0p9c1/Z88Ets1/R1VE/R+IuYOo4O/q3B+A3QEiYlNgVWBuJQPlvgQ8n1KaWekgpSjrTIkruoi4GdgV6BkRM4ERKaXfVjDSjsA3gKcLhqr5QZmfRC5UA1wXEV3I/ti7NaVUFUPVVZEvAHfmn9KtDNyUUrq/spE4Fbgx72bxMvDNSobJP/rdEzi+kjkAUkqPRcRtwBSyj+mfpPIzg90eEesAnwInp5TeLXeApn4XAj8m+wj6f8j+EPlKhfO8A/wS6AX8MSKmppT2rmCe7wOfAx7I//uflFI6oRx5Wsi0b/7H9GLgVaCieSr5/9Nm7s+uETGQ7LmX6ZTxd1Izea4BrsmHrfsEOKqcn3S08D2r+PMu7cGZEiVJkqQS2OVDkiRJKoEFtSRJklQCC2pJkiSpBBbUkiRJUgksqCVJkqQSWFBL6tQiYkxEtNvwihHRNyJSRAxqr3Pm523XnGpZe93viHgmIs5rh0iSlmOOQy2pKkTEGD6b2GMh8BrZzIIjUkoftHDoaUB7Tp/7GtmY6NUw4YEaiYiHgGdSSq1Nc93ePxeS1CwLaknVZALZZEOrADsBVwNrACc23jGfrnZRSum99gyQUloEVGx2NbWP9v65kKSW2OVDUjX5OKX0RkrptZTSTcCNwIEAEXFe/vH70RHxL+BjYI3GH+1HxEMRcUVE/F9EzI2ItyLi0ohYqWCfVfPtr0bExxHxckR8O9+2RJePiNg1X94/IqZGxEcR8UREbFdwvnUi4uaImBkRH0bEsxHR5hkjI2LziLgrIt6LiPkRMTEitsq3rRQRP4qI1/LMT0fEAQXH1uc+PCIeznM8GREDImLLiPh7RHwQEX+LiI0Kjqu/r8dExIz8uD9ERM+CfYq99iER8UBELIiIuojYs9H7q42IP0bE+/n35eaI+I+C7WMi4p6IOC0iZkXEuxFxbWSzX9Z/irELcHJ+vRQRfZu5l8vyc7FuRIzL78GrEfGtJs67VkSMzo9/P7/X9T8rXfN7eU3B/uvl1zuz5e++pM7MglpSNfuQrLW63kbA18impd4a+KiZ475O1m3kP4FTgNOBwwq2Xwf8N/AdYAvgf4B/t5LlUuBsYBDZFOt/rC/0gK5k04vvD/QHfgH8JiL2aOWcDSJiPeBvZNMU7wlsC/wa6JLvchrwv3mGrYA7gTsim9q40PnAT4Bt8vd0E9kU2j8EBudZRzY6pi9wJHAA8CWgH9k0xfWKvfbF+bm3BiYDt0REt/z91QCPAM/kOb4EdAPuKixqyT6Z2DLffhhwUH79+hwTgWvJuuXUkHXRKVZrPxdjgC/m1z6Q7Gekb/3GiAjgj0Bvsu/1Nvl7+ktE1KSUPiL7+fxaRHwl3/964J/AZW3IKamzSSn55ZdfflX8i6yYuadgeTBZP+ax+fJ5wKfAF1o57iFgYqN9HgCuzl/3Iyta92kmR998+6B8edd8+esF+3QjK1aPaeH93FJ/zaZyNrH/xcCrwKrNbJ8FnNto3UPADY1yH1+wff983cEF644G5hcsnwcsAvoUrBuWH9evhGv3ztcNy5cvAP7c6Bw98n0GF9yj14CVC/a5CpjQ6Lq/Woafp9Z+LjbNs+xYsH3D/N6cly/vDswHVmt0nqnAWQXLpwPvAj8D3gZ6V/q/L7/88qtjv+xDLama7BMR88me71gFGAecWrB9ZkrpzSLO81Sj5deBdfPX2wCLgQfbmG1i/YuU0vyIeBqoBYiILsD3yFo7ewOfA1YlK+KKtQ3wt5TSJ403RER3YD3g0Uab/gbs22hd4Xuvv1dPN1q3RkSsnlJakK+blVKaUbDPY2T3aIuIeHMZr/16/m/9fd8O2Dn//ja2CfCP/HVdSmlho/Ps0MQxy6Kln4styN5zfQ5SSq9GxOsF+28HrA7MyRqfG3Qlew/1fgEMB84AvppSmtUu6SVVLQtqSdXkEeA4spbo11NKnzba3tJoH4UaH5f4rItbR4z8cCbwXbIuCU+TtWL+H58Va8UoJlcqYt2nTWxral1bu/y16doppZQXnfXXWYmsu0RTfYkL/0hq6XtXqlJ/LlYiy7pTE9vmFbzuSVagLyLrQiJpOWcfaknVZEFK6aWU0qtNFNPtZQrZ777d2njckPoXEbEGWT/f5/JVw4C7U0q/SylNBf5F1oWgrbmGRcSqjTeklOaRtaYOa7RpGFDXxus0pXdEbFCwPJjsHj3XjteeQta//NX8e1z49X4bzvMJn/Urb0/Pkb3n7etXREQfstb5elOALwCLm3gPbxXsdzXZz8BhwPlR8ACrpOWTBbWkFUpK6UXgVuDqfFSKjSJip4j4RiuHnhMRe0ZEf7IH9j4he+AP4AVgj4gYFhGbA78ie4CyLa4g65t9a0RsHxFfjIgjCh78uwQ4M1+3aURcQNZS2h4Pu30IXBcRAyNiKDAK+GN+r9rr2r8G1gLGRsQOEbFxRHwpHzFjzTacZzowOB9ZpGejBxqXWUppGnA/2cOkQ/P7Pobs3tSbQNb1ZVxEfDn/2RkaEedHxE4AEXECWb/7b6SUbs/PcVPBA6ySlkMW1JJWRP9NVgyPBJ4nK3rWauWY75EVkFPIHmzcP3024cxFZH1v7yPrtvIB2ZB/Rcv72e5M1vf6QeBJsv7j9f2JR5IVtj8lGynjIOCQvEW8VNPJHqK8G/gL2SgmhcP+lXztlNLrwI5k/ZTvB54lK7I/zr+KdSnZHzN1wBygTxuObc3RwCtk9+Busp+R6fUbU0qJrN/4X8gelpxG9sfZZsDrEbEZ2c/IqSmlV/LDTifrWvLzdswpqcpE9vtBktSUiNiVrMDtlVJa7mZPjGxa7UNTSltWOoskdVa2UEuSJEklsKCWJEmSSmCXD0mSJKkEtlBLkiRJJbCgliRJkkpgQS1JkiSVwIJakiRJKoEFtSRJklQCC2pJkiSpBP8fgOwYRKqnzekAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# Build PCA using standarized trained data\n",
    "pca = PCA(n_components=None, svd_solver=\"full\")\n",
    "pca.fit(StandardScaler().fit_transform(X_train))\n",
    "cum_var_exp = np.cumsum(pca.explained_variance_ratio_)\n",
    "plt.figure(figsize=(12, 6))\n",
    "plt.bar(range(1, 18), pca.explained_variance_ratio_, align=\"center\",\n",
    "        color='red', label=\"Individual explained variance\")\n",
    "plt.step(range(1, 18), cum_var_exp, where=\"mid\", label=\"Cumulative explained variance\")\n",
    "plt.xticks(range(1, 18))\n",
    "plt.legend(loc=\"best\")\n",
    "plt.xlabel(\"Principal component index\", {\"fontsize\": 14})\n",
    "plt.ylabel(\"Explained variance ratio\", {\"fontsize\": 14})\n",
    "plt.title(\"PCA on training data\", {\"fontsize\": 16});"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.preprocessing import Binarizer\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[0mThe best hyperparameters for Original data:\n",
      "criterion :  gini\n",
      "max_features :  0.5\n",
      "min_samples_leaf :  1\n",
      "n_estimators :  500\n",
      "\u001b[1m\u001b[94mBest 10-folds CV f1-score: 98.23%.\n",
      "\u001b[1m\u001b[0mThe best hyperparameters for Upsampled data:\n",
      "criterion :  entropy\n",
      "max_features :  0.4\n",
      "min_samples_leaf :  1\n",
      "n_estimators :  500\n",
      "\u001b[1m\u001b[94mBest 10-folds CV f1-score: 99.80%.\n",
      "\u001b[1m\u001b[0mThe best hyperparameters for Downsampled data:\n",
      "criterion :  gini\n",
      "max_features :  0.5\n",
      "min_samples_leaf :  1\n",
      "n_estimators :  50\n",
      "\u001b[1m\u001b[94mBest 10-folds CV f1-score: 98.46%.\n"
     ]
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "# Build random forest classifier\n",
    "methods_data = {\"Original\": (X_train, y_train),\n",
    "                \"Upsampled\": (X_train_u, y_train_u),\n",
    "                \"Downsampled\": (X_train_d, y_train_d)}\n",
    "for method in methods_data.keys():\n",
    "    pip_rf = make_pipeline(StandardScaler(),\n",
    "                           RandomForestClassifier(n_estimators=500,\n",
    "                                                  class_weight=\"balanced\",\n",
    "                                                  random_state=123))\n",
    "    \n",
    "    hyperparam_grid = {\n",
    "        \"randomforestclassifier__n_estimators\": [10, 50, 100, 500],\n",
    "        \"randomforestclassifier__max_features\": [\"sqrt\", \"log2\", 0.4, 0.5],\n",
    "        \"randomforestclassifier__min_samples_leaf\": [1, 3, 5],\n",
    "        \"randomforestclassifier__criterion\": [\"gini\", \"entropy\"]}\n",
    "    \n",
    "    gs_rf = GridSearchCV(pip_rf,\n",
    "                         hyperparam_grid,\n",
    "                         scoring=\"f1\",\n",
    "                         cv=10,\n",
    "                         n_jobs=-1)\n",
    "    \n",
    "    gs_rf.fit(methods_data[method][0], methods_data[method][1])\n",
    "    \n",
    "    print(f\"\\033[1m\\033[0mThe best hyperparameters for {method} data:\")\n",
    "    for hyperparam in gs_rf.best_params_.keys():\n",
    "        print(hyperparam[hyperparam.find(\"__\") + 2:], \": \", gs_rf.best_params_[hyperparam])\n",
    "        \n",
    "    print(f\"\\033[1m\\033[94mBest 10-folds CV f1-score: {gs_rf.best_score_ * 100:.2f}%.\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "from sklearn import tree\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import VotingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      " ---Random Forest Model---\n",
      "Random Forest AUC = 0.99\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      2286\n",
      "           1       1.00      0.97      0.98       714\n",
      "\n",
      "    accuracy                           0.99      3000\n",
      "   macro avg       0.99      0.99      0.99      3000\n",
      "weighted avg       0.99      0.99      0.99      3000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "gs_rf.fit(X_train, y_train)\n",
    "print (\"\\n\\n ---Random Forest Model---\")\n",
    "gs_rf_roc_auc = roc_auc_score(y_test, gs_rf.predict(X_test))\n",
    "print (\"Random Forest AUC = %2.2f\" % gs_rf_roc_auc)\n",
    "print(classification_report(y_test, gs_rf.predict(X_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Upsampling yielded the highest CV f1-score with 99.8%. Therefore, we’ll be using the upsampled data to fit the rest of the classifiers. The new data now has 18,284 examples: 50% belonging to the positive class, and 50% belonging to the negative class.\n",
    "Let’s refit the Random Forest with Upsampled data using best hyperparameters tuned above and plot confusion matrix and ROC curve using test data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# K-Nearest Neighbors\n",
    "KNN is called a lazy learning algorithm because it doesn’t learn or fit any parameter. It takes n_neighbors points from the training data closest to the point we're interested to predict it's class and take the mode (majority vote) of the classes for the neighboring point as its class. The two hyperparameters we're going to tune are:\n",
    "\n",
    "* n_neighbors: number of neighbors to use in prediction.\n",
    "* weights: how much weight to assign neighbors based on:\n",
    "    * “uniform”: all neighboring points have the same weight.\n",
    "    * “distance”: use the inverse of euclidean distance of each neighboring point used in prediction.\n",
    "Let’s fit KNN classifier and plot confusion matrix and ROC curve."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[0mThe best hyperparameters:\n",
      "-------------------------\n",
      "n_neighbors :  1\n",
      "weights :  uniform\n",
      "\u001b[1m\u001b[94mBest 10-folds CV f1-score: 92.87%.\n"
     ]
    }
   ],
   "source": [
    "# Build KNN classifier\n",
    "pip_knn = make_pipeline(StandardScaler(), KNeighborsClassifier())\n",
    "hyperparam_range = range(1, 20)\n",
    "gs_knn = GridSearchCV(pip_knn,\n",
    "                      param_grid={\"kneighborsclassifier__n_neighbors\": hyperparam_range,\n",
    "                                  \"kneighborsclassifier__weights\": [\"uniform\", \"distance\"]},\n",
    "                      scoring=\"f1\",\n",
    "                      cv=10,\n",
    "                      n_jobs=-1)\n",
    "gs_knn.fit(X_train, y_train)\n",
    "print(f\"\\033[1m\\033[0mThe best hyperparameters:\\n{'-' * 25}\")\n",
    "for hyperparam in gs_knn.best_params_.keys():\n",
    "    print(hyperparam[hyperparam.find(\"__\") + 2:], \": \", gs_knn.best_params_[hyperparam])\n",
    "print(\"\\033[1m\" + \"\\033[94m\" + \"Best 10-folds CV f1-score: {:.2f}%.\".format((gs_knn.best_score_) * 100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      " ---KNN Model---\n",
      "kneighbors classifier AUC = 0.97\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.97      0.98      2286\n",
      "           1       0.92      0.96      0.94       714\n",
      "\n",
      "    accuracy                           0.97      3000\n",
      "   macro avg       0.95      0.97      0.96      3000\n",
      "weighted avg       0.97      0.97      0.97      3000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "gs_knn.fit(X_train, y_train)\n",
    "print (\"\\n\\n ---KNN Model---\")\n",
    "gs_knn_roc_auc = roc_auc_score(y_test, gs_knn.predict(X_test))\n",
    "print (\"kneighbors classifier AUC = %2.2f\" % gs_knn_roc_auc)\n",
    "print(classification_report(y_test, gs_knn.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import plot_confusion_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Support Vector Machine (SVM)\n",
    "SVM is comutationally very expensive to tune it’s hyperparameters for two reasons:\n",
    "* 1- With big datasets, it becomes very slow.\n",
    "* 2- It has good number of hyperparameters to tune that takes very long time to tune on a CPU.\n",
    "Therefore, we’ll use recommended hyperparameters’ values from the paper we mentioned before that showed to yield the best performane on Penn Machine Learning Benchmark 165 datasets. The hyperparameters that we usually look to tune are:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[94mThe 10-folds CV f1-score is: 92.14%\n"
     ]
    }
   ],
   "source": [
    "# Build SVM classifier\n",
    "clf_svc = make_pipeline(StandardScaler(),\n",
    "                        SVC(C=0.01,\n",
    "                            gamma=0.1,\n",
    "                            kernel=\"poly\",\n",
    "                            degree=5,\n",
    "                            coef0=10,\n",
    "                            probability=True))\n",
    "clf_svc.fit(X_train, y_train)\n",
    "svc_cv_scores = cross_val_score(clf_svc,\n",
    "                                X=X_train,\n",
    "                                y=y_train,\n",
    "                                scoring=\"f1\",\n",
    "                                cv=10,\n",
    "                                n_jobs=-1)\n",
    "# Print CV\n",
    "print(f\"\\033[1m\\033[94mThe 10-folds CV f1-score is: {np.mean(svc_cv_scores) * 100:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      " ---SVM Model---\n",
      "SVM classifier AUC = 0.96\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.98      0.98      2286\n",
      "           1       0.92      0.94      0.93       714\n",
      "\n",
      "    accuracy                           0.97      3000\n",
      "   macro avg       0.95      0.96      0.95      3000\n",
      "weighted avg       0.97      0.97      0.97      3000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clf_svc.fit(X_train, y_train)\n",
    "print (\"\\n\\n ---SVM Model---\")\n",
    "clf_svc_roc_auc = roc_auc_score(y_test, clf_svc.predict(X_test))\n",
    "print (\"SVM classifier AUC = %2.2f\" % clf_svc_roc_auc)\n",
    "print(classification_report(y_test, clf_svc.predict(X_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conclusion\n",
    "\n",
    "Let’s conclude by printing out the test accuracy rates for all classifiers we’ve trained so far and plot ROC curves. Finally, we’ll pick the classifier that has the highest area under ROC curve."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Print out accuracy score on test data\n",
    "print(\"The accuracy rate and f1-score on test data are:\")\n",
    "for estimator in estimators.keys():\n",
    "    print(\"{}: {:.2f}%, {:.2f}%.\".format(estimator,\n",
    "        accuracy_score(y_test, estimators[estimator].predict(X_test)) * 100,\n",
    "         f1_score(y_test, estimators[estimator].predict(X_test)) * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from itertools import cycle\n",
    "\n",
    "from sklearn import svm, datasets\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import label_binarize\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from scipy import interp\n",
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt  \n",
    "from sklearn import datasets, metrics, model_selection, svm\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn import svm, datasets\n",
    "from sklearn.metrics import auc\n",
    "from sklearn.metrics import plot_roc_curve\n",
    "from sklearn.model_selection import StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import roc_curve, roc_auc_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAHwCAYAAABUqPIVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABb4klEQVR4nO3debzmc/nH8dfbjH1NhgrTUCKJkp1IpaypRAmVSEqW+v0qWpVKlAoVIVJRKpHslCW7sVP0E4ooS7aS/fr98fncM9/znbPNnHO+n8+ceT8fj/OYub/3fc99zdnu6/v5fq7rUkRgZmZmZmbTzVU6ADMzMzOz2jhJNjMzMzNrcZJsZmZmZtbiJNnMzMzMrMVJspmZmZlZi5NkMzMzM7MWJ8lmVhVJH5AUA3y8eQxe7w2S9pdU1e/Dxufh5aVjmVm1fk7NzGaGf4GZWa22BdZtfVw1Bq/zBuCL+PfhaHoD/pya2WxuYukAzMwGcH1E3F46iFkhScDcEfF06Vi6JGlu4NnScZiZjQaf5ZvZbEfSApIOknSnpKfzn59tXt6XNJ+kb0u6WdK/Jf1D0m8lrdR4zP6kFU+AZ3rbOvJ9b8i339B67d42iCmNY3dJ+qmkD0q6FXga2CLft5qk0yQ9LOm/ki6V9PpZ/H9fKOkSSZtKuj7/e9dJWlvSRElfk3SfpH9J+pGkBRvPnZLj/qikb0m6X9ITkk5v/l/yY+eW9JX8/3o6//mVnAT39+8dLOle4CngOwN9TvPzviTpWkmPSnpQ0u8lrdN6/d7n/m2Svpsf90D+HC/WeuxESZ+W9EdJT+bHnd36Oi8h6QhJf5f0lKRbJe02K18DM5tzeCXZzGo1QVLzd1RExHP52DnAysABwE3AOsDngcWB/8mPnxdYGPgKcF++76PAFZJWioh/AMcAywC7ABsAz40g3o2B1wBfAu4H7pK0OvAH4DrgQ8ATwO7A+ZLWi4hrZuF1Xg58A/gq8G/gYOC0/DER+ADwyvyY+4FPtZ6/H3A9sDOwJPA14FxJr4qIZ/Jjjge2y/ddQtrq8jlgeeC9rX/vs8DVwG7ABOBaYEEG/pwuDXwbuCc/bkfgYklrRMSNrcceCpyeX3PF/H99Dnh/4zE/B95OSs7PB+YDNgReDNwqaRHgUmB+YH/gTuCtwBGS5o2IwzEz609E+MMf/vBHNR+kJC/6+bgk379Tvr1h63mfJa3gLjnAvzsBWAB4HPh44/j++d+b2Hr8G/LxNwwQ35TGsbtICfCLWo/9HfAnYJ5WHH8CTh3m5+HljWMXAs8AyzeOvS0/7vzW838N3Nm4PSU/7o/AXI3j6+fju+Tbq+Tb+7f+vc/l46u2/r1rAbUe2+/ndICvyUTgNuDQfj73x7ce/13gyd7rAW/Mj9trkNf4fH7OCq3jRwMPDhWjP/zhjzn3w9stzKxW7wDWbHzsko9vCvwVuCxfap+YV5fPBeYmrSoDIGk7SVdKeoS0V/Y/wEKkVcnRdkWk1enea88PbAT8Eni+EadIK54bzuLr/Dki7mjcvjX/eU7rcbcCy+T90U2/iojnezci4lLSqu66+VAvrp+2nte7vVHr+KkREQyTpDdLukDSQ6SvyTPAK+j/a3JG6/ZNpCsES+XbbyElyUcP8pKbAlcCd7a+X84BXki6ImFmNgNvtzCzWt0c/RfuLQm8lJRc9eeFAJK2Ak4ibR34EmnV8HngTNIl+dF2X+v24qSV0s/njxlImquZsA7Tw63bTw9yfGKOoVlM989+/s1/krZBQIobZvz//KN1PwM8bkB5+8mZpAR1l/zc50jbXvr7mvyrdfup/GfvsS8E/hUR/x3kZZckbVEZ9PvFzKzNSbKZzW4eIu0r3W6A++/Kf74HuD0iPtC7IxeetZO8gTyZ/5yndXygpKq9mvoIKSn/HvDjfp8w8wnyaFhqgGPX57/3EtMXAX9pPOZF+c+HWs8d9ioysA0pYX9nTN//jKQXkD5fM+tBYHFJ8w+SKD9E2pu99wD33zYLr2tmcwAnyWY2uzmblGz9OyJuHeRxCzBjO7KdSCurTb3VyflJ+5V7/pr/XIW0laNn8+EEGRH/kfQHYDXg2kIJcX/eJWn/XjyS1icVL16e778o//keUnFgzw75z4uH8RoDfU4XIK0cN7tdvBGYTDrxmVnnAvsCuwIDFeCdDewJ/C0i7p+F1zCzOZSTZDOb3ZxA6szwO0mHADeQVntfRipie3tEPEFKjt4u6dukDgmvA/ZixhXLP+Y//0fSWcBzETE1Iu6TdBGwn6QHSauRO+bXGa5PkJLKcyT9kLS9YAlgdWBCROw7c//1UbEwcKqkHwCTgAOB/yOvdkfELZJ+Buyf9+5eRtqv/HngZzFjB4r+9Ps5JX1N9gF+JOk40l7kzwN/n5X/SERcIOlk4FuSlgV+T9qXviFwRkRcSOqk8W7gD/l74TZSV42VgNdHxNaz8tpmNv45STaz2UpEPCPpraQVxN2A5UgFeX8hFXr19ugeDSwLfBD4MKlN2VbAKa1/8nTg+6T2cF8gFdb1it12BI4ADiNtvziW1FJusEKxZqzXSlqT1Df4MGBR4AFSR4gjZ+K/PZoOJO3R/REpWbwA+Fhz+wOpxdodpM/d54B7gYNIe7uHo9/PaUScI2kv0snDNsDNwPvya8yq9wCfzjHvAzxK+lofAxARj0paL8fxadLe60dIyfLJI3hdMxvnNBNFyWZmNpvKA0PuBD4UEccUDsfMrHpuAWdmZmZm1uIk2czMzMysxdstzMzMzMxavJJsZmZmZtbiJNnMzMzMrKXKFnBLLLFETJkypXQYZmZmZjaOXXPNNQ9GxKT+7qsySZ4yZQpTp04tHYaZmZmZjWOS/jrQfd5uYWZmZmbW4iTZzMzMzKzFSbKZmZmZWYuTZDMzMzOzFifJZmZmZmYtTpLNzMzMzFqcJJuZmZmZtThJNjMzMzNrcZJsZmZmZtbiJNnMzMzMrMVJspmZmZlZi5NkMzMzM7MWJ8lmZmZmZi1Oks3MzMzMWiaWDqAmU/Y9o/PXvOvrW3T+mmZmZmY2OK8km5mZmZm1OEk2MzMzM2txkmxmZmZm1uIk2czMzMysxUmymZmZmVmLk2QzMzMzsxYnyWZmZmZmLU6SzczMzMxanCSbmZmZmbU4STYzMzMza3GSbGZmZmbW4iTZzMzMzKzFSbKZmZmZWYuTZDMzMzOzFifJZmZmZmYtTpLNzMzMzFqcJJuZmZmZtThJNjMzMzNrcZJsZmZmZtbiJNnMzMzMrMVJspmZmZlZi5NkMzMzM7MWJ8lmZmZmZi1Oks3MzMzMWpwkm5mZmZm1OEk2MzMzM2txkmxmZmZm1jKsJFnSppJuk3S7pH37uX8HSTfmj8skrda47y5JN0m6XtLU0QzezMzMzGwsTBzqAZImAN8DNgHuAa6WdFpE/LHxsDuBjSLiYUmbAUcBazfu3zgiHhzFuM3MzMzMxsyQSTKwFnB7RNwBIOnnwNbAtCQ5Ii5rPP4KYJnRDHJONWXfMzp/zbu+vkXnr2lmZmZWm+Fst1gauLtx+558bCC7AGc1bgdwrqRrJO028yGamZmZmXVrOCvJ6udY9PtAaWNSkrxB4/D6EXGvpCWB8yTdGhEX9/Pc3YDdACZPnjyMsMzMzMzMxsZwVpLvAZZt3F4GuLf9IEmrAscAW0fEQ73jEXFv/vN+4BTS9o0ZRMRREbFGRKwxadKk4f8PzMzMzMxG2XCS5KuBFSQtJ2ke4D3Aac0HSJoM/BrYKSL+3Di+oKSFe38H3gLcPFrBm5mZmZmNhSG3W0TEs5I+BpwDTACOjYhbJO2e7z8S+ALwQuD7kgCejYg1gKWAU/KxicCJEXH2mPxPzMzMzMxGyXD2JBMRZwJnto4d2fj7rsCu/TzvDmC19nEzMzMzs5p54p6ZmZmZWYuTZDMzMzOzFifJZmZmZmYtTpLNzMzMzFqcJJuZmZmZtThJNjMzMzNrcZJsZmZmZtbiJNnMzMzMrMVJspmZmZlZi5NkMzMzM7MWJ8lmZmZmZi1Oks3MzMzMWpwkm5mZmZm1OEk2MzMzM2txkmxmZmZm1uIk2czMzMysxUmymZmZmVmLk2QzMzMzsxYnyWZmZmZmLU6SzczMzMxanCSbmZmZmbU4STYzMzMza3GSbGZmZmbW4iTZzMzMzKzFSbKZmZmZWYuTZDMzMzOzFifJZmZmZmYtTpLNzMzMzFqcJJuZmZmZtThJNjMzMzNrcZJsZmZmZtbiJNnMzMzMrMVJspmZmZlZi5NkMzMzM7MWJ8lmZmZmZi1Oks3MzMzMWpwkm5mZmZm1OEk2MzMzM2txkmxmZmZm1uIk2czMzMysxUmymZmZmVmLk2QzMzMzsxYnyWZmZmZmLU6SzczMzMxanCSbmZmZmbU4STYzMzMza3GSbGZmZmbWMqwkWdKmkm6TdLukffu5fwdJN+aPyyStNtznmpmZmZnVZsgkWdIE4HvAZsDKwPaSVm497E5go4hYFTgAOGomnmtmZmZmVpXhrCSvBdweEXdExNPAz4Gtmw+IiMsi4uF88wpgmeE+18zMzMysNsNJkpcG7m7cvicfG8guwFmz+FwzMzMzs+ImDuMx6udY9PtAaWNSkrzBLDx3N2A3gMmTJw8jLDMzMzOzsTGcleR7gGUbt5cB7m0/SNKqwDHA1hHx0Mw8FyAijoqINSJijUmTJg0ndjMzMzOzMTGcJPlqYAVJy0maB3gPcFrzAZImA78GdoqIP8/Mc83MzMzMajPkdouIeFbSx4BzgAnAsRFxi6Td8/1HAl8AXgh8XxLAs3lVuN/njtH/xczMzMxsVAxnTzIRcSZwZuvYkY2/7wrsOtznmpmZmZnVzBP3zMzMzMxanCSbmZmZmbU4STYzMzMza3GSbGZmZmbW4iTZzMzMzKzFSbKZmZmZWYuTZDMzMzOzFifJZmZmZmYtTpLNzMzMzFqcJJuZmZmZtThJNjMzMzNrcZJsZmZmZtbiJNnMzMzMrMVJspmZmZlZi5NkMzMzM7MWJ8lmZmZmZi1Oks3MzMzMWpwkm5mZmZm1OEk2MzMzM2txkmxmZmZm1uIk2czMzMysxUmymZmZmVmLk2QzMzMzsxYnyWZmZmZmLU6SzczMzMxanCSbmZmZmbU4STYzMzMza3GSbGZmZmbW4iTZzMzMzKzFSbKZmZmZWYuTZDMzMzOzFifJZmZmZmYtTpLNzMzMzFqcJJuZmZmZtThJNjMzMzNrcZJsZmZmZtbiJNnMzMzMrGVi6QBs9jFl3zM6f827vr5F569pZmZm5pVkMzMzM7MWJ8lmZmZmZi1Oks3MzMzMWpwkm5mZmZm1OEk2MzMzM2txkmxmZmZm1uIk2czMzMysxUmymZmZmVmLk2QzMzMzs5ZhJcmSNpV0m6TbJe3bz/0rSbpc0lOS/rd1312SbpJ0vaSpoxW4mZmZmdlYGXIstaQJwPeATYB7gKslnRYRf2w87F/AXsDbB/hnNo6IB0cYq5mZmZlZJ4azkrwWcHtE3BERTwM/B7ZuPiAi7o+Iq4FnxiBGMzMzM7NODSdJXhq4u3H7nnxsuAI4V9I1knYb6EGSdpM0VdLUBx54YCb+eTMzMzOz0TWcJFn9HIuZeI31I2J1YDNgD0kb9vegiDgqItaIiDUmTZo0E/+8mZmZmdnoGk6SfA+wbOP2MsC9w32BiLg3/3k/cApp+4aZmZmZWbWGkyRfDawgaTlJ8wDvAU4bzj8uaUFJC/f+DrwFuHlWgzUzMzMz68KQ3S0i4llJHwPOASYAx0bELZJ2z/cfKelFwFRgEeB5SfsAKwNLAKdI6r3WiRFx9pj8T8zMzMzMRsmQSTJARJwJnNk6dmTj7/8gbcNoewxYbSQBmpmZmZl1zRP3zMzMzMxanCSbmZmZmbU4STYzMzMza3GSbGZmZmbW4iTZzMzMzKxlWN0tzGo1Zd8zOn/Nu76+ReevaWZmZt3ySrKZmZmZWYuTZDMzMzOzFifJZmZmZmYtTpLNzMzMzFqcJJuZmZmZtThJNjMzMzNrcZJsZmZmZtbiJNnMzMzMrMVJspmZmZlZi5NkMzMzM7MWJ8lmZmZmZi1Oks3MzMzMWpwkm5mZmZm1OEk2MzMzM2txkmxmZmZm1uIk2czMzMysxUmymZmZmVmLk2QzMzMzsxYnyWZmZmZmLU6SzczMzMxanCSbmZmZmbVMLB2A2XgyZd8zOn/Nu76+ReevaWZmNt55JdnMzMzMrMVJspmZmZlZi5NkMzMzM7MWJ8lmZmZmZi1Oks3MzMzMWpwkm5mZmZm1OEk2MzMzM2txkmxmZmZm1uIk2czMzMysxUmymZmZmVmLk2QzMzMzsxYnyWZmZmZmLU6SzczMzMxanCSbmZmZmbU4STYzMzMza3GSbGZmZmbW4iTZzMzMzKzFSbKZmZmZWcuwkmRJm0q6TdLtkvbt5/6VJF0u6SlJ/zszzzUzMzMzq82QSbKkCcD3gM2AlYHtJa3ceti/gL2Ab87Cc83MzMzMqjKcleS1gNsj4o6IeBr4ObB18wERcX9EXA08M7PPNTMzMzOrzXCS5KWBuxu378nHhmMkzzUzMzMzK2I4SbL6ORbD/PeH/VxJu0maKmnqAw88MMx/3szMzMxs9A0nSb4HWLZxexng3mH++8N+bkQcFRFrRMQakyZNGuY/b2ZmZmY2+oaTJF8NrCBpOUnzAO8BThvmvz+S55qZmZmZFTFxqAdExLOSPgacA0wAjo2IWyTtnu8/UtKLgKnAIsDzkvYBVo6Ix/p77hj9X8zMzMzMRsWQSTJARJwJnNk6dmTj7/8gbaUY1nPNzMzMzGrmiXtmZmZmZi1Oks3MzMzMWpwkm5mZmZm1OEk2MzMzM2txkmxmZmZm1uIk2czMzMysxUmymZmZmVmLk2QzMzMzsxYnyWZmZmZmLU6SzczMzMxanCSbmZmZmbU4STYzMzMza3GSbGZmZmbW4iTZzMzMzKzFSbKZmZmZWYuTZDMzMzOzFifJZmZmZmYtTpLNzMzMzFqcJJuZmZmZtThJNjMzMzNrcZJsZmZmZtbiJNnMzMzMrMVJspmZmZlZi5NkMzMzM7MWJ8lmZmZmZi1Oks3MzMzMWpwkm5mZmZm1OEk2MzMzM2txkmxmZmZm1uIk2czMzMysxUmymZmZmVmLk2QzMzMzsxYnyWZmZmZmLU6SzczMzMxanCSbmZmZmbU4STYzMzMza3GSbGZmZmbW4iTZzMzMzKzFSbKZmZmZWYuTZDMzMzOzFifJZmZmZmYtTpLNzMzMzFqcJJuZmZmZtThJNjMzMzNrcZJsZmZmZtbiJNnMzMzMrMVJspmZmZlZy7CSZEmbSrpN0u2S9u3nfkk6LN9/o6TVG/fdJekmSddLmjqawZuZmZmZjYWJQz1A0gTge8AmwD3A1ZJOi4g/Nh62GbBC/lgbOCL/2bNxRDw4alGbmZmZmY2h4awkrwXcHhF3RMTTwM+BrVuP2Rr4cSRXAItJevEox2pmZmZm1onhJMlLA3c3bt+Tjw33MQGcK+kaSbvNaqBmZmZmZl0ZcrsFoH6OxUw8Zv2IuFfSksB5km6NiItneJGUQO8GMHny5GGEZWZmZmY2NoazknwPsGzj9jLAvcN9TET0/rwfOIW0fWMGEXFURKwREWtMmjRpeNGbmZmZmY2B4STJVwMrSFpO0jzAe4DTWo85DXhf7nKxDvBoRNwnaUFJCwNIWhB4C3DzKMZvZmZmZjbqhtxuERHPSvoYcA4wATg2Im6RtHu+/0jgTGBz4HbgCWDn/PSlgFMk9V7rxIg4e9T/F2ZmZmZmo2g4e5KJiDNJiXDz2JGNvwewRz/PuwNYbYQxmpmZmZl1yhP3zMzMzMxanCSbmZmZmbU4STYzMzMza3GSbGZmZmbW4iTZzMzMzKzFSbKZmZmZWYuTZDMzMzOzFifJZmZmZmYtTpLNzMzMzFqcJJuZmZmZtThJNjMzMzNrcZJsZmZmZtbiJNnMzMzMrMVJspmZmZlZi5NkMzMzM7MWJ8lmZmZmZi1Oks3MzMzMWpwkm5mZmZm1OEk2MzMzM2txkmxmZmZm1uIk2czMzMysxUmymZmZmVmLk2QzMzMzsxYnyWZmZmZmLU6SzczMzMxanCSbmZmZmbU4STYzMzMza3GSbGZmZmbW4iTZzMzMzKzFSbKZmZmZWYuTZDMzMzOzFifJZmZmZmYtTpLNzMzMzFqcJJuZmZmZtThJNjMzMzNrcZJsZmZmZtYysXQAZjZ2pux7RuevedfXt+j8Nc3MzEabV5LNzMzMzFqcJJuZmZmZtThJNjMzMzNrcZJsZmZmZtbiJNnMzMzMrMXdLcysM+62YWZmswuvJJuZmZmZtXgl2czmaLWtbtcWj5nZnMoryWZmZmZmLcNaSZa0KXAoMAE4JiK+3rpf+f7NgSeAD0TEtcN5rpmZ1csr22Y2pxoySZY0AfgesAlwD3C1pNMi4o+Nh20GrJA/1gaOANYe5nPNzMyGxUm7mXVlOCvJawG3R8QdAJJ+DmwNNBPdrYEfR0QAV0haTNKLgSnDeK6Zmdlsq7bEvbZ4zGZXw0mSlwbubty+h7RaPNRjlh7mc83MzGycqi1pry0eqC+m2uIpRWnxd5AHSNsCb42IXfPtnYC1ImLPxmPOAA6MiEvy7d8BnwKWH+q5jX9jN2C3fHNF4LYR/t+6tgTwYOkgGhzP4BzP0GqLyfEMrrZ4oL6YHM/gaosH6ovJ8QyutniG46URMam/O4azknwPsGzj9jLAvcN8zDzDeC4AEXEUcNQw4qmSpKkRsUbpOHocz+Acz9Bqi8nxDK62eKC+mBzP4GqLB+qLyfEMrrZ4Rmo4LeCuBlaQtJykeYD3AKe1HnMa8D4l6wCPRsR9w3yumZmZmVlVhlxJjohnJX0MOIfUxu3YiLhF0u75/iOBM0nt324ntYDbebDnjsn/xMzMzMxslAyrT3JEnElKhJvHjmz8PYA9hvvccaq2rSKOZ3COZ2i1xeR4BldbPFBfTI5ncLXFA/XF5HgGV1s8IzJk4Z6ZmZmZ2ZzGY6nNzMzMzFqcJJuZmZmZtThJNjMzMzNrcZI8TkhapXQMg5E0l6RFKohjueEcmxNJmlA6hjZJBw3nmNVD0k+Gc6yjWOaStF2J1x5IHtA15LE5maQFJc2V//4KSW+TNHehWCZK2krSJ/PHlpKG1fTAZn8u3JsFkn4LDPiJi4i3dRgOAJIuIQ1v+RFwYkQ80nUMbZJOBHYHngOuARYFvhUR3ygY07URsXrr2DUR8bpC8awP7A+8lNRtRqSGMcsXiOVO4FfAcRHxx65fvz8DfL1ujIhVC8VzMPAV4L/A2cBqwD4R8dOO4zicwX8H7dVhOH20v2b55OumiFi5UDwXR8SGJV67PwN8T89wrMN43tnP4UdJX7P7u44H0u9k4PXAC4ArgKnAExGxQ8dxvAS4ALgPuI70+/m1wIuAjSOi3+FoHcR1WD+HHwWmRsRvCsRzEzP+PnqU9HX7SkQ81HVMo8VnQ7Pmm6UDaIuIDSStAHwQmCrpKlKyc17BsFaOiMck7UBqA/hpUrLceZIsaSXgVcCirTeFRYD5uo6n4YfAx0mfl+cKxgGwKmngzzF5FedY4OcR8VjXgUj6CPBRYHlJNzbuWhi4tOt4Gt4SEZ+S9A7SpNFtSW+inSbJpDefqkjaD/gMML+k3veMgKcp2xbqPEn/C5wE/Kd3MCL+1WUQkjYjzRNYupXkLAI822UsLbsA65K+jwHeQEpMXyHpyxFR4iqAIuIJSbsAh0fEwZKuKxDH14AjIuI7fYKT9gIOBN5fICZI71krAb/Mt7cBbgF2kbRxROzTcTxnkd6/Tsy335P/fIy0cLdVx/GMGq8kj5Ck+YHJEXFb6Vhg2qrN24HDSN+gAj4TEb8uEMstwGtIPzjfjYiLJN0QEasViGVr0uflbfSd+vg4KRG8rOuYclxXRsTaJV57MJI2BH4GLEZaXT4gIm7v8PUXJa0iHQjs27jr8a6TmyZJt0TEqyQdDZwcEWeX+p6uUT65OiYiPlg6lp58haSt86s1klYj/T78MvCFxl2PAxdExMNdxtOI67fArhHxz3x7KeAIYFfg4ojofCtfTog/Cnwb2CUPMLspIl7dcRy3RsRKA9x3W0Ss2GU8jdf+PemE/dl8eyJwLrAJBa7aSLo0Itbv71iJr9to8kryCEjairSqPA+wnKTXAF8utN1iVdKkwy2A84CtIuLafLnocqDzJBk4ErgLuAG4WNJLSYl75/IlqN9IWjciLi8RwwAukPQN0tfnqd7BiLi260DyCdYWpO+jKcAhwAmky55nAq/oKpaIeJR0uW77HNdSpN9XC0laKCL+1lUsLb+VdCtpu8VHJU0CniwUC/n1Pw2sTOOKSES8sUQ8EfF8TgarERFV1BxExA3ADZJOjIhnSsfTMKWXIGf3A6+IiH9JKhXn3sB+wCk5QV6e6SvdXfrvIPc90VkUM1oaWJD0O5L895dExHOSnhr4aWNmIUlrR8SVAJLWAhbK95W8SjJiTpJHZn9gLeBCgIi4XtKUQrF8FziGtGo87Qc7Iu6V9Lmug8krSv+MiKUbx/4GbNx1LC27S/pTb8+2pBcAhxRc+eqtIq/ROBZAiSTn/0hvRN9oraz/Kq8sdy6Ptd8f+CfwfD4cpK0hnYuIfXPh4GP5DekJYOsSsWQnkLYRbEHa//9+4IGC8QBcIWnNiLi6cBzTSFqPdOI37T0vIn5cKJy1JO1PBXUI2R8knU7fS/cXS1oQeKTrYPJJ8VbNxaaIuAMosc++vT2vR6RtMqUcDFwv6cIcy4bA1/LX7PwC8ewKHCtpoRzPY8CuOZ4DC8QzarzdYgR6l8olXRcRr83HihUV1aa2ghlIl/F6X6vBjs1p8hvTZyPiy6VjaZJ0O7B2LYUfkhYAPkHaYrVbrgNYMSJOLxTPNRHxuubvHUkXRcRGJeLJr/9H0lWHv5L2APeSwFLFlj8BXgZcz/R9/1GquDFfiZihDqHU97gkkRLj9Ulfq0tIW4mKJQeSfl/qakgrjuMGuz8idu4qljZJLyYt0gm4qlQRYVPeJqcaGgeMFq8kj8zNkt4LTMhvlnsBpfa2VtMloaGKgpmWuSS9oLf/T9LiFPw5yL9UvkhaCQC4iLRl59GBnzX68qroxqT9kjW5m+mXFGtwHCm5WS/fvoe0AlckSQZ6l8Pvk7QFcC+wTKFYejYr/Ppta5CKiGtZEXo0Is4qHURP/rz8Kn/U4jpJp5F+tprvHZ1uGyyZBA/DXKSrRhOBl0t6eURcXCIQSfOSTrSmABPTeRfUtugyK5wkj8yewGdJe0lPBM4htYcqoaYuCT29LQx7NI4FUDJxPwS4TNKvcizbAV8tGM+xwM05DoCdSIlYf5f4xtplkr7LjCc1ne+PbrgDuFDSGfTds/2tQvG8LCLeLWn7HMd/1XtHKOMr+UTrf4DDSZeAP14wHiLir3lf8uvzoT/k/bil3Exq2XVfwRiaqqlDgGkt4A4CliQtrvQWWEpuJ1gceIi+286CjmtrJH1isPtL/R7KW77eTepo0dyGViRJBn5DWsy4hsb39Hjg7RYjIOm1EVGiLc0Mau2SUCNJK5N++Qr4XRTsCSzp+oh4zVDHOoqlv8KYKHnZU9IX+zseEV/qOhYASZcBbwIujYjVJb0M+FlErFUinhpJ2hv4ENMTmncAR0XE4R3H0etnvzCpq8RV9E1KOy+wznFV9XOWtzRtFRF/KvH6NWv8/lkRWJPpnZG2InX+2LVQXLcBq0ZEFQmppJtLdEHpgpPkEci/7F5MuiT084i4pWAsXwcmUMnqRI7pff0dL1gwA4CkDYAVIuK43B1goYjor01UF7FcDnwyIi7Jt9cHvhkR65aIp1aSFoyI/wz9yDGPYxPgc6RuEueS9nF+ICIuLBTP8cDeFRWiotTXet3e1ysX71ze9Z5kSYPuy46Ii7qKpWb9te8qTdJ8pP7Nr6Jv15Yi39eSzgW2iYjH8+2FgV9GxKaF4jkL2DYi/l3i9dskHUXqZ31T6VhGm7dbjEBEbCzpRaRL5UcpjV0+KSJKbLmoqUtCz5qNv89HWoG7FiiWJOeVgTVIKwPHAXOTBkGUepP4CHB8r+AB+BfwgUKxkPe1tt+Yiu0rk7QuaSvRQsDkfBn/wxHx0RLxRMR5kq4F1iF9vfaOiAdLxJKt2iySiYiHJZUuQhV9t3w9l491arhJsKTLuzwpVepD/DVSy67N8pWtdSPih13F0DJV0knAqfRdYCnRNrTnJ8CtwFtJdRI7ACVXuieThuL0PE3af1vKE6TuFr+j79es1KTNDYAPKPUkf4rCxbqjyUnyCEXEP4DD8qryp0hN4jtPkiOidGu1GUTEns3bOREsMb2p6R2ksaLXwrQWeQuXCiYirgdWyydYRIHpdj2SjgQWILXpOwZ4F+kSdUnfIb1Rngap12yJdnSS2iODe/tbJ0uaXPCKTVWFqNlxwJWSTiG9WW5NOtGpVdcTN39E+hx9Nt/+M6kOoNTnaBFS0vWWxrHO9/+2vDwitpW0dUQcL6lX81PKT4Cr8vd0kN5HSl4RPY2+Q7FKq61Yd9SU/mU6W5P0StLm+W2BB4GfkwpoSsVT1SpgP54AVigcw9MREZICpl0K7pykHSPip+3CkEZVcImCkPUiYtXcTuxLkg6h7BslABFxd6s2rkRh6iGD3Ffyik2zEBXS76KShahExLeU+rdukA/tXEvtxgC63nO4RET8QmmMNxHxrKRixdaVdnDodW15RNIqwD8ouHIbEV+VdDaVfE9HxPGlXrtJ0iJ5Yefx0rGMFSfJI3McaXTvJqV7FNa4CtgonIG0X/qVwC/KRQTALyT9AFhM0odIHTiOLhBHLznvbxW7VKFAbwjNE0qTGh8CSk8ru1tpEERImofUZrHzy641XqmBtL9f0lSmF6K+s2QhaotIlfclu3/U6D+SXkj+OZe0DgXaHEr6VEQcLOlw+vmdU/DSPaTtiy8APk9aMV2IvqO8S7iedAVpIkC+glRk8qekLYEDmLHla9cdSU4EtiR1tQj6/qyX7mQ1Kly4N0KS5icNFritcBw3NlYBV1WafPPriHjLkE8eu5iahTPPAn+NiHtKxdOTi6/eQvqBPicizisYy/oRcelQxzqK5fOkNmJvAr5H+iV3TER8vutYGjEtARwKvJn09TqXtA+42HCRvLLVHgPd6aXX3gpO3l4xgyjYi1zSF0gr2ieTvmZvJxU5lWqPOSh1PEwob905HFiF1J5uEvCuiLixqxhyHFtFxG8lvb+/+2tZrayBpD1J/ez/yfQ99sX23OaOJO8EbgoncWPKSfIISNoK+CYwT0QsJ+k1pEEQnbcW0vTpf1eQfngeAm6OiKLbG3KRSq+A76qIuL9kPLWRdG1ErD7Usa4pNYefLzoealK7XPj5BlKSfCZpL94lEfGujuM4PSK2zIUyzV/ixYcISfoT8NqIeDLfnh+4NiJeWSiegyLi0wMdk7RKRNzccUwTScXDAm6LiGeGeMpYxrJtRPxyqGMdx1RVcaPqm/x5AfCmiHh+yAd3QNLvIuJNQx2bHXm7xcjsTxoLeSGkIixJUwrFcrqkxYBvkIrSgjLbCKaRtF2O50LSm8Hhkj4ZEZ1PdpJ0SURsIOlx+t/O8BDwjYj4fkfxrEua2japtS95EdLWlM4oDRMY6L4iVe4DXQLuKXgp+F3AasB1EbFzfjM/pusgImLL/Gfp7TD9uYu0yv5kvj0v8Jdi0cAmwKdbxzbrHSuQIE8ANidPJwPekn/OSg3I2Y/UxnSoY136EXUVN9Y2+fNTwJmSLqLgkCWlVn0LAEvk7TG97RaLAC/pMpax4iR5ZJ6NiEdVdOBWEhEH5L+eLOl06lgF/CywZm/1WKkn8fkUGH8aERvkP/vtZJH3CF4GdJIkA/OQ9tlNpO++5MdIiViXthrkvlJV7lPzn+uTVm1Pyre3Je1/K+W/EfG8pGdzR5L7KbzvTtLSTN+bCEAUGk+bPQXcIuk80vfPJsAlkg7LsXVygiPpI8BHgeWVejf3LAx0vp2p4bekE4ibmD4trXOSNiMl60v3vjbZIqTtcSVVVdxIfZM/vwr8m3QyOk+hGAA+DOxDSoibHX4eI23Zm+05SR6ZmyW9F5ggaQVSUdFlJQLJZ3QfJVXfBulN6YjeJc9C5mptr3iING++KPUdmXtxRNwYEQ9JekNXMeQerhdJ+lFE/LWr1x0gluqq23v7ISV9ANi4dzk6F6ieWzC0qfmKzdGkZP3fFCyQ1fTxtH9ketePkuNpAU7JHz0XForjROAs4EBg38bxx0vu2QaWqaR/7L2kk9G30ffE83EKjzankuLGhr/lj3kom5T2LF6y3qgnIg4FDpW0Z3Q8UbMr3pM8ApIWIK2WTisCAw4okZhK+gXpl9tP86HtgRdExLZdx9KI6RvAqqQOIJDezG9s7w/sOKYqRuY24plEunTWbt1XakRtVW0ElcavrttLavIlvSsiYsVSMfXkrVWLdF1w1YqhqvG0PbkTyUqkJOe2iHh6iKeMdTwTgKXou9peqjPBQcDvIqLkyd40kuYuuSe6P5JeBxxG4eLGWilN2P19Rd9DC5JOrCZHxG550XDFiDi9cGgj5iR5nJB0Q0SsNtSxrknahnTJXKRV21OGeMpYx1PFyNxGPOeSthL8L7A78H7ggRInEhqgjWBE7NJ1LI2Ydibt/b8gH9oI2L9U5b3S2PDrI+I/knYEVgcOLXU1QJWNpwWQtDnwA9I+ZJHaCH44Is4qFM/HSN9D/2T69oaSnQneQVrMmIvUD7hU+65ePOuTPj/tdmKltxHVVNxY22LG46Q2ok9Rx/fQSaSrEe+LiFVyse7lEfGaEvGMJifJs0B9+//OoFB3ix8BR0bEFfn22sD7o9D43lpJuom0T7pXeT8fcHVEvLpQPNdExOuUW/flYxdFxEZDPXcMYqmujWCO60VMH7t+ZaQpl6ViuZFUuLcqaQrXD0m9iTv/euV4Ts7x1DKeFkm3AltGxO359suAMyJipULx1NaZ4A5SW7wq2nflr9fHSUnOtH2/JT9fSr2/jwV+FnmaZEk1LWYMh6RXRcQtHb7e1IhYQ412ijUs0o0G70meNd8sHUA/1gbeJ6l3CXEy8KecFBZZNcldEw4CliSd6RY9282OZfrIXEhvViVH5vZWR+7LWx3uBZYpFEtvm1BvmMi/KDRMRNJKEXGrpo+Dvjv/+RJJL4lyY6CfjYiQtDVpBfmHGqDPbEdqG08LcH8vQc7uIBU4llJbZ4L/I7XnLJ4gZ4+WWuUfxHuAnYGrc8J8HHBuwc/ZC/PP+t6NepKLCsUyHD8hXeXqytN59bi3h/xlNE7aZ2dOkmdB/iEZkqSTI2KbsY4n27Sj15kZBwNbRUTnE9L6I2ku4ErgIlKBoyg/MvcrkhYljTM/nFRZXqpo5reqp43gJ4Dd6H8cdMkx0I/nivsdgQ3zXte5C8VS68CHWySdSZquGaSOJFfnk+YSLQVr60xwX47nrEriuSDXj/y6FU+pE1HySdZnlQYcbUla3Hhe0rGkk9OuCy9rWswYjq5bbn0ROBtYVtIJpC2WH+g4hjHhJHlsdbanKyL+moualqVvcUqxX3TAP2tJkAFy665DImJd+rarKaZR2PAoaS9wSbcCz0XEyUrN+1cHTi0RSETslv8s/TlpezfwXmCXiPiHpMmkk4oiKt1POh9p/29vC8oDwOKkVoMlWgrW1pngzvxRSzy9rUxrNI6VPBEFQNKqpNXkzUnTG08gLW78HnhNx+H0t5ixT8cxzIyuV9yvIQ0xW4f0O2hv+rY2nW15T/IYUoeT0yQdQDpz+wvTf0CiRGGBpg+n2Ah4ESnRaq5QlOi7C4CkLwE3kvbaFv/ml3Qc/fxCi4gPFoiltxd5A9K0q0OAz0TE2kM8dSxj2gM4ISIeybdfAGwfHQ19mVmSLs8nYV29XnX7SWslacFewW4NaounJpKuAR4hbYU7udm9RdKvI2LAAUhdkbR+RJTstz2gLnOP/HqXAptFxGP59itJo+hX6SqGseKV5PFjO+BlpVstZc3hFE+QWuT1lBpO0fMJUlXwc5J6e3BL7pNutsiZj9SS7t5CsfSSrC1IRaC/kbR/oVh6PhQR05rSR8TDkj5Ed0NfZtZ8Qz9kVFW3n7SmEz+YNt3yh6ThPZOV+qR/uFRRc4XxfKG/41Gw9SOpY8sd/d3RZYKct1NtBywNnB0RN0vaEvgMMD/w2q5imUld5wFfI23X25zU+vHHwA4dxzAmnCSPrS73Bd0MLEbZAhlg+MMpJO0XEQeOdTxNMcDEvVIi4uTmbUk/I00lLOHvkn4AvBk4SNK8lB/+Mpck9Vb985tWDZeoB9LJ1YlGQWN1+0mp68QP4DvAW8kFjhFxg6QNHc80zdXs+Uh7gItuk+slyPmq1lqkQscSPYF/SNrCeBVwmKS/AusC+0bEqQXiAUDS7yLiTQMdi4h1uownIs6QNDdwHmmbxdsj4v+6jGGsOEkeW122hzkQuE7SzfR9s+y8Hd1M2JYUd6fydpDeZMI/lPxl148VSJ1JStiOVAD6zYh4RNKLgU8WiqXnHOAXSj2cg9R+6eyyIVWhXdBYzX7Syk78AIiIu6U+axYlRxxXFU9E9PlekvRNCnVMkXRVRKyV//4hYA/S9MYvSlo9Ir7ecUhrkIb1PJ/bhT4IvLxUG8ocwwLAEnnrWe+baBHSaOiu4zmcvgsDi5AKZfeUVLQV5WhxkjwCQxXNdHzmezyp3dpNTG+YX7uuK3CR9H3g5UyfAri7pE0iYo+uY8nxPE76JaP85z/o9uRqmoh4gsZWmIi4j1SJX9KnSV0uPkL6HJ1LGnRSq06+pyssaBxMyRM/gLslrQeE0iTAvSi7UlpbPG0L0GHReUuzU8xuwCYR8UBO3K8Auk6Sn46I5wEi4klJfy6VIGcfJhUMvoRUh9D7ffMY8L0BnjOWprZuX9Pvo2ZjLtwbgZqKZlRoAMVIdF1ckF/zFmCVxuX7uUhN/V/VZRw2PEoTEZ+MiOfy7QnAvDmhLxHPQdEaINA8JmmViLi5w3i+BhzcKmz8n4j4XFcx9BNT78Sv5x/Afu0V5g7jWQI4lLSNqHeitXep4sYK47mJ6V+vCaQR0F+OiO8WiOUG4A2kbV7nRMQajfuuizyoosN4ngB6Pb8FvCzf7i2IlZrauGdEHF7itdvy7+TjI2LH0rGMBa8kj0xNRTPXSDqQdJmslr2JQ+l8JRm4jbSq1RsjvCyp20WnGntK+1X5161LvyMlE72xy/OTkor1CsWzCTOu9G/WO9Zlgtx77Yj4TO9GLmzcHCiWJFe47/9BKioiqi0e0h7knmdJrTufLRTLokxfIQ1JL8qtFheizPvFKwu85nA8L2mxGrr+RMRzkiZJmqeSxgGjyknyyNRUNNM7w25u2C+6N1HS4jF40/dfdhbMdC8kTSK8Kt9eE7hcUq+Ipqs93P0Nyegp3qO0IvNFRC9BJiL+LWmBroOQ9BHgo8DySqOpexYGSraBmiBp3l6LLKWpV/MWjKe3De36iPiPpB1J/bYPjYi/DvHUsYpnOWBPYAp9e8gXqdeQdFg/hx8FpkbEb7qOJ1KP/dWA1+dDF1Ng4SDHMmWAu54nFYACKSmMDsZVD/d7tuvWj9TX9ecu4NL8PjqtEDTKDcgZNU6SR6aaJuyV7lG8UtL1pJGiZ/W2OPRExNcKxNRvu6OuVfr1qtF/csHOtQCSXgf8t0AcJwJnkQpN920cf3yIE8Gx9lPgd422ax8k1SeUdASwWk68PkXqEPBjpg8X6dqpOYbfUke9xnykNlm9RYJtgFuAXSRtHBH7dBmMpL2BDzG9HuEESUfVcjkfptVL3Nk49Du6Hbs8lK5bP9bW9efe/DEX42SISI/3JI8TStOAvgj0WgldRNpX9mjBmES6VP5BUhufk4AfRcSfS8U0lAIrAkhaBViZxi/aiPhxlzHUStKawM+Z3kLsxcC7I6JYgUh+Q1qKvquSfysYz6Y09rdGxDmlYsnxXBsRq+f+u3+PiB+WqD9oxHNlFByI0ybp98BbelsaJE0kbSHahFQfsXLH8dwIrBt5sEmuA7i81H7b4SixP3kwXX9/5yvYU4Bm15+7I+J/uophTuGV5BGoLDE9ltQrebt8eyfSCm6xyUT5LPc84DxJG5NWvT6aizP2jYjLS8U2iE5XBCR9kVSosjJwJml/6yWklbc5XkRcLWklYEVSEnhrRDxTKh5JHyN1tPkn01clAyiZUPwJeDYizpe0gKSFI+LxgvE8Lmk/YEdgw3xSMfcQzxlLh+afs3Mpvy0O0mCKBUlbLMh/f0ne2/nUwE8bM6JvC7rnKLP/d2bM6at7nyZ1uqii64+kSaSrRq+i72LPbL9t0EnyyNSUmL4sIrZp3P5S3upQjKQXkt4odyIlFXuSCgtfQ7rUuFyx4AbW9S/fdwGrAddFxM6SlqLuFmclrMj0lfbX5v6bpU4i9gFWLNWJoC3vQ9wNWJxUeb80aXXpTYM9b4y9G3gvsEsuupoMfKNgPK8m/Q56I31PbEq9gR8MXC/pQlKCsyHwtbyCW6Kf9HGkrXGn5NtvJ21PseHr9KQit6U7In/U4ATSleItSava7wceKBrRKPF2ixGQdH1EvGaoYx3FcjnwyYi4JN9enzQUotOtA62Y/gz8BDguIu5p3ffpiDioTGQDK3DZ7KqIWEvSNcDGwOOk6VJuScfAK+0R8a5C8VxA6t1aqvq/j3wivBZwZe/ys6SbIuLVRQMbRNdbmnKrzlVrqrxXGtSzFim5uioiSk4k7HXb2SDHc3FEXFcynqF0vd2iwtaPd9L/6Pci/a0lXRMRr5N0Y2+bzuzYlrY/Xkkemf9K2qCVmJYoKoJ02eX4vAUE4GHgA4Vi6VmxXazXU2OCnHV9mXGqpMWAo0mtj/5NGoFqSW0r7XcAF0o6g76X7ktVcT8VEU8rT2/L+1trX/nousjpBmAx4P6OX3cwT5IG9cwHvFzSyyPi4hKBSFoHuKVRHLuwpLUj4soS8eQYfhIROw1yrOsrJbW1fmw2C5iPNL128Y5jaOptgbtP0hakGpJlCsYzapwkj0wzMRXwLwolphFxPamifJF8+7EScQBI+i35jVqaMecs2HppAqlB/ZsHedhOg9w36iLio/mvR0o6G1gkIoq0X6rUfyONhH02f2/fT7lpYAB/yx/zULaavOciSZ8B5pe0CalN3W8LxzSUrpP4pYBbJV1N3xObUr+HdgX2JiUR15Padl5Oue0fR9C3U8R/+jnWtT5X0vLv7tf1bnfVUabW1o/9bPf6jqRLKNe96Ss5D/of4HDSeOqPF4plVDlJHoHKEtOaJm99s8BrDikXxjwhadGBiiu7XhGQ9BvSXq7fRMRdXb72bKKqlfaI+BKkDgC9bgCF7QvsQhpH/2HgzIg4umxI1fli6QBa9ib1Z78iIjbOhalfKhjPtFZikPa75isS3QeSCj57J32991MBTwNHFQipytaP6juMai7SynKx1msRcXr+66OkbYPjhvckzwJJO0bETyV9or/7S1x67W+PVsm2S/n1946IQ4c61nFMvyCt3JxH36bnexWKZyNSodMWpOTvJOD0iHiyRDw1kzSF1kq7pFdFxC0dxrAuqahpoYiYrNQL+MONKwKdqvFnbCi17SftmqSrI2LNvJ987Yh4qlQtS47n18CFTC8C+yiwcUS8vUQ8OaYDI2K/Uq/fH1XU+jHXRvQ8Sxrm8c2IuK1QPFUN7BlNTpJngaQPR8QPclFRW0TElwvEdCOwZvSdvDW1ZAFYf0l612+Q/cT0/v6OR0TRAQz5F/AbSU39N42IRUrGM7soUGh5JWmf9GmNQrmbI2KVrmJoxVPjz1htRU79fY6mFRh1LXeR2JnUKeWNpPqRuSNi80LxLAkclmMJ0qCOfSKi2B5u1Te1sd/Wj6W+h2qj1Nb1h6QrWtMG9kTERcWCGiVOkkdA0voRcelQxzqK5VPA20jtfHqTt06LiIMLxLI9qQXUBsAfGnctDDw3xJ7gMZdPICaXOutuy/FsRVpRXp20krxn2ahmDwVWJa+MiLWbryvphohYrasY8mtW+zNWS1La2E/6MuD2xl0LA5dGxI5dxtOffCVpUeDsqKj7Rml50Wc1Uv/xn5ASsHeW6pYg6XbSqn/R1o8DXb3uKVVArMoG9owm70kemcOZsbihv2NjLiIOzr9YepO3Dohyk7cuI1VuLwEc0jj+OFC0KE3SVqQ90/MAy0l6DWkATKkinpNI483PBr4HXBipB6YNT9dn+XdLWg8ISfMAe5GGeXStup+xCoucqtxPCn0u3fdGLb+IVBBaIpbjgb1b9SyHRMQHS8STPRsRIWlr0gryDwe6CtiRu5k+/KWk3r7jFUn72k/Lt7cCinRHyQ5VXQN7Ro1XkmdB3pe4Huly2bcbdy0CvKPrVaXhUIFxyzVS6kf8RlIyWryvrNJI4fMi4rkhH2wzKLDdYgngUBpjoEkJRhXDRdq6/LnP1e0voKKkVNJcwI2ltsP0R9KepGLCKi7dD1DPUnrLzkWkhYOdScNWHiBtvyj1e/qHpMS0itaPks4Ftok8WVPSwsAvI2LTQvEcSOoM9Rf6fk974t4cah5gIdLnr1lR+hhpv2KNuu5NiqR3AgcBS5ISCpF+cErut302Ih5V39Z0Jc8ULwb2kzQ5InaTtAKpv/TpQz3RgFT13pmIeBDYocvXHKHOfu4jdYx5FNi+VeS0kKSFShQ5RerUcEP++SqyUtuPvaloaiMwl6QXRMTDAJIWp3xuUNvUxtpaP06m7+++p0lFc6W8A1h+PG4ZKv2DMFvKm9EvkvSjUoUEs6BEIngwsFVElLgcPZCbJb0XmJAT0r1Il65LOY7U2my9fPse0shuJ8mApC9HxBcatycAP46IHQAiYp2O45ndqrg7/7kfqMiJtL+0hBcDt0i6ir4dbUp9zWq5dN9zCHCZpF/l29sCXy0YDxHxD+Bbjdt/A0qNoq+x9eNPgKvUd5R4yeLzGgf2jAonySNzjKRtW3u5fh4Rby0bVjX+WVmCDCnB+SzpktnPgHOAAwrG87KIeHcuxCIi/iv1M4FlzjVZ0n4RcaCkeUknECX3uZ1KKiL6LY0qbutjH+paKS3Zg7g/VU1tjIgf521oG5Ou9r0zIv5YIpYepSmAhwOvJK3cTgD+HRGLDvrEsYtnWutH0u+koq0fI+Krks4CXk86Ad05yo4Sr2pgz2hykjwyS/QSZICIeDi306lRicRrai5MO5W+Pzi/LhBL77WfAD4r6aB0M+3pKujp3N2iN6HwZTQ+V8bOwAlKQwY2Bs6KiG8P8Zyx9GREHFbw9WdWiZ/7qlZKK2xDVduleyLiFkkPkLfnVLA95bvAe0gnxWsA7wNWKBjPd4C3kgvlIuIGSRsWjAfgOdKJelD+hL22gT2jxknyyDzf/GUi6aWU3d86mE7HLWeLAE8Ab2kcC6BYkixpTeBY8l5ySY8CH4yIawqF9EVSgcqykk4A1qfQaPOaqO9EqUOBH5A6JFwkafWCVdNVVXEP1ZOYMj/3Va2UDrAq+Z9StRG9S/e1kPQ20paLl5Aul7+U1LGlWI99gIi4XdKEXNR8nKSS2+KIiLtbF/mKFVtL2pvUU/9k0onwTyUdFRGHl4hnqBPR2blxgJPkkfkscEmuxIVUhbtblwFIepxBEvPeG0F0PG45v+bOXb/mMPwQ+GhE/AFA0gakfcFF9ktGxHmSriVNARSpU8KDvfvV8US5ihzSuv0wsHI+HqQOJSW8mpR4vpG++21LxbMJ0J4ct1nvWImfe+pbKa1qVVLSJOBTpCR0WmFlwU4AB5B+/5wfEa+VtDGwfaFYep7ILRavl3Qwqd3hggXjqaX1Y88upL7N/4F0YgxcTjoZrFHnjQNGi5PkEYiIs/OKVy/B+Xgzwekoht6K6JeBf5A29ItUgV9slnuOaRnSD+36pETiElISeE/BsB7vJcgAEXFJPtEoJu/dPGOAu39Cgb7bpUXExrl917YRcVLpeBqqqOKusCfxNBUWOdW2KnkCafz8lsDuwPtJLc5KeSYiHpI0l6S5IuKCnHSVtBNpxf9jwMeBZYFtCsazO+mK1tKk4upzgT0KxiP6rmQ/R5mtVcNV6xX2ITlJHrnnSJeo5gNWlkRElGjq/dboO/HmCKURup1P3Gs4jtTQf9t8e8d8bJOuA2lcvr9K0g9IRXtBajV0YdfxzISaf/GNqdy+aw9SQlGLWqq4ax6UUVWRE/WtSr4wD8fYu9EpqeS+6UckLURqR3mCpPuBZwvGQ6Nr1H+poPCywtaPxwJXtrpb/LBcOOOXh4mMgKRdST0vlwGuJ60oX17islleGfke8HNS8rc9sEdErDfoE8c2pusj4jVDHesolgsGubvapufqeFhGbSR9nvRGeRJ923eVGk5xIWlrTjVV3K2exL14ihVd5ZPzdwGnxfSBPTdHoYEeuVbkn6StHx8njYH+fkTcPugTxy6eKyJiHUnnAIcB9wK/ioiXFYpnQdLP2FykRHBR4IQS3Ukk3cTg2wdLDVyppvVjvsK2DvAkaSy9gIsLd7cYlAoPpxkJJ8kjkH+g1wSuiIjXSFoJ+FJEvLtALFNIl4N6WxsuBfaJiLu6jqUR0/nAj0irtpAS950j4k2lYprdOEnWnf0cjohYvvNgAEkb9Xe8VAeFgXoSl0omckxXRsTazTdGSTdEwUmkeSV5JdLvxttKbpeRtCXwB9IWgsNJBc5fiojTBn1iIV0WXeUTmgFFobkEkm4grdTeRKOTRMGf+6oK4YYqIJa0SqH6iBHzdouReTIinpSEpHkj4lZJK5YIJCfDW5d47UF8kFQ0823Sm9Nl+VgxkhYjFe5Moe+KwF6FQhrKuJtgNDMiYrnSMbRs3t+bAVDqcvk+1NWTGCorcpK0BXAkaWSugOUkfTgizioRT0yfpvkoqa1h7bqc2jisJLhAklhb68dzJW0D/DrqWOmssYB4VHgleQTyfqCdSW9UbyRV4M8dEZsXiGUSqSXMFPomf0WT0trkbSlXMOOKQJFpRZJ+115Z7+/YnErS3MBHSJ1jIO0f/0FEPFMonhlW9iXdWPAy8AXAJhFRdA9pk6QlSFe13kxKSs8lFewWSeQl3Qps2dteodSL/IyIWKlQPNVcuh+OGq9mdX35XmlK6wrU0/rxcdK++udI2y5yON22NWwWEJNOQnsWBi6NiB27jGcseCV5FkhaLiLujIh35EP75zerRUk9b0v4DekS3vkU7N/YVOmbwXwR8YmCrw+ApPmABYAllCY19gr0FiH1K7XkCGBu4Pv59k752K5dBtF4M3hZZd0kqupJnF+7tiKn+1v7j++gbOHlqXhq40h1vbpXVevHXlerClRbQDxavJI8CyRdExGvq2nFr1RB3GBq28eVY/o48G/gdPomFZ3+QOdm8PuQEuK/Mz1Jfgw4OiK+22U8tepvL2uJ/a2SFgVeQGVvBkqDTWYQBQdW1HZyLOkI0oCMX5ASm22B28gnN9HxBNDenu0uX3Mkaiy66np1O1+NWLV068cmSe8kFe4F8IeIOLVwPFUVEI8WJ8mzQNJ1pNWAXUn7bfsosYoj6SvAZRFxZtevPZAa3wxyS7GvAo8wfTWiZCHYnlFoStLsQGnQyrYR8Zd8e3lSJ4DOL//mqvIbS3VpGExNPYlrOzmWdNwgd0fXW9IqvHQ/2xVdFdhucRKwZ0SUbv0IgKTvAy9nelH8u4G/RESR3s01FhCPFifJsyAX572dtBJ4ZPv+Eqs4jT1KT+cPUWCPUiumqt4Mckx/IU0q6nToy0AkbQucHRGPS/ocaXDIV0p+jmogaR/SSt8LgKOBXpeLKaQx4r8vFNcJwH61rJCo0ZM4ImroSVzlyXFNJB1IunT/F/omFEUu3de2z344uk7ca2v9KOkWYJVe0V4+gb8pIoqMEpd0O+l9taYC4lHhPcmzICJuAw7Kv0iKVEi3VbRHqamqfVzZLcATBV+/7fMR8Uul8dhvBb5J2nM7pycZy5CKv14J/Bn4F3ANcFxE3FswrhcDt0i6ir59m0vts/8O6fvmtBzHDZI2HPQZY+/QvA2kipNjSa8g/UwtFRGrSFoVeFtEfKVEPHhq45DyVoKDgCVJCz59Fn0KrGz3u62poNuAyUCvG8iywI0DP3zM3U3q1jLuOEkemVdIuhR4HDgGeC2wb0Sc23UgknqjqJeLiAMkLQu8OCKu6jqWhireDFqeI03euoC+b+ClWsD1iiy3AI6IiN9I2r9QLNWIiP+Faf1t1wDWA9YF9pD0SESsXCi04tO/2iLi7vTjP03pwt3aTo6PBj4J/AAgIm6UdCJQKkn21MahHQxsFRHFWge21Nb68YXAn/LJOqR5DZdL6p0sd33SXl0B8WhxkjwyH4yIQyW9FZhEagd3HGkFpWvfJ70hvRE4gFSc9j3SD08ptbwZNJ2aP2rxd6Ux2W8mXZ2YlzT5ypL5SR0/Fs0f95L2uhZRsuh0AFX1JM5qOzleICKuap1IlGyZtxRwq6Sil+4j4lHS6t/2raKrhSQtVHhL0T8rSpBhiD7ABXyh0OsO5G/5Y578MW44SR6Z3m/dzUmXgW9Q6zdxh9aOiNVzUSER8XB+0yypijeDpijUD3kQ2wGbAt+MiEckvZi06jVHk3QU8CrSVZorSYNovhURDxeOax3SlLRXkt4MJgD/Kbj3f3fStpSlgXtIJ+hFincaajs5fjD3Ru7t33wXcF/BeKq6dD9Q0RVpD24pU3Ox3Kn0fe/ouhNJla0fhzpZV8fDVnp1WDUVEI8WJ8kjc42kc4HlgP0kLUy5vpfP5NWA3hvBpIKx9FT1ZgDTxhzPUK1aqrtFRDwB/FrSkpIm58O3loilMpOBeYH/I7XIu4fUkaS07wLvAX5J2gbyPlJxahEV9iSG+k6O9wCOAlaS9HdSEWixz1ltCQ51Tm1chFQ78pbGsQA6TZKpe0vKYDqbkgh9C4iBKgqIR4u7W4xArih9DXBHXgV8IbB0RHS+gV7SDqQ2MK8DfgS8C/hcRPyy61iGq8CbAflr1DMfqWfq4hFR5PKVpLcBh5D6Jd9PSg5vLVWlXJN8VeZVpP3I6wGrkAr4Lo+IIidgkqZGxBrN6n9Jl0XEeoXiqaoncY5po/6Od71VRVJ7aND8pK1M/8nxVLlfskB7s+qmNtak5taPAynQR/pKUs5xWu97V9LNs9PnbCBeSZ4FklaKiFtJCTKk6uCCEUFEnCDpGqA33OTtle3p6k+nZ7sA/ayWfEfSJZTb43UAsA5wfkS8VtLGwPaFYqlKbm90s6RHSHsnHwW2BNai3FWKJ/I2puslHUy6bL9goVigzulttRQ59Tr+rEiqzfgNaYvcTsDFHccyM7peuaqu6ErSMqRtTeuTPh+XkEab39N1LBHxvKQbJE2upfVjjSosIB4VTpJnzSeA3UgrgG0lq7gXIO2RDNKqSe06v4whqXl2PRfpknnJ9nnPRMRDkuaSNFdEXJATijmapL1Iq8frA8+Q9v9dDhxLwcI9UoI1F/Ax4OOk1kvbFIznyYg4rODr96eKIqfGPslzgdUj4vF8e3/SdhlLaiy6Oo601WHbfHvHfGyTQvHU1vpxKF2v2tVYQDwqvN1iBCTNFxFPDnWso1i+QPqFcjLpB+TtwC8L9gIdUteXhPJrXsD05PxZ4C5S0dyfu4yjEc/5pK/VgcASpC0Xa5a6fF8LSd8iFetdGhEli6xmkN8EViJ9H91WsouDKhrY0yxyAm5v3LUw6eu4Y9cx5bhuBVaLiKfy7XmBGyJipRLxDKXr7RaN162m6ErS9RHxmqGOdRhPFVuIelTZlERJS5AKiN9Myj/OJa3817TPfZY4SR6B/pK8Eolfft0/Aa/tJeiS5geujYhXdh3LcJV4M5A0H2nlbwrTr6RERHy5yzga8SwIPEn6xbIDqc3ZCePhl8t4JGkL0pTNv5C+ZsuRClSKDBVSRdPbJC1KmpBYVZGTpM+SusicQjqxeQdwUkQcWCie2hKcGqc2nk+qremNXd4e2Dki3jTgk+YgA+QeVU9JnF15u8UskPQiUsul+SW9lumXNhYhbXko4S7SHt/eKva8pDfOoiS9FFghIs7PifvE3mVP0pt7104ldUm4lumfq2JaKze1taezGR0CbBwRtwPk1mJnkCrgS6imJ3FEPCrpceDVEfHXIZ/QkYj4qqSzgNfnQztHxHUFQxp0O0qXCXL2Heqb2vhBUieZb5NObC7Lx4qopfWjKp2SWGMB8Whxkjxr3gp8gDQ6t1nc8DjwmRIBkS613iLpPNIvlU2ASyQdBmUmykn6EGnv9uKkS7DLkFbh3pRj6vrNAGCZiNi0wOv2kZOJ/i7j9Bm/atW5v5cgZ3dQth9wVT2Jay1yyttPiozF7qk1wYH6iq7y905NCVYtrR9rbUl3KvUVEI8KJ8mzINJAiuMlbRMRJ5eOJzslf/RcWCiOpj1InQiuBIiI/5O0ZNmQuEzSqyOiZPEXEVGyWNBm3S2SzgR+QTrJ2Ra4WtI7ofthB9TXkxhmvyKnrtSa4FRTdCXpUxFxsKTD6b+ffeeLPY3Xvl3ShIh4DjhO0mUFYqh1SmKNBcSjwknyCETEyXmP4qtotDMrsb81GpPkJL0AWDYK9GtueSoinu6tUEiaSIGOFi0bAB9QGiryFNNXbr2Xy4ZjPtJksl4hzwOkKyVbUWbYQXUDe4AvlQ6gRhUnODVNbewl51MLvf5Aqmr9qPqmJB4q6YtUUEA82pwkj4CkI0l7kDcGjiE1076qUCwXki5PTQSuBx6QdFFEtBvqd+kiSZ8h7d3ehHSp8bcF44G0989slkTEzqVjaKmlJ/E0pSr+Zxe1JThR0dTGiOi9PzwRrUFYkrbt5yldqa314z7UNSXx1aTP0Rvp+z1dqh3uqHF3ixHoVZM2/lwI+HVEvGXIJ49+LNflYRS7klaRv1i62lVpUtEupNGiAs4Bjgl/09lsStIrgCOApSJiFUmrAm8r1Wqxxir3WoqcaiXpdmDtWhKcGouuauoc1Xj9mlo/VjUlMbdZXLWGAuLR5pXkkel1R3hC0ktII3OXKxTLREkvJrU6+myhGPqIiOeBo/OH2XhwNPBJ4AcAEXGjpBOBTpPkZk/i2orAqKfIqVZ3k7Zd1OJUKim6krQZsDmwdK/oPFuE1Ne+iP5aP0oq1vqR+qYkVlVAPJqcJI/MbyUtBnyDVDkdlEsIv0xaqb0kIq6WtDzwf4ViAUDSTcy4B/lR0n6zr9SykmI2ExaIiKtanQBKvHnXWgQG1FHkVLHaEpyaiq7uJb0/vA24pnH8cdI2h1Jqa/1Y25TEGguIR4WT5JG5FXguF/CtDKxOOivvXN6/9cvG7Tto7JmStF+B5vlnkVoJnZhvvyf/+RipUfxWHcdjNlIP5jfIAJD0LlIRT6dq7UmcVVXkVKHaEpxqiq4i4gbghnx1RsAr8l23RcQzXcfTUFXrx5g+cr2WKYk1FhCPCu9JHoHGXuQNgK+RzjY/ExFrFw5tBiX2c0m6NCLW7++YpJsi4tVdxmM2UvkKzVHAesDDwJ3ADqUSVUknAPvV1JM4DxD6JykB/DhpiuT3W0nGHK+WBEcVTW1sxLQR8GPSkCyRCuXeHxEXF4rnCOCl9G39eBt5a1PXrR9V2ZREDTFFcnbmJHkEGsVyBwI3RcSJKjBqeThKxCXpBmC3iLgy314LODoiVqv182TWH0ntLjHzk6rd/wPlLpVL+j2wJqmrTjU9iWsqcqpNhQlOdUVXkq4B3hsRt+XbrwB+FhGvKxTPcYPcHRHR6TRASVeSummd1nsflXRzRKzSZRyNeKorIB4t3m4xMn+X9APgzcBBkuYlvXHWqMTZ0K7Asbnrh0jbLHaVtCBpL6XZ7KI3/GVFUlL6G9L39E5AkdWtrLqexBUWOdXmO9Q1BrrGoqu5ewkyQET8WdLcpYKpsPVjFVMSKy8gHhVOkkdmO2BT4JsR8UjuLvHJwjENREM/ZHRFxNXAqyUtSrpq8Ujj7l90HY/ZrGrsATwXWD0iHs+396dRC1Agrhp7EtdW5FSdGhKchhqLrqZK+iHwk3x7B/oW8nWqttaP1DMlseoC4tHgJHkEIuIJGhO2IuI+ChTxDFORN3I1JhL23hSiwERCs1EyGWheln6a1F+2iEp7EldV5FShWhKcnhqLrj5Cmvq3F2mB52Lg+wXjqaL1Y0MVUxIrLyAeFU6Sx4mhznQj4msFYqpmIqHZKPkJcJWkU0hbmN4BHD/4U8ZUjT2Jb5F0Jn2LnK6W9E7ovsipQlUkOA01Tm18CvhW/qhBLa0fgeqmJD4v6QZJk2sqIB4tLtwbJyRdRD7TrWEjf379aiYSmo0WSasDr883L46I6wrGMjUi1mgWyUi6LCLWKxhTVUVONrgai64kbQkcQOooMZG0mhylrpBIOos0kvqXEbF6bv24S0RsViieqqYk1lpAPBq8kjx+VHWmm7UnEj5EuYmEZqMi94/tvIfsAKrrSVxjkVNNaklwKi+6+g7wTlLXqBpW8vYgtX5cSdLfya0fC8ZzKpVMScyqKyAeLU6Sx48qhhy01DSR0Gw82onUUedjpJ7Ey9IYIlRChUVOtTmVOhKcmouu7gZuLp0gt1o/nglcwPTWj9tQbjtITVMSay0gHhXebjFODDDkYMeIuKtQPHMB60TEZfn2vMB8EfFoiXjMxqvaehLXuPWrJpKurGXgVP49fWNtXxtJa5K2W1xEwdHdeRIhzNj6cSvSVqtdu4ynEdd7SbUHxack5nhqLCAeFV5JHicijaF+c+5BPFevRVXBeJ6XdAiwbr79FI0fZjMbuUp7Ete49asmh6qeMdC1Fl19Ffg3MB8FR3fX2voReDXpKtIbaUxJzLdLqLGAeFQ4SR4n8raG95H3uTXare1VLirOlbQNqVjPlyzMRl+NPYlr3PpVk9oSnBeTOpLUVHS1eGUF3lW1fiR11Vm+9FWjpoi4XdKEiHgOOE7SZaVjGg1OksePM4ErgJuoYyM/wCdIRUTPSfovhSuUzcahGnsS11bkVJvaEpwai67Ol/SWiDi3dCBZba0fa5uSWF0B8WjxnuRxor82PmY2vkk6gtQmq9mT+DZyd4IuexK3ipwA5md6kVPn+0lrJekkYM+IqCXBqU4eULEgacX2mXy46AJLZa0fLwRWBaqYkijppcA/SVtjPg4sCny/dQI/W3KSPE5I+jhpD9fp9P2hKValrLTnYwdguYg4QNKywIsjwgNFzEZBTT2Jay1yqk2FCc64LboaryRt1N/xkl0maisgHi1OkscJSXuQih0eIe8FJL1JLl8wpiNIWz/eGBGvlPQC4NyIWLNUTGY2tnKR0zaNIqeFSUMYNi0bWR1qS3AkTaWfoquI+EyJeBpxvQ3YMN+8MCJOLxlPTSQd1N+UxPaxDuOZoYAYKF1APCrmKh2AjZpPAC+PiCkRsVz+KJYgZ2tHxB7koSIR8TAFK5XNxhtJr5D0O0k359urSvpc4bBqK3KqzeYRcVHzA9i8ZED5sviEiHguIo4D3lAyHklfB/YG/pg/9s7HLNmkn2NFpv9lvQLiN0TERsDGwLcLxjNqnCSPH7cAT5QOouUZSROYXuU+iXqKCs3Gg6OB/cj7NiPiRtKqYEm9Iqf98xaMKylb5FSb2hKcPkVXeete6aKrzYFNIuLYiDgW2JTCJxI1kPQRSTeRimJvbHzcCdw41PPHUI0FxKPC3S3Gj+dIv+QuoO8+t5It4A4DTgGWlPRV4F1A6VUus/Gkup7EEfFVSWcxvchp55JFTrWoeAx0dVMbs8WAXk3NogXjqEmtUxJvkXQmfQuIr5b0Tui2gHi0OUkeP07NH9WIiBMkXQO8ibRP6e0R8afCYZmNJ1X2JM6DMYpM/6pYlQlORPw1ryRPAX5NHUVXXwOuy4s+Iu1N3q9sSOVFxKO588erI+KvpeNpmI/U3aK33/4BYHFS0W6Qvq9mSy7cszEj6VDgpN5oajMbXQOMo9+hsjdQy2ocA11b0VX+HL0L+AOpS4qAKyPiHyXiqZGkE4D9KpuSOC45SZ7NSfpFRGyX9ym1v5gREauViAtA0vuBdwOvIG27OCkippaKx2y8cE/i2VdtCY6kW4Et21MbI2KlgjFdHBEbDv3IOZOk35NOIKqYkijpFcARwFIRsYqkVYG3RcRXSsQzmrzdYva3d/7zT8AnG8cFHNx9ONNFxPHA8ZIWJ+1xO0jS5IgYFzPdzQpaOP/Z7km8E3BxqaBsWGobA11j0dV5kv4XOIm+n6OS+25rUtuUxKNJ+ccPIBUQSzoRcJJsZUVEb//hy9uXWCUVWwloeTmpyfgUUjsfMxuBiPgSTOtJvHqjJ/H+pH63Vq/aEpwai64+mGP5aOt46bamVSg5NGQA1RUQjxYnybO5RsX08pVVTCPpIOCdpL1uJwEHRMQjJWMyG2fck3g2U2GCU2PR1cqk97UNcgx/IO2bNqqcklhlAfFo8J7k2ZykRYEXUFnFNICk3YGTSWf/8/aOR4QvB5uNAkmfBbYj7fkP4B2kvf8HFg3MBlRhglMdSb8AHgNOyIe2BxaLiO3KRVWP2qYkjucCYifJNmYkfQjYC1gGuB5YB7g8It5YMi6z8UTS6kzvSXyxexLXrcIEp7qiK0k3tIvO+zs2p5I0NSLWkHRjRKyaj10WEet1HMe4LyD2xD0bS3uRior+GhEbA68lXcozs1ESEddGxKH5wwnybKCyMdA1Tm28Lq+4AyBpbQpvH6xMLVMSF84fawAfIV3VXgzYnbRlZrbnPck2lp6MiCclIWneiLhV0oqlgzIzK6hPgkPau1lyDHSNRVdrA++T1GuTNxn4U6/VaW/1dA5WxZTEOaGA2EmyjaV7JC1GmgR4nqSHgXuLRmRmVlYVCU5DjUVXmxZ+/apVOCVx3BYQe0+ydULSRsCiwNkVjDw1MysmJzgrkRLTognOeC66Gq8qnJI4bguInSSbmZl1pJYEZ04ouhqvKp2SOC4LiL3dwszMrDuHABu3Exyg61VAT22cfVU3JTEirgWuLRnDWPBKspmZWUckXRwRGzZuC7ioeazjeM4FtmkUXS0M/DIivC+4UpKOAF5K3ymJt5E7gBSakjgueSXZzMysO7WNgR63RVfjWI1TEsclrySbmZl1RNJxg9wdEfHBzoJhfBddmY2Uk2QzM7M52HgtuhqvapySOF45STYzM+uIExwbKUkXAZ8EfhARr83Hbo6IVcpGNv54LLWZmVl3ahwDbbOXBSLiqtax0lMSxyUnyWZmZt1xgmMjVeOUxHHJ3S3MzMy64wTHRmoP0pTElST9nTwlsWxI45P3JJuZmXXEY6BtVnlKYvecJJuZmY0xJzg2UpK+mP/anpK4Fakrya6lYhuvvN3CzMxs7HkMtI1IRHwJpk1JXL0xJXF/4JcFQxu3vJJsZmbWEY+BtpGSdCuwWkQ8lW/PC9wQESuVjWz88UqymZlZdzwG2kbqJ8BVkppTEo8vG9L45JVkMzOzjngMtI0GT0nshpNkMzOzDjnBMZs9OEk2MzMzM2vxxD0zMzMzsxYnyWZmZmZmLU6SzczMzMxanCSbmZmZmbU4STYzMzMza/l/FKp5Wdx2RPMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Refit RF classifier\n",
    "clf_rf = RandomForestClassifier(n_estimators=50,\n",
    "                                criterion=\"entropy\",\n",
    "                                max_features=0.4,\n",
    "                                min_samples_leaf=1,\n",
    "                                class_weight=\"balanced\",\n",
    "                                n_jobs=-1,\n",
    "                                random_state=123)\n",
    "\n",
    "\n",
    "clf_rf.fit(StandardScaler().fit_transform(X_train), y_train)\n",
    "\n",
    "# Plot features importance\n",
    "importances = clf_rf.feature_importances_\n",
    "indices = np.argsort(clf_rf.feature_importances_)[::-1]\n",
    "plt.figure(figsize=(12, 6))\n",
    "plt.bar(range(1, 18), importances[indices], align=\"center\")\n",
    "plt.xticks(range(1, 18), df.columns[df.columns != \"left\"][indices], rotation=90)\n",
    "plt.title(\"Feature Importance\", {\"fontsize\": 16});"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Descision Tree "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "dt = DecisionTreeClassifier()\n",
    "dt.fit(X_train, y_train)\n",
    "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
    "max_features=None, max_leaf_nodes=None,\n",
    "min_impurity_split=1e-07, min_samples_leaf=1,\n",
    "min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
    "presort=False, random_state=None, splitter='best')\n",
    "y_pred = dt.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9774642140320696"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_curve, auc\n",
    "false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, y_pred)\n",
    "roc_auc = auc(false_positive_rate, true_positive_rate)\n",
    "roc_auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAs40lEQVR4nO3de3xU9Z3/8dcndyCB4RIiAgIigogQJHi3glqrVuqtWrUXtd31pz9t1T66Vbvb1XX769rWtfcta7eU1Vaw9Var1LtU6xVQKqAgCAECAhEIJEAgl8/vj3OGDGGSDGGGyUzez8fjPM6Zc5l8T4bMm3P7fszdERERaS0n3Q0QEZGuSQEhIiJxKSBERCQuBYSIiMSlgBARkbjy0t2AZBowYIAPHz483c0QEckYCxYs+MTdS+Mty6qAGD58OPPnz093M0REMoaZrW5rmU4xiYhIXAoIERGJSwEhIiJxKSBERCQuBYSIiMSVsoAwsxlmtsnMFrex3MzsZ2a2wszeM7PjY5ada2bLwmW3p6qNIiLStlQeQcwEzm1n+XnAqHC4DvgVgJnlAr8Ml48FrjSzsSlsp4iIxJGy5yDc/RUzG97OKhcCD3jQ3/ibZhYxs0HAcGCFu68EMLPZ4brvp6qtIpnOHZqbg3HsEF0WHZqbYc+etofdu4NxQwM0NbU/NDZ2vE50kNQqLoZvfzv575vOB+UGA2tjXleF8+LNP7GtNzGz6wiOQDjiiCOS30qRQ6SxEerrYdeufYfNm6G6Ohg2bWqZjn29ZUu6W98+s3S3ILuVlWVfQMT7J+PtzI/L3e8H7geoqKhQ9SPpEpqagi/vdetg/fpgiE5Hx9u27RsIjY0dv68Z9O8PAwdCaSkcd1ww3b8/5OUFy1sP0e3MICcHCgqgsDAYtzXk50NubsdDXl7H6+ToVpiMlc6AqAKGxrweAqwHCtqYL5IWu3fDe+/BsmWwfTvU1u4/1NW1TH/yCWzYEJzOiZWTA4cdBocfDiNGQN++0KMHFBUF47am+/VrCYR+/YIvXZFDIZ0B8SRwU3iN4URgm7t/bGbVwCgzGwGsA64ArkpjO6UbaWyE99+H+fNh3rxgeO+94Jx8LDMoKdl/GDAAJk2CwYODIDj88JbpsjJ9uUtmSVlAmNksYAowwMyqgDuBfAB3nw7MAc4HVgA7gWvDZY1mdhPwLJALzHD3Jalqp3Rf7vDRR/DWWy2B8O67sHNnsLx3b6iogG9+EyZPhmOPhUgkCIKePXVeXbKfuWfPafuKigpXb67SlupqePvtYHjrrWC8dWuwrEcPmDgxCILocNRROn8u2c/MFrh7RbxlWdXdt0hUQ0NwRPDWWy1hsGpVsCwnB8aNg0svhRNPbDk6yNNfg8g+9CchWcEdli6F558PhrlzgwvHAMOGwQknwP/9v8F40iTo1SutzRXJCAoIyVgbN8KLL7aEwrp1wfyjjoIvfxnOPhtOOSW4c0hEDpwCQjKGO7zzDvzhD/DMM8HdRRDc+nnWWfDpTweDqs6KJIcCQro0d1i0CB5+OBg++ii4VnD66fD97weBMHGibh8VSQUFhHRJS5cGgTB7djCdkwNnngm33w4XXxw8OSwiqaWAkC7BPXhS+bHHgmB4773gOYNPfQq+8Y3gjqOBA9PdykNoxw5YvRqqqoKr7KNGdb17blevDvoKOfLIoG+OTBS9zT+TH2pxD27bKyhI+lsrICRt1q4NLjK/9FIwXh92qHLyyfCTn8BllwVPIO9n+/bgy2nTpvaHTz4JvlSLioLOh4qK9p2OHccbop0WRYeSkuDLevhwGDKk81+K7kGfHGvXQmVly7B6dct0dfW+20QiwT25J54IJ50U3I51qA+j1qwJbg97+eVgXFkZzM/Lg5EjYcwYGD1633G/foe2jfE0Nga/648+2n9YuTL4co1+rvGGsrLOBUh9ffA7W716/yF6R0Ve3v5DtJOraOdau3cHQ339/uPo9KBBLX9ASaSAkEPmk0+C75ZoKCxfHswvLQ1OH515Jpx7LsTtlNcd/vY3+NWv4NFHgz6pY+XmBocY0WHkyKDfC/d9/5Bip2trgy/i+vqWvq5jh9Y/I1ZOTtCHRrwvlKKi4BarDRv2H0end+3a9/0KC1u+pCZObHmvww9vedz7zTfhe99r6eRp1KiW0KioCH6RxcVBkPXocfD/K66q2jcQVq4M5vfrB1OmBI+Yl5QEh37LlgXnAufM2bdfkgEDgrDo2XP/323r13v2tPyPvj1mQXh3FPx79gRtrqzctyfEgoKgM6yRI4ND1IKClnBesCD4hxqrqCj4RxmJdNwzoVnwGa9eHYxjRf/NDBsWPHyTkxO0KzpE+1CPDnv2BJ919D8n8fY1Oh2JHOCHmxg9SS0ptXkz3H9/cOfRwoXBvJISOOOM4M6jM88MHlpr8+zJ9u3w4IMwfTosXhz0f/GVrwR/2NEwKCsL/kCSfQrGfd9CCdu27fu//Nhh3br9e+eD4AtjwICgjYcdtu946NCWIBg4MLH219UF/YJEA+PNN/f/Ior+3OLilsCITvfo0bJv8QpGROetXQsrVgTTffsGH9iUKTB1avsfWGNj8ERiNDCi4dHQEP+oLPZoraAgsd9BtKhF68BvPc7JaQmC2GHw4Pbvaqira/mco+NVq4L/ULRV8KK5uWVcWhqEQDTwo9ODB3fJU3HtPUmtgJCUWLo0OE30wAPArp2cddJOTvpsf84626ioSOCp5YULg6OF3/8+OB9//PFwww1w5ZVd8ym3PXuC/3FXVgbT0SAoLU3tI9ruwZf5u+8GAVZX19K1bHQ6dt7OnW33BR479OsXhMLUqTB+fNe7/iFJo4CQ1GlshI8/hjVr8DVrWfHSGpY+v5bm1WsYZmsYWbCWkt2bg3ULCvbv4rT1+I03gmB4663g8PnKK4NgqKjI7AuJIl2U+mKSg9PUFBxqL18OH34YjKNDZeXempJGUGC81CLUlx1B3+OGUjjq5OD8bY8eQZCsWxcM770Hf/lLS38YsUaPhh//GK6+Oji9ISJpoYCQfbkH57WfeAI++CAIhOidHtFViotpHDGKmiMmsfTwL/CnhcNYXHsEPY8eyqW3DOXzX+1NpDDBn1db2xIa69YFYXLGGTpaEOkCFBASWLECfve7YPjoIygooGnUaLYPPpaPR1/ECkbx952jeP2To3ljZRnbFrV8gV9wAXz71uB09QF/r5eUBLdDjhmT3P0RkYOmgOjONm+Ghx+m+X8fJOftN3Ezlg89kz8d+11+vfkSli8pgbBUk1nwn/ujj4YvnRqMjz46uKFlyJD07oaIpIYCohtxh7XL69n4m6fo8+SDHLlsDnneyBLG8SA/4CG/ik82DmFsfzj5HLh2TBACo0cHdwdG75AUke5BAZGt6upoen8Z615axsa/LqVxyTL6bFjKiIYPOYJ61jOI3/a+mSXHf5mS0yZQcRx8dXzQVbYK54gIKCC6vhUrYMmS4HbShoZgaGO6cXUVtfOXkbt8Kb23ryMXOAIYTA5rc0ewdeBolo45m8LPfYah15zFP0bUBaqItE0B0RXt2AF//CPMmAGvvpr4ZvTmQ8awjDPZUjqGogmjOfzMMYy/5CiGHV3IcN0YJCIHQAHRVURvL50xI+jjuq4OHzWKD6/9D55vPptV6wtZuTaflWvyqNmZTyN5NJBPs+Ux6Ih8jjgyj7HHF3H6p4zzTwl6dxARORgKiHTbsCHoa2jGjKB/il692H3h5TzR76v8y19OZcVvjby8oEflo46CKecE45Ejg/Hw4Snp5VdERAGRFu5BD5k//Sk8/XTwJPKpp7L2rt/ww1WXMeOPJezcCaeeCv/+PbjkEoWAiBx6CohDbe5cuPNOeOUVKCuj6dZv8ezh13LP46N59a7gVtIvfhFuvBHKy9PcVhHp1lIaEGZ2LvBTIBf4H3e/p9XyvsAMYCRQD3zV3ReHyyqBWqAJaGyrM6mM8corQTDMnQuDBrHznp9xX+0/8l8zivj44+AU0r33wrXXdo0aKyIiKQsIM8sFfgl8GqgC5pnZk+7+fsxq3wEWuvvFZjYmXP+smOVT3b1V9Y4M89prQTC8+GLQ/fNPfsIHp1/HBZf1YOVKOO88+J//CQrlqEdlEelKUvmVdAKwwt1XuvseYDZwYat1xgIvArj7UmC4mZWlsE2HzhtvwDnnwGmnwaJFcN998NFHPH3UzZw4pQc7dsDrrwcFuM4/X+EgIl1PKr+WBgNrY15XhfNi/R24BMDMTgCGAdGefRx4zswWmNl1KWxnctXVBd/4p5wSFL350Y9g5Ur8llv50S97Mm1aUCly3ryg9rKISFeVymsQ8R7Lal2d6B7gp2a2EFgEvAtEi8ee6u7rzWwg8LyZLXX3V/b7IUF4XAdwRNxixofYU08FdQ6++1247Tbo1Yv6erju6uBu1ssvh9/+NijRKyLSlaXyCKIKGBrzegiwPnYFd9/u7te6eznwFaAUWBUuWx+ONwGPE5yy2o+73+/uFe5eUVpamvSdOGBbtwbjG26AXr3YsCHoBvvBB+Huu4Nn4BQOIpIJUhkQ84BRZjbCzAqAK4AnY1cws0i4DOAfgFfcfbuZ9TKzknCdXsA5wOIUtjV5amqCcSTCO+/A5MlB8bRHHgkOKlQHR0QyRcpOMbl7o5ndBDxLcJvrDHdfYmbXh8unA8cAD5hZE/A+8LVw8zLgcQu+TfOAh9z9mVS1NalqaqCggD/+uYirrwm6vHjtNT3TICKZJ6XPQbj7HGBOq3nTY6bfIChj3Hq7lcCEVLYtVXxrDTvyI1z+BeOUU+Cxx6AsO+7LEpFuRjdXJtnGZTWs2xHh6qvhpZcUDiKSuRQQSda4uYYaItx9NxQWprs1IiKdp4BIstztQUBEIuluiYjIwVFAJFn+jhq2EaG4ON0tERE5OAqIJCvYVcPOgoi6zhCRjKevsWRyp8fuGuqLIuluiYjIQVNAJFN9PfnNe2joFUl3S0REDpoCIpnCp6gbiyNpbYaISDIoIJIpDIjmPpG0NkNEJBkUEMkUBoTpHlcRyQIKiGQKAyKnXyStzRARSQYFRBI1b6kBIL80ktZ2iIgkgwIiiXZtqAEUECKSHRQQSbQ7DIgegyJpbYeISDIoIJKoobqG3RRQUlqU7qaIiBw0BUQSRXtyjfRV2TgRyXwKiCTyLerJVUSyhwIimbZvU0CISNZQQCRRbq2OIEQkeyggkii/LgiI3r3T3RIRkYOngEiigl017MxXLQgRyQ76KkuiHvWqBSEi2UMBkSz19eQ372aPakGISJZQQCRL2FFfk2pBiEiWUEAkS7QWRO9IWpshIpIsKQ0IMzvXzJaZ2Qozuz3O8r5m9riZvWdmb5vZuES37XKitSD6RtLaDBGRZElZQJhZLvBL4DxgLHClmY1ttdp3gIXuPh74CvDTA9i2a1EtCBHJMqk8gjgBWOHuK919DzAbuLDVOmOBFwHcfSkw3MzKEty2S4nWgsjtH0lrO0REkiWVATEYWBvzuiqcF+vvwCUAZnYCMAwYkuC2hNtdZ2bzzWx+dXV1kpp+4HZvrAGgsCyStjaIiCRTKgMiXpem3ur1PUBfM1sIfB14F2hMcNtgpvv97l7h7hWlpaUH0dyDUx/Wgig6LJK2NoiIJFNeCt+7Chga83oIsD52BXffDlwLYGYGrAqHnh1t29WoFoSIZJtUHkHMA0aZ2QgzKwCuAJ6MXcHMIuEygH8AXglDo8Ntu5rGT1QLQkSyS8qOINy90cxuAp4FcoEZ7r7EzK4Pl08HjgEeMLMm4H3ga+1tm6q2JkPz1hpq1ZOriGSRVJ5iwt3nAHNazZseM/0GMCrRbbsy2xYcQZRG0t0SEZHk0JPUSZK7XbUgRCS7KCCSJG+HakGISHZJKCDM7DQzi95tVGpmI1LbrMxTsLOGHXkR8lJ60k5E5NDpMCDM7E7gNuCOcFY+8LtUNioT9ditWhAikl0SOYK4GPgcsAPA3dcDJalsVMaprye/SbUgRCS7JBIQe9zdCZ9kNrNeqW1SBlItCBHJQokExB/M7L+BiJn9I/AC8OvUNivDqBaEiGShdi+pht1fPAyMAbYDo4F/dffnD0HbMkcYELrHVUSySbsB4e5uZk+4+yRAodAW1YIQkSyUyCmmN81scspbksF8aw0AeQMiaW2HiEgyJXLX/lTgejOrJLiTyQgOLsansmGZZPfGGoqAgoGRdDdFRCRpEgmI81LeigxXvyEICNWCEJFs0uEpJndfDUSAaeEQCedJaI9qQYhIFkrkSeqbgd8DA8Phd2b29VQ3LJM0qRaEiGShRE4xfQ040d13AJjZD4A3gJ+nsmGZpHlrDdvVk6uIZJlE7mIyoCnmdRPxa0Z3W1ajrr5FJPskcgTxW+AtM3s8fH0R8JuUtSgD5dQGAXFkn3S3REQkeToMCHe/z8zmAqcRHDlc6+7vprphmSSvroYahtFHASEiWaTDgDCzk4Al7v5O+LrEzE5097dS3roMUbCzhrq8CAUF6W6JiEjyJHIN4ldAXczrHeE8CakWhIhko4QuUofdfQPg7s0kdu2ie4jWgugZSXdLRESSKpGAWGlm3zCz/HC4GViZ6oZljLCjvkbVghCRLJNIQFwPnAKsA6qAE4HrUtmojBIGhKsWhIhkmUTuYtoEXHEI2pKZogHRJ5LWZoiIJFsiXW380Mx6h6eXXjSzT8zsS4m8uZmda2bLzGyFmd0eZ3kfM/uzmf3dzJaY2bUxyyrNbJGZLTSz+Qe2W4eQakGISJZK5BTTOe6+HbiA4BTT0cA/dbSRmeUCvyToDXYscKWZjW212o3A++4+AZgC/KeZxd4sOtXdy929IoF2poVqQYhItkokIPLD8fnALHffkuB7nwCscPeV7r4HmA1c2GodB0rC0qbFwBagMcH37xL2bKoBVAtCRLJPIgHxZzNbClQAL5pZKVCfwHaDgbUxr6vCebF+ARwDrAcWATeHt9FCEB7PmdkCM2vzoriZXWdm881sfnV1dQLNSq76DTWAakGISPZJpB7E7cDJQIW7NwA72f9IIJ54Hfp5q9efARYChwPlwC/MrHe47FR3P57gFNWNZvapNtp3v7tXuHtFaWlpAs1KLtWCEJFslcgRBO6+1d2bwukd7r4hgc2qgKExr4cQHCnEuhZ4zAMrgFXAmPDnrA/Hm4DHCU5ZdTmqBSEi2SqhgOikecAoMxsRXni+Aniy1TprgLMAzKwMGE3wYF4vMysJ5/cCzgEWp7Ctnda8VV19i0h2SlmXGe7eaGY3Ac8CucAMd19iZteHy6cD/w7MNLNFBKekbnP3T8zsSODx4No1ecBD7v5Mqtp6UKK1INSTq4hkmTYDwsw+A5S4+yOt5n8R2OTuz3f05u4+B5jTat70mOn1BEcHrbdbCUzosPVdQM72ICCGR9LdEhGR5GrvFNO/AX+NM/9F4O7UNCfz5NcFAaFaECKSbdoLiJ7uvt99o+EF6l6pa1JmKdhZQ21uhCLdxCQiWaa9gCgys/1OQZlZPtAjdU3KLEWqBSEiWaq9gHgM+HV4FxGw946i6eEyUS0IEcli7QXEvwAbgdXh08zvAJVAdbhMorUgekXS2gwRkVRo8y4md28EbjezfwOOCmevcPddh6RlmSAMiGbVghCRLNTeba6XtJrlQMTMFrp7bWqblSHCgNBTciKSjdp7UG5anHn9gPFm9jV3fylFbcocYUBY30hamyEikgrtnWK6Nt58MxsG/IGg9Gi35ltrMFQLQkSy0wH3xeTuq2mpEdGtNVTXAKoFISLZ6YADwsxGA7tT0JaMo1oQIpLN2rtI/Wf2r9/QDxgEfDmVjcoUqgUhItmsvYvU97Z67cBmYHlYQrTbUy0IEclm7V2kjtdRH2Z2qpld5e43pq5ZmaF5Sw3b1VGfiGSphOpBmFk5cBVwOUHVN3W1AS21ICLpboiISPK1dw3iaIIqcFcSnFp6GDB3n3qI2tblRWtBHBFJd0tERJKvvSOIpcCrwLSwXjRmdushaVWGyNtRQw3DdAQhIlmpvdtcLwU2AC+b2a/N7CyCsqASKthZQ22OakGISHZqMyDc/XF3/wIwBpgL3AqUmdmvzGy/MqHdUVF9DbuKIphiU0SyUIcPyrn7Dnf/vbtfAAwBFgK3p7phXZ5qQYhIljugJ6ndfYu7/7e7n5mqBmUM1YIQkSx3wF1tSCgMiKaSSFqbISKSKgqIzlItCBHJcgqIzgoDIqdfJK3NEBFJlZQGhJmda2bLzGyFme13YdvM+pjZn83s72a2xMyuTXTbtAsDQrUgRCRbpSwgzCwX+CVwHjAWuNLMxrZa7UbgfXefAEwB/tPMChLcNq2itSDySyNpbYeISKqk8gjiBGCFu68Me3+dDVzYah0HSszMgGJgC9CY4LZpVb+xBlAtCBHJXqkMiMHA2pjXVeG8WL8AjgHWA4uAm929OcFtATCz68xsvpnNr66uTlbbO7Rnk2pBiEh2S2VAxHu+uHUBos8QPHh3OFAO/MLMeie4bTDT/X53r3D3itLS0s639gBFa0H0iegxahHJTqkMiCpgaMzrIQRHCrGuBR7zwAqCrsTHJLhtWjVtUVffIpLdUhkQ84BRZjbCzAoIug5/stU6a4CzAMysDBgNrExw2/RSLQgRyXIJFQzqDHdvNLObgGeBXGCGuy8xs+vD5dOBfwdmmtkigtNKt7n7JwDxtk1VWzsjWgtiSCTdLRERSY2UBQSAu88B5rSaNz1mej0Qt2fYeNt2JXl1qgUhItlNT1J3UsHOGrZbhJ49090SEZHUUEB0kmpBiEi2U0B0RrQWRI9IulsiIpIyCojOCPthaiiOpLUZIiKppIDYswfOOw+mT+943agwIJpVC0JEspgCoqCA5g+WsueZlxLfRrUgRKQb6PYB0dAAj62dTN3ceYlvFAaE9Y2kpE0iIl1Btw+I/HxYM3Ay/bZVQqKd/akWhIh0A90+IAB2j58cTMyfn9D6jZ/UAFAwMJKaBomIdAEKCKBkyiSasYRPM0VrQRSWRVLXKBGRNFNAAMeeVMIHHMOuv76d0PoNYS2I4gGqBSEi2UsBAUyYAPOYTM8l88Djlp3YR0NYCyLSV49Ri0j2UkAA/frBR30n06tuE6xd2+H6zaoFISLdgAIitGtceKF6XgLXIbYqIEQk+ykgQn0+NYE95NPwescBEa0FoYAQkWymgAgdV1HIe4xn5ysdB0RunQJCRLKfAiJUXh5cqO6xeD40N7e7bsHOGrYRobj40LRNRCQdFBChYcNgSY/JFNRvh+XL2123qL6GetWCEJEsp4AImcGOY08IXrR3oTqsBbFbtSBEJMspIGJETj6GOnrR/FY7D8xFa0H0ihySNomIpIsCIsaE43N5h+PZ/Wo7RxDRWhC9I4ekTSIi6aKAiBG9UJ3//sKgH/B4woDwPpFD1CoRkfRQQMQYOxbezZ1MXkM9LF4cf6UwIHL6RQ5Zu0RE0kEBEaOgALYd3cET1WFA5PaPHJI2iYikiwKilQEnHMkW69dmQDRtrgFUC0JEsl9KA8LMzjWzZWa2wsxuj7P8n8xsYTgsNrMmM+sXLqs0s0XhssQq+SRB+URjnlfQ8Eb8gNgd1oJQQIhItktZQJhZLvBL4DxgLHClmY2NXcfdf+Tu5e5eDtwB/NXdt8SsMjVcXpGqdrZWXg5vcwK5HyyGnTv3W74nrAVRUqpaECKS3VJ5BHECsMLdV7r7HmA2cGE7618JzEphexISrQ2R09wECxfut1y1IESku0hlQAwGYosrVIXz9mNmPYFzgUdjZjvwnJktMLPr2vohZnadmc03s/nV1dUH3ehIBDYMCS9Uv73/A3PNm9VRn4h0D6kMiHj/xW6rXNs04LVWp5dOdffjCU5R3Whmn4q3obvf7+4V7l5RWlp6cC0ODa4YxIa8wfEvVNcoIESke0hlQFQBQ2NeDwHWt7HuFbQ6veTu68PxJuBxglNWh0R5ObzROJnmt/cPCFMtCBHpJvJS+N7zgFFmNgJYRxACV7Veycz6AGcAX4qZ1wvIcffacPoc4O4UtnUfEyfCm0zm4hVPBM89xKRBXm0NNQxTQIgcAg0NDVRVVVFfX5/upmS8oqIihgwZQn5+fsLbpCwg3L3RzG4CngVygRnuvsTMrg+XTw9XvRh4zt13xGxeBjxuQX/aecBD7v5MqtraWnk5/JzwOsT8+XD22XuX5Ye1IEpKDlVrRLqvqqoqSkpKGD58OKb+9TvN3dm8eTNVVVWMGDEi4e1SeQSBu88B5rSaN73V65nAzFbzVgITUtm29gwdCiv6VMA2gusQMQFRWL+NnQURcvSIoUjK1dfXKxySwMzo378/B3ojj77m4jCDEcf3ZU3RqH0vVNfXU9BUz56ekbS1TaS7UTgkR2d+jwqINpSXw+sNk/HYgNi2DVAtCBHpHhQQbSgvhzebJmNVVfDxx8HMsKO+JtWCEOkWNm/eTHl5OeXl5Rx22GEMHjx47+s9e/a0u+38+fP5xje+ccA/891338XMePbZZ/fOq6ysZNy4cfusd9ddd3HvvffufX3vvfcyZswYxo0bx4QJE3jggQcO+Ge3ltJrEJmsvBz+m5ieXT/3ub0BgWpBiHQL/fv3Z2HYo8Jdd91FcXEx3/rWt/Yub2xsJC8v/tdoRUUFFRUH3kvQrFmzOO2005g1axaf+cxnEtpm+vTpPP/887z99tv07t2bbdu28cQTTxzwz25NAdGGMWNgSf5EmhpzyW0VENY3kta2iXR3t9wStyechJSXw09+0vmffc0119CvXz/effddjj/+eL7whS9wyy23sGvXLnr06MFvf/tbRo8ezdy5c7n33nt56qmnuOuuu1izZg0rV65kzZo13HLLLXGPLtydRx55hOeff57TTz+d+vp6ioo67vft+9//Pi+//DK9e/cGoE+fPlx99dWd38mQAqINBQVw5LierF5+LEdGr0OEAZE3IJK2dolI+n344Ye88MIL5Obmsn37dl555RXy8vJ44YUX+M53vsOjjz663zZLly7l5Zdfpra2ltGjR3PDDTfs90zCa6+9xogRIxg5ciRTpkxhzpw5XHLJJe22pba2ltraWkaOHJnUfQQFRLsmToTXP5jMiHmPY+40b6khB8gvjaS7aSLd2sEcASTDZZddRm5uLgDbtm3j6quvZvny5ZgZDW2UK/7sZz9LYWEhhYWFDBw4kI0bNzJkyJB91pk1axZXXHEFAFdccQUPPvggl1xySZt3IJkZ7p6yO710kbod5eXwav1kbMsWWLVKtSBEBIBevXrtnf7ud7/L1KlTWbx4MX/+85/bfOq7sLBw73Rubi6NjY37LG9qauLRRx/l7rvvZvjw4Xz961/nL3/5C7W1tfTv35+tW7fus/6WLVsYMGAAvXv3plevXqxcuTKJexhQQLSjvDzo+huAefPYrVoQItLKtm3bGDw46Kh65syZnX6fF154gQkTJrB27VoqKytZvXo1l156KU888QTFxcUMGjSIF198EQjC4ZlnnuG0004D4I477uDGG29k+/btAGzfvp3777//4HYMBUS7xo+HRRxHY14hzJtHY7VqQYjIvr797W9zxx13cOqpp9LU1NTp95k1axYXX3zxPvMuvfRSHnroIQAeeOABvve971FeXs6ZZ57JnXfeufe6ww033MDUqVOZPHky48aN44wzzqBnz56d36mQubfVA3fmqaio8Pnzk1uddORI+EvNyRw9roCNeYdT89I7rH9pGVOnJvXHiEgcH3zwAcccc0y6m5E14v0+zWxBW1U7dQTRgegDcyxYgG3erK6+RaTbUEB0oLwcXtg2GXbsIPLh2woIEek2FBAdKC+Ht8ML1QW7tikgRKTbUEB0YOJE+JCj2V0UPKFYQ4TwYUURkaymgOjA4MHQr38Oq/pOAmBnQYTw+RgRkaymgOiAWXiayYPTTHt6RNLaHhGRQ0UBkYDycvjL5hMA1YIQ6U4OprtvgLlz5/L666+3u86FF17IySefvM+8a665hkceeWSfecXFxXunP/zwQ84//3yOOuoojjnmGC6//HI2btx4AHuWGPXFlIDycniw4TS2Wl82Djg23c0RkUOko+6+OzJ37lyKi4s55ZRT4i6vqanhnXfeobi4mFWrViVUL7q+vp7Pfvaz3HfffUybNg2Al19+merqasrKyhJuWyIUEAkoL4dNlNHPNzNtmJ6iFkmLg+njuy2d6Pt7wYIFfPOb36Suro4BAwYwc+ZMBg0axM9+9jOmT59OXl4eY8eO5Z577mH69Onk5ubyu9/9jp///Oecfvrp+7zXo48+yrRp0ygrK2P27NnccccdHf78hx56iJNPPnlvOABMTdGTuwqIBIweDYWFsHu36RZXkW7M3fn617/On/70J0pLS3n44Yf553/+Z2bMmME999zDqlWrKCwspKamhkgkwvXXX9/uUcesWbO48847KSsr4/Of/3xCAbF48WImTZqU7F2LSwGRgPx8GDcOFixAASGSLunu4xvYvXs3ixcv5tOf/jQQ9MA6aNAgAMaPH88Xv/hFLrroIi666KIO32vjxo2sWLGC0047DTMjLy+PxYsXM27cuLjdd6eqS+/26CJ1giZODMZ9+qS3HSKSPu7Osccey8KFC1m4cCGLFi3iueeeA+Dpp5/mxhtvZMGCBUyaNGm/7rxbe/jhh9m6dSsjRoxg+PDhVFZWMnv2bID9uveOdu0NcOyxx7JgwYIU7eG+FBAJKi8PxjqCEOm+CgsLqa6u5o033gCgoaGBJUuW0NzczNq1a5k6dSo//OEPqampoa6ujpKSEmpra+O+16xZs3jmmWeorKyksrKSBQsW7A2IKVOm8PDDD++9U2rmzJl7rzNcddVVvP766zz99NN73+uZZ55h0aJFSd/flAaEmZ1rZsvMbIWZ3R5n+T+Z2cJwWGxmTWbWL5FtD7VoQOgIQqT7ysnJ4ZFHHuG2225jwoQJlJeX8/rrr9PU1MSXvvQljjvuOCZOnMitt95KJBJh2rRpPP7445SXl/Pqq6/ufZ/KykrWrFnDSSedtHfeiBEj6N27N2+99RYXXHABp59+OpMmTaK8vJzXXnuNH/zgBwD06NGDp556ip///OeMGjWKsWPHMnPmTAYOHJj0/U1Zd99mlgt8CHwaqALmAVe6+/ttrD8NuNXdzzzQbaNS0d13VEMD/Ou/ws03w2GHpeRHiEgr6u47ubpSd98nACvcfaW77wFmAxe2s/6VwKxObpty+fnwH/+hcBCR7iOVATEYWBvzuiqctx8z6wmcCzzaiW2vM7P5Zja/urr6oBstIiKBVAZEvHuy2jqfNQ14zd23HOi27n6/u1e4e0VpaWknmikiXVk2Vb1Mp878HlMZEFXA0JjXQ4D1bax7BS2nlw50WxHJUkVFRWzevFkhcZDcnc2bN1NUVHRA26XyQbl5wCgzGwGsIwiBq1qvZGZ9gDOALx3otiKS3YYMGUJVVRU6fXzwioqKGDJkyAFtk7KAcPdGM7sJeBbIBWa4+xIzuz5cPj1c9WLgOXff0dG2qWqriHRN+fn5CXVgJ6mRsttc0yGVt7mKiGSjdN3mKiIiGUwBISIicWXVKSYzqwZWt5o9APgkDc1JJu1D16B96BqyYR+g6+zHMHeP+4xAVgVEPGY2v63za5lC+9A1aB+6hmzYB8iM/dApJhERiUsBISIicXWHgLg/3Q1IAu1D16B96BqyYR8gA/Yj669BiIhI53SHIwgREekEBYSIiMSVtQHR1UqWdpaZVZrZorAsa0b0I2JmM8xsk5ktjpnXz8yeN7Pl4bhvOtvYkTb24S4zWxdTJvf8dLaxI2Y21MxeNrMPzGyJmd0czs+Yz6KdfciYz8LMiszsbTP7e7gP/xbO7/KfQ1Zeg+hsydKuyMwqgQp37woP1CTEzD4F1AEPuPu4cN4PgS3ufk8Y2H3d/bZ0trM9bezDXUCdu9+bzrYlyswGAYPc/R0zKwEWABcB15Ahn0U7+3A5GfJZmJkBvdy9zszygb8BNwOX0MU/h2w9guhyJUu7E3d/BdjSavaFwP+G0/9L8EfeZbWxDxnF3T9293fC6VrgA4LKjBnzWbSzDxnDA3Xhy/xwcDLgc8jWgEi4ZGkGcOA5M1tgZteluzEHoczdP4bgjx4YmOb2dNZNZvZeeAqqy50SaIuZDQcmAm+RoZ9Fq32ADPoszCzXzBYCm4Dn3T0jPodsDYgDKXfa1Z3q7scD5wE3hqc+JD1+BYwEyoGPgf9Ma2sSZGbFBPXeb3H37eluT2fE2YeM+izcvcndywmqY55gZuPS3KSEZGtAZE3JUndfH443AY8TnD7LRBvD88nR88qb0tyeA+buG8M/9Gbg12TAZxGe834U+L27PxbOzqjPIt4+ZOJnAeDuNcBc4Fwy4HPI1oDYW7LUzAoISpY+meY2HTAz6xVemMPMegHnAIvb36rLehK4Opy+GvhTGtvSKdE/5tDFdPHPIrw4+hvgA3e/L2ZRxnwWbe1DJn0WZlZqZpFwugdwNrCUDPgcsvIuJoDwtref0FKy9P+lt0UHzsyOJDhqgKA87EOZsB9mNguYQtCd8UbgTuAJ4A/AEcAa4DJ377IXgdvYhykEpzQcqAT+T/QccldkZqcBrwKLgOZw9ncIzuFnxGfRzj5cSYZ8FmY2nuAidC7Bf8r/4O53m1l/uvjnkLUBISIiBydbTzGJiMhBUkCIiEhcCggREYlLASEiInEpIEREJC4FhHRrZtY/pkfQDa16CC1Iwc+ba2adKlRvZheZ2dhkvJdIIvLS3QCRdHL3zQT308ftrdXM8ty9MT2t289FwFNAxvVKLJlJRxAirZjZTDO7z8xeBn5gZiPN7Jmww8RXzWxMuF6pmT1qZvPC4dQ479XDzGaHnco9DPSIWXaOmb1hZu+Y2R/D/oaiNUB+ENYQeNvMjjKzU4DPAT8Kj25Ghm9zWbjOh2Z2esp/OdKt6AhCJL6jgbPdvcnMXgSud/flZnYi8F/AmcBPgR+7+9/M7AjgWeCYVu9zA7DT3ceHT9S+A2BmA4B/CX/GDjO7DfgmcHe43XZ3P8HMvgL8xN0vMLMngafc/ZHwPQDywvXOJ3ja++xU/UKk+1FAiMT3xzAcioFTgD+GX8gAheH4bGBszPzeZlYS1i2I+hTwMwB3f8/M3gvnnwSMBV4Lty8A3ojZblbM+MfttDPaAd8CYHjCeyeSAAWESHw7wnEOUBN21dxaDnCyu+/q4L3i9WdjBHUBrkxgm/b6w9kdjpvQ37Mkma5BiLQjrD2wyswug6B3UTObEC5+Drgpuq6Zlcd5i1eAL4bLxwHjw/lvAqea2VHhsp5mdnTMdl+IGUePLGqBkoPdJ5FEKSBEOvZF4Gtm9ndgCS3la78BVIQXoN8Hro+z7a+A4vDU0reBtwHcvZqgNvSscNmbwJiY7QrN7C2C2sW3hvNmA/9kZu/GXKQWSRn15irSxZhZJVDh7p+kuy3SvekIQkRE4tIRhIiIxKUjCBERiUsBISIicSkgREQkLgWEiIjEpYAQEZG4/j/6QsN0mBcL3QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "max_depths = np.linspace(1, 32, 32, endpoint=True)\n",
    "train_results = []\n",
    "test_results = []\n",
    "for max_depth in max_depths:\n",
    "    dt = DecisionTreeClassifier(max_depth=max_depth)\n",
    "    dt.fit(X_train, y_train)\n",
    "    train_pred = dt.predict(X_train)\n",
    "    false_positive_rate, true_positive_rate, thresholds = roc_curve(y_train, train_pred)\n",
    "    roc_auc = auc(false_positive_rate, true_positive_rate)\n",
    "   # Add auc score to previous train results\n",
    "    train_results.append(roc_auc)\n",
    "    y_pred = dt.predict(X_test)\n",
    "    false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, y_pred)\n",
    "    roc_auc = auc(false_positive_rate, true_positive_rate)\n",
    "   # Add auc score to previous test results\n",
    "    test_results.append(roc_auc)\n",
    "from matplotlib.legend_handler import HandlerLine2D\n",
    "line1, = plt.plot(max_depths, train_results, 'b', label='Train AUC')\n",
    "line2, = plt.plot(max_depths, test_results, 'r', label='Test AUC')\n",
    "plt.legend(handler_map={line1: HandlerLine2D(numpoints=2)})\n",
    "plt.ylabel('AUC score')\n",
    "plt.xlabel('Tree depth')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "the tree perfectly predicts all of the train data,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      " ---Descision Tree Model---\n",
      "Decision Tree classifier AUC = 0.98\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.98      0.99      2286\n",
      "           1       0.95      0.97      0.96       714\n",
      "\n",
      "    accuracy                           0.98      3000\n",
      "   macro avg       0.97      0.98      0.97      3000\n",
      "weighted avg       0.98      0.98      0.98      3000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "dt.fit(X_train, y_train)\n",
    "print (\"\\n\\n ---Descision Tree Model---\")\n",
    "dt_roc_auc = roc_auc_score(y_test, dt.predict(X_test))\n",
    "print (\"Decision Tree classifier AUC = %2.2f\" % dt_roc_auc)\n",
    "print(classification_report(y_test, dt.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}