code/ML-outsample-vdisc-comparison-and-stacking-binary.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook is an outsample test we got for vdisc dataset. Generally, it got about 65% acc. :("
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Packages and Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.seed(1248)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow\n",
    "import keras\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pickle\n",
    "from keras.layers import Dense\n",
    "from keras.models import Sequential\n",
    "from keras.optimizers import RMSprop, Adadelta, Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('../data/vdisc_train_buffer_over_data_small.csv.gz')\n",
    "labels = data.copy()\n",
    "del labels['Unnamed: 0']\n",
    "del labels['filename']\n",
    "del labels['code']\n",
    "del labels['flaw_loc']\n",
    "labels = labels.drop_duplicates().sort_values('testcase_ID').reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = pd.read_csv(\"../data/vdisc_embedding.csv\")\n",
    "x['testcase_ID'] = x['type']\n",
    "del x['type']\n",
    "x = x.sort_values(\"testcase_ID\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "y = labels['buffer']\n",
    "x = x.drop('testcase_ID', axis='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.2, random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5746, 128)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# [Neural tutorial](https://towardsdatascience.com/neural-networks-from-scratch-easy-vs-hard-b26ddc2e89c7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    }
   ],
   "source": [
    "with open('../data/dense-model-binary','rb') as f:\n",
    "    model = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5746/5746 [==============================] - 0s 37us/step\n"
     ]
    }
   ],
   "source": [
    "nn_evaluate = model.evaluate(x_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn_score = nn_evaluate[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.505395057452003"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = model.predict_classes(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "from sklearn import metrics\n",
    "import matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_predict = model.predict(x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualisations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from matplotlib.colors import LogNorm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "cell_style": "center"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAEFCAYAAABjDmX4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3de7xUdb3/8deHOwgod7lDgBewUjPU0KRMuahhp6NpalIkXiuNTOzYT/LK8Zyyi5fCREkN8pQaxyxDEtG8ABYqiOiWi7C5gyAopsDn98f3O7j2MDN778WwZ88+7+fjMY89811rfdd3fddan70usz5j7o6IiNROo1I3QESkHCl4ioikoOApIpKCgqeISAoKniIiKSh4ioik0GCDp5m1NLP/NbMtZvY/e1HPOWb212K2rVTM7HgzW1zkOieY2f3FrDPHPP5sZufvy3kk5lW05SnWNthQmdkQM3vDzLaZ2emlbk9tlTx4mtlXzWxe7MDVcUc5rghV/zvQBejg7mekrcTdH3D3k4vQnn3KzNzM+hcax92fdveD66pNxeLuI9x9SrHrNbOhZray2PUmFGUbbMCuA25z99bu/kipG1NbJQ2eZvZd4KfATYSNrBdwBzCqCNX3Bl539x1FqKvsmVmTUrfh/6CC26DWCb2BhaVuRGruXpIXsD+wDTijwDjNCcF1VXz9FGgehw0FVgLjgHXAauDrcdiPgA+AD+M8xgATgPsTdfcBHGgSP48GlgBbgaXAOYnyZxLTfQaYC2yJfz+TGDYLuB74e6znr0DHPMuWaf/3E+0/HRgJvA5sAn6QGH8w8BywOY57G9AsDpsdl+XduLxfSdR/FbAGuC9TFqfpF+dxZPzcDVgPDM3T3quAyrhci4ETY/kE4EHgN3HYQuCoxHSHxn7ZHId9MZb3jWWN4ue7gHWJ6e4DLk/06zeT6wP4b+DtuK5GJKbrG/tjK/AEcHtyvSfG2w/YDuyKfbYt9kF1y9MN+EPsq6XAt/P0V65tcHTcNm4FNgI3xHG/ASyKy/M40DtRz0nAa4Tt7TbgqURfTKDwNr0/cDdhe6kEbgAa17Af2wP3EPa7t4FHYvkC4LTEeE2BDcARefrhAqCCsK1NB7rF8jdj32+P/dO8VLEodQwr2YxhOLAjs6LzjHMd8DzQGegEPAtcH4cNjdNfF1fgSOA9oF2eDSvvhhZ3pHeAg+OwrsCg5EaW2KDeBs6L050dP3dI7ORvAgcBLePniXmWLdP+/xfbfwFhh/wt0AYYFDesvnH8TwHHxPn2Iexslyfqc6B/jvr/k/BPqCWJ4JnYsF8FWhF22v/O09aDgRWJDb8P0C/Rr+/H/m8M3Aw8n9ixKoAfAM2AzxMCUqaf3wI+Fd8vJvzzOjQx7IhEvyaD54ex7Y2Biwk7uMXhzxECQjPguLhe9wieiT5amVVWaHkaAS/GddYM+Fhs87A89U+g6jY3Oq6Tb8X12JJwllVB+CfTBLgGeDaO3zH217/HvrwiTl/T4Pkw8CvC9t0ZmANcWMN+/BPwO6BdnPcJsfz7wO8S8xwFvJJn+T9PCKxHErbBXwCzE8OXAV8oVQza6xhWshnDOcCaasZ5ExiZ+DwMWJbY8LeTCL6EI7hj8mxYeTe0uHFtBr4MtMxqw2g+Cp7nAXOyhj8HjI7vZwHXJIZdAvwlz7Jl2p85EmgT23N0YpwXgdPzTH858HDic67g+QHQIqssO1hMB14BXibPf3+gf+zbLwBNs4ZNAJ5IfB4IbI/vjycc9TZKDJ8KTIjv7wO+CxxICJ63ABex51HpLKoGz4pEfa3ish9IuOyzA2iVGH4/tQ+e+ZbnaOCtrPGvBu7JU3/2Njc6x/R/BsYkPjciHAT0Br5GDNxxmBHOJqoNnoTLYP8isT0T/tk/WYN+7Eo4KmyXY5m6EQJ62/j598D38yz/3cAtic+tCQG7T/y8jDIOnqW85rkR6FjNdZ9uwPLE5+WxbHcdXvV60nuEFVQr7v4u4VT3ImC1mf3JzA6pQXsybeqe+LymFu3Z6O474/vt8e/axPDtmenN7CAze9TM1pjZO4TrxB0L1A2w3t3fr2acu4DDgF+4+79yjeDuFYRgPQFYZ2bTzCy5HrKXuUVcr92AFe6+KzE82V9PEQLYZwmn2rOAE+Lr6azpknbPz93fi29bx/ltSpRBOGKurXzL0xvoZmabMy/CUXWXWtSd3Z7ewM8S9W0iBMnuxP7LjOgh4tR0eXoTjhhXJ+r+FeEINCNfP/Yk9OPb2ZW6+yrCpYcvm9kBwAjggTxtqLK/uPs2wn7fPc/4ZaWUwfM5wn/GQl9RWEXYCDJ6xbI03iX8d804MDnQ3R9395MI/3VfIwSV6tqTaVNlyjbVxp2Edg1w97aEndaqmcYLDTSz1oTryHcDE8ysfd6K3H/r7scRlt8JlwOqswroaWbJ7SzZX08Rjk6HxvfPAEMIwfOpGtSfbTXQ3syS67lngfEL9k8OK4Cl7n5A4tXG3UfWoo7sea4gnEon62zp7s8Slmd3+83MqLo8hbbpFYT9q2Oi3rbuPqiGy9k+BsdcpgDnAmcAz7l7vu2/yv5iZvsBHaib/WWfK1nwdPcthGtHt5vZ6WbWysyamtkIM7sljjYVuMbMOplZxzh+2u/gzQc+a2a9zGx/wukWAGbWxcxGxZX7L8IF7FxHPY8BB8WvVzUxs68QTuseTdmm2mhDuH63LR4VX5w1fC3hGlxt/AyY5+7fJFzj+mWukczsYDP7vJk1J1wPzNxoqc4LhCO378d1OxQ4DZgG4O5vxLrOBZ5y93ficnyZFMHT3ZcD8wj/CJqZ2bFxfvmsBTrE7aEm5gBbzeyq+B3OxmZ2mJl9urZtTfglcLWZDQIws/3NLPO1pj8Bg8zs3+KR77epGiDzbtPuvppww/LHZtbWzBqZWT8zO6G6BsVp/wzcYWbt4rr7bGKURwjXMb9DuLGWz1Tg62Z2eNx2bgJecPdl1bWhHJT0q0ru/mPCNa9rCDdLVgCXEVYOhLuD8wjX414B/hHL0sxrBuEC+MuEa4nJgNcotmMV4bTpBPYMTrj7RuBUwh3+jYSL56e6+4Y0baql7wFfJVxvuouwLEkTgCnxFO3M6iozs1GEm3aZ5fwucKSZnZNj9ObARMLF/zWEU7+rc4xXhbt/QAheI+K0dwBfc/fXEqM9Rbh8sSLx2QjrOo1zgGOJd7MJ/ZTvcsRrhB18Sey3brnGS4y/k7D+Dyfcnd4A/JpwVzsVd3+YcBQ/LV6OWUDoL+J2dQah7zcCAwinzJlpC23TEK6ZNiPcFHybcH2yaw2bdh7h+uRrhOvdlyfmu53wjYO+wEMFlu0J4Idx3NWEb3icVcP513uZO2siDZKZ/Q54zd2vLXVbisHMZhFuEv26xO34f8BB7n5uKdtRSiV/wkikmMzs0/H0tJGZDSd8labsnl6pz+K18THApFK3pZQUPKWhOZBw134b8HPgYnf/Z0lb1ICY2QWEy2t/dvfZpW5PKem0XUQkBR15ioikoOBZA9Vl34kZoWr7NaGyYmazzOybJW7DPk1/Z2YL49epsOAeM3vbzObYPkjnF+fTK24/jYtd976W3CZsL1I3Wh2mHCymsgyedZBKrFY8pNRaUup2pGVmo83smVK3o9TcfZC7z4ofjyMk5ejh7oO9SOn8zGyZmX0hMc+34vazs9B09Z3XMHVjrn+Avo9SDu5rdRI8cz2CqXRc6ak/60RvQh6Fd0vdkLqg7SeF6h5+JzwO9hDhS+wbCclLIQTeawjPrq4jPGmwv1dNUDCGkB1ndq6yOO4xhGxJm4GXSKREI0daLPKkEsvR7lOAfxKeyllBTEaR1b7zY1s2AP+RGN4SuDfO81XgSrISSGTNa3dSjjjd7YSnQ7YSnrLplxh3EDCD8GX8tcS0c9Qs/d4e6eWSZXHcUwlPnmyO/fqJQuuSkM3nfWBn7MvNeZZxFiHD0JzYp38E2ifblzX+MmLSh9ifU2J/LiI8XFCoP/P10QSqJsL4n7jsWwjb2KDEsJFx3W0lPA74vVjekfBl8sxz5E/zUQKSZYTkJ2Oy+uRH2cuYqy9jeT/gb7FsA+G57wPisPuomobt++yZCakbIVnLJkK2pQsS85xAgXR5ebbLbxMyP20A/iuxrKMpbnq80VRN3bjHOiQ8lJFM0/dSYtvK1FOTuJJvvx1MeKgm86TaT6qLb3vzqi5wNiYEtFsJQasFcFyikysIjwS2jhtSZgfOLORv4nQt85R1jytuZOy0k+LnTrGefGmxhlJg50uM8/FY7ydiZ56e1b67Yjs+SXgKJZMObSJhp2pP2EkWFJofewbPjXFFNiHsPNPisDaEJy3Gxb5sQ8yiRM3S72Wnl8suOyJucEfHdXc+ISA0r2Zdjiax4RcInpWEJCL7EZ4aub+GwXMiYUdrB/QgPBGTsz+r6aMJVA2e34jDM/945ieGrQaOj+/b8VHe0psJj0Q2ja/j+ehbJ8k2V+mT5DJW05f9Cdtx87geZwM/zdUvWdtiJnjOJjyJ1YLwJNN64POJ5c+ZLq/AdvkkYTvuRcgTmwx2xUyPt7u/arMOcwTPmsSVfPvtc8B58X1rYoa1UgXPY+PK2yPnJjATuCTx+WDCf5QmiYX8WI6NJFl2VaZjEmWPE3b6QmmxhlJN8MwxzU+BW7Pa0iMxfA5wVny/BBieGDa20PzYM3j+OjFsJOEJFwgpwf6Zp47q0u/lSi+XXXYnMeAmyhYTHjcttC5HU7PgOTHxeWCcf+Nc64OqgahKzkvgm/n6s5o+mkD+9HIHxPWQOUp5C7iQmDotMd51hKPm/jnqSLa5Sp9QNXjm7cscdZ6eXB4KBE/CP+qdQJvE8JuBexPLnzNdXoHtMrkdXwLMTCxfMdPj7e6v2q5DqgbPmsSVfPvtbMJZQs4E5MV+VXfNsyew3HP/jECudHGZPIIZudJnJct6A2dkpfg6jhA486bFqgkzO9rMnjSz9Wa2hZBuLjuFW770cVVSgbFnGrrq5Ku3JyFI5lJd+r1c6eWyy3oD47L6s2esp9C6rKnsPmlK9WnxYM/+LJRWrVAf7RaTckw0szfjM+HL4qBMe75M+Me13MyeiklCIJy6VgB/NbMlZja+Bu3P1cacfRmTzEwzs8rYrvupWR/BRyn1tibKqkt52KKa65XZ66xbnmFQvPR4NVqHedQkruTbv8YQEpG/ZmZzzezUlG2okeqC5wqgV56Vkytd3A6q5qP0HNMly1YQjjyT6bj2c/eJFE6LlavebL8lXDvq6e77E07VqkvhllElFRhh2YphBfkzH1WXfq+6vszUf2NWf7Zy96kUXpc16U/Ys08+JFx3qpIaLX7tplNi3NWE0/Vc9WQr1EdJXyWcZn6BkJijT2b2AO4+191HES6DPEK4Voi7b3X3ce7+MeCLwHfN7MQazC+7jfn68iZCf37cQ+rAc6m63RXq61WEbb5NomxvUx5mr7NC29TepMfLriffOqxuW6tJXMldsfsb7n42YZ3/J/B7C5nS9onqguccQqdNNLP9zKyFmQ2Jw6YCV5hZXwt5IW8ipOevzZHN/cBpZjYsHkm0iF9D6uGF02LVJJVYG8J/8ffNbDBhZ6upBwlpwtqZWQ/CdaFieBToamaXm1lzM2tjZkfHYcVIv3cXcFE86ra4zk6JO2OhdbkW6GFmzaqp/1wzG2ghX+Z1wO89fMXmdcIR0Clm1pRwrax5Yrpkf3YnZM7Kp1AfJbUhXO/aSAjcN2UGWEhHd46Z7e/uHxJuIOyKw041s/5x599COE2uSXq9pEJ92YZwM2RLXNYrs6bNmzrQQ2apZ4GbY52fIBxN7c13W6+M/d6TkEIuOxtX0t6kx0sqtA7XAn2sao7XpNRxxczONbNOHpJob47FtV23NVYweMYd4zTCRfC3CNc4vhIHTybcPZxNSM/1PrUMMnFjGUW4E5dJSXdlol0502J5zVKJXQJcZ2ZbCYHowVo07UeE04WlhJyI99VmufKJp2MnEfp0DfAG8Lk4eK/T77n7PMJv0txGuFtaQbgWVd26/Bvhzu0aMyuUXu8+wjXdNYQbAd+OdW8h9PevCUdJ78b6M66Ln5cSfpTt9+RPE1eoj5J+Q1hHlYS76s9nDT8PWBZPnS8ipKqDkNbtCUKAew64w92fLLDMudpYqC9/RMh1uYUQcLJTtt1M+Ce52cy+l6P6swlH0asIv0F0rYfUbmn9kZCubn5sz935RvS9SI+XVU+hdZj5/fqNZpYr7eDexJXhwEIz20bIVXuWh/R5mQdZjq9hPTWiZ9ulzpnZxYQN+4RSt6UhMzMn/PJARanb0hCV5RNGUl7MrKuZDbGQJu5gwldYHi51u0T2hp4qkLrQjPDjY5lfxZxG+C6jSNnSabuISAo6bRcRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8CwDZjbczBabWYWZjS91e6TmzGyyma0zswWlbosUl4JnPWdmjYHbgRHAQOBsMxtY2lZJLdwLDC91I6T4FDzrv8FAhbsvcfcPgGnAqBK3SWrI3WcDm0rdDik+Bc/6rzuwIvF5ZSwTkRJS8BQRSUHBs/6rBHomPveIZSJSQgqe9d9cYICZ9TWzZsBZwPQSt0nk/zwFz3rO3XcAlwGPA4uAB919YWlbJTVlZlOB54CDzWylmY0pdZukOMzdS90GEZGyoyNPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDzLhJmNLXUbJD2tv4ZHwbN8aOcrb1p/DYyCp4hICk1K3YBiaX3mvQ362/7NPnV+g13G+bd9pdRN2Oeu/69fULFue4NcfwD9O7e0NNNVt01ve3B0qnrrgo48y0TTjw0tdRNkL5z1NT2V2dAoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSJlycx6mtmTZvaqmS00s+/E8glmVmlm8+NrZGKaq82swswWm9mwRPnwWFZhZuNrMv8G8xtGIvJ/zg5gnLv/w8zaAC+a2Yw47FZ3/+/kyGY2EDgLGAR0A54ws4Pi4NuBk4CVwFwzm+7urxaauYKniJQld18NrI7vt5rZIqB7gUlGAdPc/V/AUjOrAAbHYRXuvgTAzKbFcQsGT522i0i9ZWZjzWxe4jU2z3h9gCOAF2LRZWb2splNNrN2saw7sCIx2cpYlq+8IAVPEam33H2Sux+VeE3KHsfMWgN/AC5393eAO4F+wOGEI9Mf74u26bRdRMqWmTUlBM4H3P0hAHdfmxh+F/Bo/FgJ9ExM3iOWUaA8Lx15ikhZMjMD7gYWuftPEuVdE6N9CVgQ308HzjKz5mbWFxgAzAHmAgPMrK+ZNSPcVJpe3fx15Cki5WoIcB7wipnNj2U/AM42s8MBB5YBFwK4+0Ize5BwI2gHcKm77wQws8uAx4HGwGR3X1jdzBU8RaRk+hxU7X2ZvNz9GcByDHqswDQ3AjfmKH+s0HS56LRdRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQV+SL5HuHVpx16XH0/mAlrg79zzxOnf8eRGH9W7Hzy44ltYtmrJ8/TbG/Hw2W7d/CMC40z/O1z4/gJ27nCvveYGZL63aXV8jM56eeCqrNr3HGf85s1SLJVnmvfB3Jv3sFnbt2sXJp36JM8/9RqmbJEWi4FkiO3Y6V983l5eWbqJ1iyY8PfE0/vbyKm6/cAj/cd9cnlm0lvM+15/Lv3gY1//unxzSfX/+/TN9+fR3H6Fru1b87w9P5vDvPMwudwAuGXkoiyu30KZl0xIvmWTs3LmTO39yMzfc+ks6durCFRecwzFDTqBX336lbpoUQb05bTezQ8zsKjP7eXxdZWaHlrpd+8razdt5aekmALa9v4PFlVvo2r4V/bu15ZlFISnM315exaijewNwyqd78ftnl/LBjl0sX7+NJWu2clT/jgB0a9+K4Uf2YMrM10uzMJLT64sW0K17T7p260HTpk357InDeP6ZWaVulhRJvQieZnYVMI3wnOqc+DJgak1/T6Sc9erUmk/2bc+8ig0sWrGZUz/dC4AvHdOH7h32A0KAXLnx3d3TVG56l27tWwFwy+jBXHP/i+zyum+75Ldx/To6dj5w9+eOnbqwccO6ErZIiqleBE9gDPBpd5/o7vfH10RCivwx+SZKZpn+cMmsumprUe3XvAkPjBvKVffOYev2D7nkzr9zwckH8/TEU2nTsikf7NhZcPrhR/Zg/Zb3mb90Yx21WESg/lzz3EX4QablWeVd47CcYlbpSQCtz7y37I67mjQ2Hhj3OX739BKmz3kLgNdXbWHUjeE3rPp3bcuwI3sAsGrTe/SIR6EA3dvvx6pN7zHyqJ6MPKonJx/RgxbNGtOmZVN+/a3j+eYvnq77BZIqOnTqzIZ1a3Z/3rB+LR06di5hi6SY6kvwvByYaWZv8NFvifQC+gOXlaxV+9gdFw1hceUWbvvTR78z1altC9a/8z5m8P1/+wR3z1gMwGPzVjD525/lF48upGu7VvTr2pZ5FRuY88Z6Jkz9BwDHDzyQb582SIGznjjokEFUrnyLNasq6dCpM7NnPs6V195U6mZJkdSL4Onuf4k/ATqYj354qRKYm0lW2tAce3BnvnpCfxYs38Szt3wRgAlTX6T/gW25YNghAEyf8xb3PVkBwKKVm3nouWXM+8np7NjlfPfu53ffaZf6qXGTJlx8xXh+OO5idu3axUmnjKJ33/6lbpYUiXkD2QHL8bRdgvm3faXUTZC91L9zy1xJiat12DUzCu63C244KVW9daG+3DASESkrCp4iIikoeIpIWTKznmb2pJm9amYLzew7sby9mc0wszfi33ax3OIDOBVm9rKZHZmo6/w4/htmdn5N5q/gKSLlagcwzt0HAscAl5rZQGA8MNPdBwAz42eAEYSfGx4AjAXuhBBsgWuBowk3ra/NBNxCFDxFpCy5+2p3/0d8vxVYRPi2zihgShxtCnB6fD8K+I0HzwMHxN94HwbMcPdN7v42MAMYXt38FTxFpN5KPkUYX2PzjNcHOAJ4Aeji7qvjoDVAl/i+Ox99jxxgZSzLV15Qvfiep4hILsmnCPMxs9bAH4DL3f0ds4++3eTubmb75GuMOvIUkbJlZk0JgfMBd38oFq+Np+PEv5lsLJVAz8TkPWJZvvKCFDxFpCxZOMS8G1jk7j9JDJoOZO6Ynw/8MVH+tXjX/RhgSzy9fxw42czaxRtFJ8eygnTaLiLlaghwHvCKmc2PZT8AJgIPmtkYQrKhM+Owx4CRQAXwHvB1AHffZGbXA3PjeNe5+6bqZq7gKSIlc0i/DqmndfdnCHl/czkxx/gOXJqnrsnA5NrMX6ftIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4iULTObbGbrzGxBomyCmVWa2fz4GpkYdrWZVZjZYjMbligfHssqzGx8Teat4Cki5exeYHiO8lvd/fD4egzAzAYCZwGD4jR3mFljM2sM3A6MAAYCZ8dxC9KvZ4pI2XL32WbWp4ajjwKmufu/gKVmVgEMjsMq3H0JgJlNi+O+WqgyHXmKSEN0mZm9HE/r28Wy7sCKxDgrY1m+8oIUPEWk3jKzsWY2L/EaW4PJ7gT6AYcDq4Ef74u26bRdROotd58ETKrlNGsz783sLuDR+LES6JkYtUcso0B5XjryFJEGxcy6Jj5+CcjciZ8OnGVmzc2sLzAAmAPMBQaYWV8za0a4qTS9uvnoyFNESmZwn/33anozmwoMBTqa2UrgWmComR0OOLAMuBDA3Rea2YOEG0E7gEvdfWes5zLgcaAxMNndF1Y3bwVPESlb7n52juK7C4x/I3BjjvLHgMdqM2+dtouIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iUrbMbLKZrTOzBYmy9mY2w8zeiH/bxXIzs5+bWYWZvWxmRyamOT+O/4aZnV+TeSt4ikg5uxcYnlU2Hpjp7gOAmfEzwAjCzw0PAMYCd0IItoRf3TwaGAxcmwm4hSh4ikjZcvfZwKas4lHAlPh+CnB6ovw3HjwPHBB/430YMMPdN7n728AM9gzIe1DwFJGGpou7r47v1wBd4vvuwIrEeCtjWb7yghQ8RaTeMrOxZjYv8Rpbm+nd3QHfF21rsi8qFREpBnefBEyq5WRrzayru6+Op+XrYnkl0DMxXo9YVgkMzSqfVd1MFDxFpGSOOHD/fVHtdOB8YGL8+8dE+WVmNo1wc2hLDLCPAzclbhKdDFxd3UwUPEWkbJnZVMJRY0czW0m4az4ReNDMxgDLgTPj6I8BI4EK4D3g6wDuvsnMrgfmxvGuc/fsm1B7UPAUkbLl7mfnGXRijnEduDRPPZOBybWZt24YiYikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCvqpUIt07tOKuS4+n8wEtcXfueeJ17vjzIj7euz0/u+BYWjRrzI6du7ji18/z4psbdk93ZL8O/O2GUxj906d45IXlfLx3e356wTG0bdmUnbuc/3roZf7w3LKSLZdUNe+FvzPpZ7ewa9cuTj71S5x57jdK3SQpEgXPEtmx07n6vrm8tHQTrVs04emJp/G3l1dxw7mf4ubfz2fG/EpOPqI7N5x7FCN+9BcAGplx/TlHMfOlVbvr2f7BDsbe9jRvrtnKge1a8puh7dgAAAV1SURBVMzE03jipVVsee+DUi2aRDt37uTOn9zMDbf+ko6dunDFBedwzJAT6NW3X6mbJkWg0/YSWbt5Oy8tDQ8xbHt/B4srt9C1fSvcoW3LpgDs36oZq99+b/c0F404lD++sJz177y/u6xi9Tu8uWYrAGve3s76Le/TsW3zOlwSyef1RQvo1r0nXbv1oGnTpnz2xGE8/8ysUjdLiqTeB08z+3qp27Cv9erUmk/2bc+8ig1cNWUON5x3FK/dcQY3nncU1/72RQC6tmvFFwf34q6/vpa3nk/160izJo1YsnZrXTVdCti4fh0dOx+4+3PHTl3YuGFdgSmknNT74An8KN+AZLqqD5fMqsMmFc9+zZvwwLihXHXvHLZu/5Bvnnww46fM5ZBL/ofxU+Zyx0VDALhl9GB++MCLeJ7kWl0OaMld3zqei+78e95xRKR46sU1TzN7Od8gPkpkuodkuqrWZ95bdiGjSWPjgXGf43dPL2H6nLcA+OoJ/bnynjkAPPTcMm678DMAHNGvA/d+5wQAOrRtzrAjurNjl/Po3Ldo07Ipfxj/Ba6b+g/mvrG+JMsie+rQqTMb1q3Z/XnD+rV06Ni5hC2SYqoXwZMQIIcBb2eVG/Bs3Tenbtxx0RAWV27htj+9urtszab3OH7ggTz96hqGHtaVN9e8A8Bhl/1h9zi/vOQ4/vLiCh6d+xZNGzdi6vc+x29nv8kjLyyv82WQ/A46ZBCVK99izapKOnTqzOyZj3PltTeVullSJPUleD4KtHb3+dkDzGxW3Tdn3zv24M589YT+LFi+iWdv+SIAE6a+yGW/epZbvj6YJo0a8f6HO/nWr54rWM+/faYPQw49kPZtWnDu0P4AXHj7M7yyvNqMWrKPNW7ShIuvGM8Px13Mrl27OOmUUfTu27/UzZIiMW8gF8jK8bRdgvm3faXUTZC91L9zS0sz3YxFGwrutycd2jFVvXWhHG4YiYjUOwqeIiIpKHiKiKSg4CkikoKCp4iULTNbZmavmNl8M5sXy9qb2QwzeyP+bRfLzcx+bmYVZvaymR25N/NW8BSRcvc5dz/c3Y+Kn8cDM919ADAzfgYYAQyIr7HAnXszUwVPEWloRgFT4vspwOmJ8t948DxwgJl1TTsTBU8RKWcO/NXMXjSzsbGsi7uvju/X8NEj3t2BFYlpV8ayVOrLE0YiInuIAXFsomhSzGmRcZy7V5pZZ2CGmVVJO+bubmb75AEaBU8RKZm+HfYrODyZ/CfP8Mr4d52ZPQwMBtaaWVd3Xx1PyzN5ACuBnonJe8SyVHTaLiJlycz2M7M2mffAycACYDpwfhztfOCP8f104GvxrvsxwJbE6X2t6chTRMpVF+BhM4MQy37r7n8xs7nAg2Y2BlgOnBnHfwwYCVQA7wF7lWhdwVNEypK7LwE+maN8I3BijnIHLi3W/HXaLiKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIpI2TKz4Wa22MwqzGx8Xc5bwVNEypKZNQZuB0YAA4GzzWxgXc1fwVNEytVgoMLdl7j7B8A0YFRdzVzBU0TKVXdgReLzylhWJxrM77Zve3C0lboNIlI7/Tu3LLjfmtlYYGyiaJK7T9q3raqZBhM8RaThiYEyX7CsBHomPveIZXVCp+0iUq7mAgPMrK+ZNQPOAqbX1cx15CkiZcndd5jZZcDjQGNgsrsvrKv5m7vX1bxERBoMnbaLiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIp/H+P8TqzExsl3QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.DataFrame(\n",
    "    data=metrics.confusion_matrix(y_test, np.rint(y_predict)), \n",
    "    \n",
    ")\n",
    "\n",
    "confusion_figure, confusion_axes = matplotlib.pyplot.subplots()\n",
    "confusion_figure.set_size_inches(5, 3)\n",
    "confusion_axes.set_title(\n",
    "    'Confusion matrix showing the frequency of \\n'\n",
    "    'correct and incorrect bug classification predictions.'\n",
    "    '\\n\\n'  # hack to avoid overlap with x-axis labels below\n",
    ")\n",
    "confusion_axes.xaxis.tick_top()  # move x-axis labels to top of matrix\n",
    "_ = sns.heatmap(\n",
    "    confusion_matrix,\n",
    "    annot=True,\n",
    "    fmt=\"d\",\n",
    "    cmap=sns.color_palette(\"Blues\"),\n",
    "    vmin=0,\n",
    "    ax=confusion_axes,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "tn, fp, fn, tp = metrics.confusion_matrix(y_test, np.rint(y_predict)).flatten().tolist()\n",
    "\n",
    "fpr_nn = fp/(fp+tp)\n",
    "fnr_nn = fn/(fn+tn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0394990366088632"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fpr_nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.06247191011235955"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fnr_nn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
       "                      max_features='auto', max_leaf_nodes=None,\n",
       "                      min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                      min_samples_leaf=1, min_samples_split=2,\n",
       "                      min_weight_fraction_leaf=0.0, n_estimators=100,\n",
       "                      n_jobs=None, oob_score=False, random_state=42, verbose=0,\n",
       "                      warm_start=False)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "# Instantiate model with 100 decision trees\n",
    "rf = RandomForestRegressor(n_estimators = 100, random_state = 42)\n",
    "# Train the model on training data\n",
    "rf.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.14304878361860562"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.score(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_rf = pd.DataFrame(rf.predict(x_test))\n",
    "y_pred_rf = y_pred_rf.set_index(y_test.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "bin_y_pred_rf = np.rint(y_pred_rf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_accuracy = accuracy_score(y_test, np.rint(y_pred_rf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import metrics\n",
    "import matplotlib\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAEFCAYAAABjDmX4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxV5Z3H8c8vYQuyKajsmwgK6oyoqMWFuoLVYqe11aoVq6VqW6tWQafOSB0X6nSxM9VWXKm2UmRcaOtSraJtRUArVRGUyCKJ7LLvgd/88TzBk3DvTXISchP4vl+vvMg963Oec84355x77w9zd0REpGYK8t0AEZHGSOEpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIinsseFpZkVm9gczW2NmT9RiORea2Z/rsm35YmYnmtkHdbzMMWb2WF0uM8M6njOzS3bnOhLrqrPtqatjcE9lZoPNbK6ZrTezc/PdnprKe3ia2dfN7M3YgYvjiXJCHSz6K8CBQHt3Py/tQtz9t+5+Rh20Z7cyMzezPrmmcfe/unu/+mpTXXH3Ye4+vq6Xa2ZDzKykrpebUCfH4B7sVuCX7t7K3Z/Od2NqKq/haWbXAXcDdxAOsu7AvcDwOlh8D+BDdy+rg2U1embWJN9t2AvlPAa1T+gBzMp3I1Jz97z8AG2B9cB5OaZpTgjXT+LP3UDzOG4IUAL8AFgGLAYujeN+BGwFtsV1XAaMAR5LLLsn4ECT+HoEMA9YB8wHLkwM/1tivs8BM4A18d/PJcZNAf4L+Htczp+BDlm2rbz9oxLtPxc4C/gQ+BT498T0g4CpwOo47S+BZnHca3FbNsTt/Vpi+aOBJcCj5cPiPAfFdQyMrzsDy4EhWdo7GiiN2/UBcGocPgaYCPwmjpsFHJ2Y79DYL6vjuC/G4b3isIL4+n5gWWK+R4FrEv16eXJ/AD8BVsV9NSwxX6/YH+uAl4B7kvs9Md0+wCZgR+yz9bEPqtqezsD/xb6aD1ydpb8yHYMj4rHxc2AlcFuc9pvA7Lg9LwA9Ess5HZhDON5+Cbya6Isx5D6m2wIPEo6XUuA2oLCa/bgf8DDhvFsFPB2Hvweck5iuKbACODJLP3wLKCYca5OBznH4R7HvN8X+aZ6vLEqdYXlbMQwFysp3dJZpbgXeAA4A9gdeB/4rjhsS57817sCzgI3AvlkOrKwHWjyR1gL94rhOwIDkQZY4oFYBF8f5Loiv2ydO8o+AvkBRfD02y7aVt/8/Y/u/RTghfwe0BgbEA6tXnP4o4Li43p6Ek+2axPIc6JNh+T8m/BEqIhGeiQP7faAl4aT9SZa29gMWJQ78nsBBiX7dHPu/ELgTeCNxYhUD/w40A04hBFJ5P38MHBV//4Dwx+vQxLgjE/2aDM9tse2FwJWEE9zi+KmEQGgGnBD36y7hmeijkkrDcm1PAfBW3GfNgN6xzWdmWf4YKh5zI+I++V7cj0WEu6xiwh+ZJsDNwOtx+g6xv74S+/LaOH91w/Mp4D7C8X0AMB34djX78U/A74F947pPjsNHAb9PrHM48G6W7T+FEKwDCcfg/wKvJcYvAE7LVwbVOsPytmK4EFhSxTQfAWclXp8JLEgc+JtIhC/hCu64LAdW1gMtHlyrgS8DRZXaMILPwvNiYHql8VOBEfH3KcDNiXFXAc9n2bby9pdfCbSO7Tk2Mc1bwLlZ5r8GeCrxOlN4bgVaVBpWOSwmA+8C75Dlrz/QJ/btaUDTSuPGAC8lXvcHNsXfTyRc9RYkxj8OjIm/PwpcB3QkhOddwBXselU6hYrhWZxYXsu47R0Jj33KgJaJ8Y9R8/DMtj3HAh9Xmv4m4OEsy698zI3IMP9zwGWJ1wWEi4AewDeIwR3HGeFuosrwJDwG20LieCb8sX+lGv3YiXBVuG+GbepMCPQ28fUkYFSW7X8QuCvxuhUhsHvG1wtoxOGZz2eeK4EOVTz36QwsTLxeGIftXIZXfJ60kbCDasTdNxBuda8AFpvZn8zskGq0p7xNXRKvl9SgPSvdfXv8fVP8d2li/Kby+c2sr5n90cyWmNlawnPiDjmWDbDc3TdXMc39wGHA/7r7lkwTuHsxIazHAMvMbIKZJfdD5W1uEfdrZ2CRu+9IjE/216uEADuJcKs9BTg5/vy10nxJO9fn7hvjr63i+j5NDINwxVxT2banB9DZzFaX/xCuqg+swbIrt6cH8IvE8j4lhGQXYv+VT+ghcaq7PT0IV4yLE8u+j3AFWi5bP3Yj9OOqygt1908Ijx6+bGbtgGHAb7O0ocL54u7rCed9lyzTNyr5DM+phL+MuT6i8AnhICjXPQ5LYwPhr2u5jsmR7v6Cu59O+Ks7hxAqVbWnvE2lKdtUE78itOtgd29DOGmtink810gza0V4jvwgMMbM9su6IPffufsJhO13wuOAqnwCdDOz5HGW7K9XCVenQ+LvfwMGE8Lz1Wosv7LFwH5mltzP3XJMn7N/MlgEzHf3domf1u5+Vg2WUXmdiwi30sllFrn764Tt2dl+MzMqbk+uY3oR4fzqkFhuG3cfUM3t3C+GYybjgYuA84Cp7p7t+K9wvpjZPkB76ud82e3yFp7uvobw7OgeMzvXzFqaWVMzG2Zmd8XJHgduNrP9zaxDnD7tZ/BmAieZWXcza0u43QLAzA40s+Fx524hPMDOdNXzLNA3fryqiZl9jXBb98eUbaqJ1oTnd+vjVfGVlcYvJTyDq4lfAG+6++WEZ1y/zjSRmfUzs1PMrDnheWD5Gy1VmUa4chsV9+0Q4BxgAoC7z43Lugh41d3Xxu34MinC090XAm8S/hA0M7Pj4/qyWQq0j8dDdUwH1pnZ6PgZzkIzO8zMjqlpWxN+DdxkZgMAzKytmZV/rOlPwAAz+7d45Xs1FQMy6zHt7osJb1j+1MzamFmBmR1kZidX1aA473PAvWa2b9x3JyUmeZrwHPP7hDfWsnkcuNTM/jUeO3cA09x9QVVtaAzy+lEld/8p4ZnXzYQ3SxYB3yXsHAjvDr5JeB73LvCPOCzNul4kPAB/h/AsMRl4BbEdnxBum05m13DC3VcCZxPe4V9JeHh+truvSNOmGroe+DrhedP9hG1JGgOMj7doX61qYWY2nPCmXfl2XgcMNLMLM0zeHBhLePi/hHDrd1OG6Spw962E8BoW570X+Ia7z0lM9irh8cWixGsj7Os0LgSOJ76bTeinbI8j5hBO8Hmx3zpnmi4x/XbC/v9XwrvTK4AHCO9qp+LuTxGu4ifExzHvEfqLeFydR+j7lcDBhFvm8nlzHdMQnpk2I7wpuIrwfLJTNZt2MeH55BzC8+5rEuvdRPjEQS/gyRzb9hLwH3HaxYRPeJxfzfU3eOXvrInskczs98Acd78l322pC2Y2hfAm0QN5bsd/An3d/aJ8tiOf8v4NI5G6ZGbHxNvTAjMbSvgoTaP79kpDFp+NXwaMy3db8knhKXuajoR37dcD/wNc6e5v57VFexAz+xbh8dpz7v5avtuTT7ptFxFJQVeeIiIpKDyroarqO7EiVE0/JtSomNkUM7s8z23YreXvzGxW/DgVFjxsZqvMbLrthnJ+cT3d4/FTWNfL3t2Sx4TVonSj1WPJwbrUKMOzHkqJ1YiHklrz8t2OtMxshJn9Ld/tyDd3H+DuU+LLEwhFObq6+yCvo3J+ZrbAzE5LrPPjePxszzVfQ+fVLN2Y6Q+g76aSg7tbvYRnpq9gqhxXeurPetGDUEdhQ74bUh90/KRQ1ZffCV8He5LwIfaVhOKlEIL3ZsJ3V5cRvmnQ1isWKLiMUB3ntUzD4rTHEaolrQb+SaIkGhnKYpGllFiGdn8BeJvwrZxFxGIUldp3SWzLCuCHifFFwCNxne8DN1CpgESlde0syhHnu4fw7ZB1hG/ZHJSYdgDwIuHD+EuJZeeoXvm9XcrLJYfFac8mfPNkdezXI3LtS0I1n83A9tiXq7Ns4xRChaHpsU+fAfZLtq/S9AuIRR9if46P/Tmb8OWCXP2ZrY/GULEQxhNx29cQjrEBiXFnxX23jvB1wOvj8A6ED5OXf4/8r3xWgGQBofjJZZX65EeVtzFTX8bhBwEvx2ErCN/7bhfHPUrFMmyj2LUSUmdCsZZPCdWWvpVY5xhylMvLclxeTaj8tAL478S2jqBuy+ONoGLpxl32IeFLGckyff9MHFvly6lOrmQ7bwcRvlRT/k21n1WVb7X5qSo4CwmB9nNCaLUATkh0cjHhK4Gt4oFUfgKXb+Rv4nxFWYZ1iTvurNhpp8fX+8flZCuLNYQcJ19imsPjco+InXlupfbdH9vxL4RvoZSXQxtLOKn2I5wk7+VaH7uG58q4I5sQTp4JcVxrwjctfhD7sjWxihLVK79Xubxc5WFHxgPu2LjvLiEEQvMq9uUIEgd+jvAsJRQR2YfwrZHHqhmeYwkn2r5AV8I3YjL2ZxV9NIaK4fnNOL78D8/MxLjFwInx9335rG7pnYSvRDaNPyfy2adOkm2u0CfJbayiL/sQjuPmcT++BtydqV8qHYvl4fka4ZtYLQjfZFoOnJLY/ozl8nIcl68QjuPuhDqxybCry/J4O/urJvswQ3hWJ1eynbdTgYvj762IFdbyFZ7Hx523S81N4C/AVYnX/Qh/UZokNrJ3hoMkOWx0ecckhr1AOOlzlcUaQhXhmWGeu4GfV2pL18T46cD58fd5wNDEuJG51seu4flAYtxZhG+4QCgJ9naWZVRVfi9TebnKw35FDNzEsA8IXzfNtS9HUL3wHJt43T+uvzDT/qBiEFWoeQlcnq0/q+ijMWQvL9cu7ofyq5SPgW8TS6clpruVcNXcJ8Mykm2u0CdUDM+sfZlhmecmt4cc4Un4Q70daJ0YfyfwSGL7M5bLy3FcJo/jq4C/JLavLsvj7eyvmu5DKoZndXIl23n7GuEuIWMB8rr+qeqZZzdgoWf+bwQylYsrryNYLlP5rOSwHsB5lUp8nUAIzqxlsarDzI41s1fMbLmZrSGUm6tcwi1b+bgKpcDYtQxdVbIttxshJDOpqvxepvJylYf1AH5QqT+7xeXk2pfVVblPmlJ1WTzYtT9zlVXL1Uc7xaIcY83so/id8AVxVHl7vkz4w7XQzF6NRUIg3LoWA382s3lmdmM12p+pjRn7MhaZmWBmpbFdj1G9PoLPSuqtSwyrquRhiyqeV1beZ52zjIO6K49XrX2YRXVyJdv5dRmhEPkcM5thZmenbEO1VBWei4DuWXZOpnJxZVSsR+kZ5ksOW0S48kyW49rH3ceSuyxWpuVW9jvCs6Nu7t6WcKtWVQm3chVKgRG2rS4sInvlo6rK71XVl+XLv71Sf7Z098fJvS+r05+wa59sIzx3qlAaLX7sZv/EtIsJt+uZllNZrj5K+jrhNvM0QmGOnuWrB3D3Ge4+nPAY5GnCs0LcfZ27/8DdewNfBK4zs1Orsb7KbczWl3cQ+vNwD6UDL6LicZerrz8hHPOtE8NqW/Kw8j7LdUzVpjxe5eVk24dVHWvVyZXMC3af6+4XEPb5j4FJFiql7RZVhed0QqeNNbN9zKyFmQ2O4x4HrjWzXhbqQt5BKM9fkyubx4BzzOzMeCXRIn4MqavnLotVnVJirQl/xTeb2SDCyVZdEwllwvY1s66E50J14Y9AJzO7xsyam1lrMzs2jquL8nv3A1fEq26L++wL8WTMtS+XAl3NrFkVy7/IzPpbqJd5KzDJw0dsPiRcAX3BzJoSnpU1T8yX7M8uhMpZ2eTqo6TWhOddKwnBfUf5CAvl6C40s7buvo3wBsKOOO5sM+sTT/41hNvk6pTXS8rVl60Jb4asidt6Q6V5s5YO9FBZ6nXgzrjMIwhXU7X5bOsNsd+7EUrIVa7GlVSb8nhJufbhUqCnVazxmpQ6V8zsIjPb30MR7dVxcE33bbXlDM94YpxDeAj+MeEZx9fi6IcI7x6+RijPtZkahkw8WIYT3okrL0l3Q6JdGctiefVKiV0F3Gpm6whBNLEGTfsR4XZhPqEm4qM12a5s4u3Y6YQ+XQLMBT4fR9e6/J67v0n4P2l+SXi3tJjwLKqqffky4Z3bJWaWq7zeo4RnuksIbwRcHZe9htDfDxCukjbE5Ze7Nb6eT/hP2SaRvUxcrj5K+g1hH5US3lV/o9L4i4EF8db5CkKpOghl3V4iBNxU4F53fyXHNmdqY66+/BGh1uUaQuBULtl2J+GP5Gozuz7D4i8gXEV/Qvg/iG7xUNotrWcI5epmxvY8mG1Cr0V5vErLybUPy///+pVmlqnsYG1yZSgwy8zWE2rVnu+hfF75F1lOrOZyqkXfbZd6Z2ZXEg7sk/Pdlj2ZmTnhfx4ozndb9kSN8htG0riYWSczG2yhTFw/wkdYnsp3u0RqQ98qkPrQjPCfj5X/r5gTCJ9lFGm0dNsuIpKCbttFRFJQeIqIpKDwFBFJQeEpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIikoPEVEUlB4ioikoPAUEUlB4SkikoLCU0QkBYWniEgKCk8RkRQUniIiKSg8RURSUHiKiKSg8BQRSUHhKSKSgsJTRCQFhaeISAoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIikoPBsBMxtqZh+YWbGZ3Zjv9kj1mdlDZrbMzN7Ld1ukbik8GzgzKwTuAYYB/YELzKx/flslNfAIMDTfjZC6p/Bs+AYBxe4+z923AhOA4Xluk1STu78GfJrvdkjdU3g2fF2ARYnXJXGYiOSRwlNEJAWFZ8NXCnRLvO4ah4lIHik8G74ZwMFm1svMmgHnA5Pz3CaRvZ7Cs4Fz9zLgu8ALwGxgorvPym+rpLrM7HFgKtDPzErM7LJ8t0nqhrl7vtsgItLo6MpTRCQFhaeISAoKTxGRFBSeIiIpKDxFRFJQeDYSZjYy322Q9LT/9jwKz8ZDJ1/jpv23h1F4ioiksMd8SL7VVx/ZMzYki23zptC095B8N2O3GDrsiHw3YbebN+VJeg/5t3w3Y7eZdOlASzNfVeft+okjUi23PujKs5HYU4Nzb7EnB+feSuEpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIikoPEVEUlB4ioikoPAUEUlB4SkikoLCU0QkBYWniEgKCk8RkRQUniIiKSg8RURSUHiKiKSg8BQRSUHhKSKSgsJTRCQFhaeISAoKTxFptMzsITNbZmbvVRr+PTObY2azzOyuxPCbzKzYzD4wszMTw4fGYcVmdmN11t2k7jZDRKTePQL8EvhN+QAz+zwwHPgXd99iZgfE4f2B84EBQGfgJTPrG2e7BzgdKAFmmNlkd38/14oVniLSaLn7a2bWs9LgK4Gx7r4lTrMsDh8OTIjD55tZMTAojit293kAZjYhTpszPHXbLiJ7mr7AiWY2zcxeNbNj4vAuwKLEdCVxWLbhOSk8RaTBMrORZvZm4mdkNWZrAuwHHAfcAEw0M6vrtum2XUQaLHcfB4yr4WwlwJPu7sB0M9sBdABKgW6J6brGYeQYnpXCU0TypmffKu+O03ga+DzwSnxDqBmwApgM/M7MfkZ4w+hgYDpgwMFm1osQmucDX69qJQpPEWm0zOxxYAjQwcxKgFuAh4CH4seXtgKXxKvQWWY2kfBGUBnwHXffHpfzXeAFoBB4yN1nVbVuhaeINFrufkGWURdlmf524PYMw58Fnq3JuvWGkYhICgpPEZEUFJ4iIikoPEVEUlB4ioikoPAUEUlBH1XKo3uvHMywgV1ZvmYzg65/BoDDe+zHL751PC2aFVK2fQfXPvAGb320gjZFTXng6pPo1n4fmhQav/jDLB6bUgzAf114FEMHdqXAjJff/YQbHp6ez83aK1w1uDtHdWvLms1lXPf0bABaNSvk2iG9OKB1M5at28rPpsxnw9btdG7bnO+c0IPe7Vvy+D8+YfJ7y3Yup2WzQq4c3J3u7Ypw4N6/LeTD5RvytFVSE7ryzKPfTinm3DterDDstouO4s5JM/ncqMncNvFtbrvoaABGDj2EOSWrOX7UZIaNeZ47vnEMTQsLOLbv/hzX7wCOvX4yx/zgGQYe1IET+3fMx+bsVV4p/pTbXiyuMOzcIzry7uJ1fO//3ufdxev40hEHArB+y3YemlZSITTLffPYrswsWcv3n3qf65+ZTcmazfXSfqm9BhOeZnaImY02s/+JP6PN7NB8t2t3+vvspaxav7XCMHdoU9QUgLYtm7F41cadw1u3CMP3adGUVeu3ULZjB+7QolkhzZoU0LxpAU0LC1i2ZlP9bsheaPbS9azfsr3CsGO6t2VK8UoAphSv5Jju7QBYu7mMj1ZsZPsOrzB9y6YFHHpgK/4yN8xTtsPZuLXiMqXhahC37WY2GrgAmED4rimEL+c/bmYT3H1s3hpXz0aPn87TPzyd2y8+hoICOPXm8KWH+56fzcRRp1J831dpVdSUS37+Ku4wfe5yXpu1hOJxX8MMxj0/mw9K1+R5K/ZO7Vo0YfWmMgBWbyqjXYvcp9cBrZuzdnMZ3zmhBz33K+KjlRt5eFoJW8p21EdzpZYaypXnZcAx7j7W3R+LP2MJhUovyzZTslzVtnlT6qutu9XlZ/TjxvEzOOSqJ7hx/AzuvWIwAKf9SxfeWfgpfb49kc/dMJmfXnYsrYua0vvA1vTr0pZ+V0yk77cnctJhnfjcIQfkeSsEwKsYX2hG7/Yt+fOc5dwweQ5bynbwpcMPrJe2Se01lPDcQahyUlmnOC4jdx/n7ke7+9FNew/ZTU2rX18/uQ/PTFsIwJNTF3BUnw4AXPT5PkyOw+ctXcfCZevp27kt5wzqzoy5y9mwpYwNW8p48e1SBvVVeObD6s1ltCsKV5vtipqwZnNZzulXbtzKyg1bmbsiPJp5Y8EqerVvudvbKXWjoYTnNcBfzOw5MxsXf54H/gJ8P89tq1dLPt248w2fIYd14qMlawEoWbGBIYeHvy8HtG3BwZ3bsGDZOkpWbOCEQztSWGA0KTRO6H8gH5Suzlv792ZvfryGIX3aAzCkT3tmfJz78cnqTWWs3LCNzm2aA3B4pzaUrNYbRo2FhUpN+WdmBYTb9PICf6XAjPKSUVVp9dVHGsaG1MDD3z+JE/t3pH3rFixbs4nbJ85k7idruOvSQTQpKGDztu1c+8AbzJy/ko77FnHfVSfQcd+WGPDTZ97l93+dR4EZd19+HIMPPRAHXpxZyk2/mZHvTauRocOOyHcTauyak3syoGNrWrdowppN2/j924uZ/vFqfjCkFx1aNWP5+q387JX5rN+6nXZFTfjxOYdQ1LQQd2dz2Q6ueep9Nm3bQc/9irhycHeaFBSwdN0W7vnbQjY0wjeNJl06MFWl9sNufjHnefvebafXeQX4utJgwrO2GmN4StAYw1Mq2hvDs6HctouINCoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpKCwlNEJAWFp4hICgpPEZEUGkQ9TxHZOx1yUPt8NyE1XXmKSKNlZg+Z2TIzey8x7L/NbI6ZvWNmT5lZu8S4m8ys2Mw+MLMzE8OHxmHFZnZjddat8BSRxuwRYGilYS8Ch7n7EcCHwE0AZtYfOB8YEOe518wKzawQuAcYBvQHLojT5qTwFJFGy91fAz6tNOzP7l5eifoNwn/pAzAcmODuW9x9PlBMKIM5CCh293nuvpXw3wENr2rdCk8RabCS/9VO/BlZw0V8E3gu/t4FWJQYVxKHZRuek94wEpEGy93HAePSzGtmPwTKgN/WaaMihaeI7HHMbARwNnCqf1bxvRTolpisaxxGjuFZ6bZdRPYoZjYUGAV80d03JkZNBs43s+Zm1gs4mPBfnc8ADjazXmbWjPCm0uSq1qMrTxFptMzscWAI0MHMSoBbCO+uNwdeNDOAN9z9CnefZWYTgfcJt/PfKf8/0szsu8ALQCHwkLvPqmrdCk8RabTc/YIMgx/MMf3twO0Zhj8LPFuTdeu2XUQkBYWniEgKCk8RkRQUniIiKSg8RURSUHiKiKSg8BQRSUHhKSKSgsJTRCQFhaeISAoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpLCHlMMeeiwI/LdBEnp+efeyXcTpLYuHZjvFtS7PSY8RaTxGdSzbb6bkJpu20VEUlB4ioikoPAUEUlB4SkikoLCU0QkBYWniEgKCk8RkRQUniLSaJnZtWY2y8zeM7PHzayFmfUys2lmVmxmvzezZnHa5vF1cRzfszbrVniKSKNkZl2Aq4Gj3f0woBA4H/gx8HN37wOsAi6Ls1wGrIrDfx6nS03hKSKNWROgyMyaAC2BxcApwKQ4fjxwbvx9eHxNHH+qmVnaFSs8RaRRcvdS4CfAx4TQXAO8Bax297I4WQnQJf7eBVgU5y2L07dPu36Fp4g0WGY20szeTPyMTIzbl3A12QvoDOwDDK2vtqkwiIg0WO4+DhiXZfRpwHx3Xw5gZk8Cg4F2ZtYkXl12BUrj9KVAN6Ak3ua3BVambZuuPEWksfoYOM7MWsZnl6cC7wOvAF+J01wCPBN/nxxfE8e/7O6eduUKTxFplNx9GuGNn38A7xLybBwwGrjOzIoJzzQfjLM8CLSPw68DbqzN+nXbLiKNlrvfAtxSafA8YFCGaTcD59XVunXlKSKSgsJTRCQFhaeISAoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpKCwlNEJAV9PVNE8ubIjm3z3YTUdOUpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIikoPEVEUlB4ioikoPAUEUlBH5JvIM7qvz+n9e2AAS99uII/vb+cnvsVMfL47jQtNHa4c//URRSv2MiAjq0YdepBLFu3BYBpC1cz6Z9L8rsBe5l7rxzMsIFdWb5mM4OuD/854+E99uMX3zqeFs0KKdu+g2sfeIO3PlrBif07MmHUKSxcth6AydMWMvb//snBndow/tohO5fZ84BW3DZxJvc++34+NklqSOHZAHRr14LT+nbgxj/MoWyHc/MZfXhr0VouProLT8xczNulazmyaxsuProLtzw/F4A5S9dz50sf5bnle6/fTinmvudnc/93Ttw57LaLjuLOSTN5cWYpZxzZhdsuOpphP3oegNdnL+W8Hw3KO/0AAAXzSURBVP+lwjLmLl7L50ZNBqDAjLn3fZU/TF9YfxshtaLb9gaga7sWzF2+ga3bnR0O7y9Zz7E92uFAUbNCAFo2LeTTjdvy21DZ6e+zl7Jq/dYKw9yhTVFTANq2bMbiVRurvbwhh3di3pK1LFqxoU7bKbtPg7/yNLNL3f3hfLdjd/p41WYuGNiZVs0L2Vq2gyO7tuGjFRt5eFoJN5/Rh28c0wUDfvinD3fO03f/ffjJ8ENYtXEb42eUUrJ6c/42QAAYPX46T//wdG6/+BgKCuDUm5/dOW5Q3/2ZetcXWbxqIz989E1ml6yuMO9XBvdi0t/n13eTpRYaw5Xnj7KNMLORZvammb05b8qT9dmmOlW6ZjNPv7uU/zjjYG4+ow8LPt3EDoczD+nAI9NLuGLiezwyvYSrTugOwLyVG7nyife4/pk5PDt7OaNP7Z3nLRCAy8/ox43jZ3DIVU9w4/gZ3HvFYABmzl9J/6smcfyoyfz6+dk8fsMpFeZrWljAF47qxlNvLMhDqyWtBhGeZvZOlp93gQOzzefu49z9aHc/uveQf6vHFte9l+euZPQf5vCfz81lw5btLF67mZP7tGfawnCFMnXBavp02AeATdt2sLlsBwBvl6yl0IzWzQvz1nYJvn5yH56ZFp5ZPjl1AUf16QDAuk3b2LClDIA/v11K08IC2rduvnO+M47swsz5K1m2RncPjUmDCE9CQH4DOCfDz8o8tqvetGkRnqB02Kcpx/Zox1/nrWLVxm0M6NgKgMM7tWbx2vDueruiz5629OnQEjNj3Zbt9d9oqWDJpxs5sX9HAIYc1omPlqwF4IC2RTunOeqgDhQUwMr4SQmA8wb35gndsjc6DeWZ5x+BVu4+s/IIM5tS/82pfzd8vjetWhSyfYfzwBuL2Lh1O7/++0IuPbYbhQWwbbtz3+vhqua4nvtyZr8ObHdna5lz96s68erbw98/iRP7d6R96xZ88KvzuH3iTL573+vcdekgmhQUsHnbdr5331QAvnRcDy4/ox9l251NW7cz4u5Xdy6nZfMmfP6ITlw97vV8bUqjZ2aFwJtAqbufbWa9gAlAe+At4GJ332pmzYHfAEcRLsq+5u4LUq/X3Wvd+IbgKw//Y8/YkL3Q88+9k+8mSC2tnzjC0sz34uwVOc/b0w/tUOVyzew64GigTQzPicCT7j7BzH4N/NPdf2VmVwFHuPsVZnY+8CV3/1qadkPDuW0XEakxM+sKfAF4IL424BRgUpxkPHBu/H14fE0cf2qcPhWFp4g0WMlP1MSfkZUmuRsYBeyIr9sDq929LL4uAbrE37sAiwDi+DVx+lQayjNPEZFduPs4YFymcWZ2NrDM3d8ysyH12jAUniLSeA0GvmhmZwEtgDbAL4B2ZtYkXl12BUrj9KVAN6DEzJoAbanFp3l02y4ijZK73+TuXd29J3A+8LK7Xwi8AnwlTnYJ8Ez8fXJ8TRz/stfiHXOFp4jsaUYD15lZMeGZ5oNx+INA+zj8OuDG2qxEt+0i0ui5+xRgSvx9HjAowzSbgfPqap268hQRSUHhKSKSgm7bRSRverXfJ99NSE1XniIiKSg8RURSUHiKiKSg8BQRSUHhKSKSgsJTRCQFhaeISAoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIikoPEVEUlB4ioikoPAUEUlB4SkikoLCU0QkBYWniDRaZjbUzD4ws2Izu7E+163wFJFGycwKgXuAYUB/4AIz619f61d4ikhjNQgodvd57r4VmAAMr6+VKzxFpLHqAixKvC6Jw+pFk/pa0e426dKBlu82SEqXDsx3CyRP+hxQlPO8NbORwMjEoHHuPm73tqp69pjwFJE9TwzKbGFZCnRLvO4ah9UL3baLSGM1AzjYzHqZWTPgfGByfa1cV54i0ii5e5mZfRd4ASgEHnL3WfW1fnP3+lqXiMgeQ7ftIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpKCwlNEJAWFp4hICgpPEZEU/h+nnXnkNBl99AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.DataFrame(\n",
    "    data=metrics.confusion_matrix(y_test, np.rint(y_pred_rf)), \n",
    "    \n",
    ")\n",
    "\n",
    "confusion_figure, confusion_axes = matplotlib.pyplot.subplots()\n",
    "confusion_figure.set_size_inches(5, 3)\n",
    "confusion_axes.set_title(\n",
    "    'Confusion matrix showing the frequency of \\n'\n",
    "    'correct and incorrect bug classification predictions.'\n",
    "    '\\n\\n'  # hack to avoid overlap with x-axis labels below\n",
    ")\n",
    "confusion_axes.xaxis.tick_top()  # move x-axis labels to top of matrix\n",
    "_ = sns.heatmap(\n",
    "    confusion_matrix,\n",
    "    annot=True,\n",
    "    fmt=\"d\",\n",
    "    cmap=sns.color_palette(\"Blues\"),\n",
    "    vmin=0,\n",
    "    ax=confusion_axes,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "tn, fp, fn, tp = metrics.confusion_matrix(y_test, np.rint(y_pred_rf)).flatten().tolist()\n",
    "\n",
    "fpr_rf = fp/(fp+tp)\n",
    "fnr_rf = fn/(fn+tn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# KNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='kd_tree', leaf_size=30, metric='minkowski',\n",
       "                     metric_params=None, n_jobs=None, n_neighbors=1, p=2,\n",
       "                     weights='distance')"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier \n",
    "\n",
    "oneknn = KNeighborsClassifier(n_neighbors=1, algorithm = 'kd_tree', weights = 'distance')\n",
    "oneknn.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_oneknn = pd.DataFrame(oneknn.predict(x_test))\n",
    "y_pred_oneknn = y_pred_oneknn.set_index(y_test.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "oneknn_score=oneknn.score(x_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6059867734075879"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oneknn_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAEFCAYAAABjDmX4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZwV1Zn/8c9DI5tssojsIOACxqBRXBAlGn8BR8VJ1NEkjjhE4pbExLgw4+8XdNSQZGZ0kqgRxTWOiCZRNK4xohnFgBmIIwLSAgrNKkvLvj6/P87pprq593Z39XL7tt/363VffevUdupU1dO1PtfcHRERqZlm+a6AiEghUvAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUmhyQZPM2ttZs+ZWamZPVWL6XzTzF6py7rli5mNMLOFdTzNiWb2m7qcZoZ5vGhml9bnPBLzqrPlqattsKkys+FmtsjMNpvZefmuT03lPXia2TfM7N3YgCvjjnJKHUz6fKAb0NndL0g7EXd/3N3/Tx3Up16ZmZvZwFzDuPuf3f3whqpTXXH30e7+SF1P18xGmtnyup5uQp1sg03YrcCv3L2tuz+T78rUVF6Dp5n9ELgLuIOwkfUB7gHG1MHk+wIfuvvuOphWwTOz5vmuw+dQzm1Q64S+wLx8VyI1d8/LB+gAbAYuyDFMS0JwXRE/dwEtY7+RwHLgOmANsBK4LPa7BdgJ7IrzGAdMBH6TmHY/wIHmsXsssBjYBCwBvpko/+/EeCcDs4HS+PfkRL8ZwL8Cb8XpvAJ0ybJsZfW/IVH/84CzgA+B9cA/J4YfBswENsZhfwW0iP3ejMuyJS7vPySmfyOwCnisrCyOMyDO49jY3QNYC4zMUt8bgZK4XAuBM2L5RGAa8GjsNw84LjHekbFdNsZ+58by/rGsWey+H1iTGO8x4NpEu347uT6AfwM2xHU1OjFe/9gem4A/Ancn13tiuAOBbcDe2GabYxtUtTw9gN/GtloCfC9Le2XaBsfGbeNOYB1wWxz2n4D5cXleBvompnMmsICwvf0KeCPRFhPJvU13AKYQtpcS4DagqJrt2Al4iLDfbQCeieXvA+ckhjsA+BQ4Jks7XA4UE7a16UCPWP5RbPttsX1a5isWpY5heZsxjAJ2l63oLMPcCrwDHAx0Bd4G/jX2GxnHvzWuwLOArcBBWTasrBta3JE+Aw6P/boDQ5IbWWKD2gBcEse7OHZ3TuzkHwGHAa1j96Qsy1ZW//8X6385YYf8L6AdMCRuWP3j8F8CTozz7UfY2a5NTM+BgRmm/1PCP6HWJIJnYsP+AGhD2Gn/LUtdDweWJTb8fsCARLtuj+1fBPwEeCexYxUD/wy0AE4nBKSydv4E+FL8vpDwz+vIRL9jEu2aDJ67Yt2LgCsJO7jF/jMJAaEFcEpcr/sFz0QbLa9Ulmt5mgF/jeusBXBorPNXs0x/IhW3ubFxnXw3rsfWhLOsYsI/mebAzcDbcfgusb3Oj235gzh+dYPn74H7CNv3wcAs4DvVbMc/AE8CB8V5nxbLbwCeTMxzDPC/WZb/dEJgPZawDf4SeDPRfynwlXzFoFrHsLzNGL4JrKpimI+AsxLdXwWWJjb8bSSCL+EI7sQsG1bWDS1uXBuBrwOtK9VhLPuC5yXArEr9ZwJj4/cZwM2JflcBL2VZtrL6lx0JtIv1OSExzF+B87KMfy3w+0R3puC5E2hVqaxysJgO/C/wHln++wMDY9t+BTigUr+JwB8T3YOBbfH7CMJRb7NE/yeAifH7Y8APgUMIwfNnwBXsf1Q6g4rBszgxvTZx2Q8hXPbZDbRJ9P8NNQ+e2ZbnBOCTSsNPAB7KMv3K29zYDOO/CIxLdDcjHAT0Bf6RGLhjPyOcTVQZPAmXwXaQ2J4J/+xfr0Y7diccFR6UYZl6EAJ6+9j9NHBDluWfAvws0d2WELD7xe6lFHDwzOc1z3VAlyqu+/QAPk50fxzLyqfhFa8nbSWsoBpx9y2EU90rgJVm9gczO6Ia9SmrU89E96oa1Gedu++J37fFv6sT/beVjW9mh5nZ82a2ysw+I1wn7pJj2gBr3X17FcPcDxwF/NLdd2QawN2LCcF6IrDGzKaaWXI9VF7mVnG99gCWufveRP9ke71BCGCnEk61ZwCnxc+fK42XVD4/d98av7aN81ufKINwxFxT2ZanL9DDzDaWfQhH1d1qMO3K9ekL/GdieusJQbInsf3KBvQQcaq7PH0JR4wrE9O+j3AEWiZbO/YmtOOGyhN19xWESw9fN7OOwGjg8Sx1qLC/uPtmwn7fM8vwBSWfwXMm4T9jrkcUVhA2gjJ9YlkaWwj/Xcsckuzp7i+7+5mE/7oLCEGlqvqU1akkZZ1q4l5CvQa5e3vCTmtVjOO5eppZW8J15CnARDPrlHVC7v/l7qcQlt8JlwOqsgLobWbJ7SzZXm8Qjk5Hxu//DQwnBM83qjH9ylYCncwsuZ575xg+Z/tksAxY4u4dE5927n5WDaZReZ7LCKfSyWm2dve3CctTXn8zMyouT65tehlh/+qSmG57dx9SzeXsFINjJo8A3wIuAGa6e7btv8L+YmYHAp1pmP2l3uUteLp7KeHa0d1mdp6ZtTGzA8xstJn9LA72BHCzmXU1sy5x+LTP4M0FTjWzPmbWgXC6BYCZdTOzMXHl7iBcwM501PMCcFh8vKq5mf0D4bTu+ZR1qol2hOt3m+NR8ZWV+q8mXIOrif8E3nX3bxOucf0600BmdriZnW5mLQnXA8tutFTlL4Qjtxviuh0JnANMBXD3RXFa3wLecPfP4nJ8nRTB090/Bt4l/CNoYWYnxfllsxroHLeH6pgFbDKzG+MznEVmdpSZHV/Tuib8GphgZkMAzKyDmZU91vQHYIiZfS0e+X6PigEy6zbt7isJNyz/3czam1kzMxtgZqdVVaE47ovAPWZ2UFx3pyYGeYZwHfP7hBtr2TwBXGZmQ+O2cwfwF3dfWlUdCkFeH1Vy938nXPO6mXCzZBlwDWHlQLg7+C7hetz/Av8Ty9LM61XCBfD3CNcSkwGvWazHCsJp02nsH5xw93XA2YQ7/OsIF8/PdvdP09Sphn4EfINwvel+wrIkTQQeiadoF1Y1MTMbQ7hpV7acPwSONbNvZhi8JTCJcPF/FeHUb0KG4Spw952E4DU6jnsP8I/uviAx2BuEyxfLEt1GWNdpfBM4iXg3m9BO2S5HLCDs4Itju/XINFxi+D2E9T+UcHf6U+ABwl3tVNz994Sj+Knxcsz7hPYiblcXENp+HTCIcMpcNm6ubRrCNdMWhJuCGwjXJ7tXs2qXEK5PLiBc7742Md9thCcO+gO/y7FsfwT+bxx2JeEJj4uqOf9Gr+zOmkiTZGZPAgvc/cf5rktdMLMZhJtED+S5Hv8POMzdv5XPeuRT3t8wEqlLZnZ8PD1tZmajCI/SFNzbK41ZvDY+Dpic77rkk4KnNDWHEO7abwZ+AVzp7nPyWqMmxMwuJ1xee9Hd38x3ffJJp+0iIinoyFNEJAUFz2qoKvtOzAhV08eECoqZzTCzb+e5DvWa/s7M5sXHqbDgITPbYGazrB7S+cX59InbT1FdT7u+JbcJq0XqRmvAlIN1qSCDZwOkEqsRDym1Fue7HmmZ2Vgz++981yPf3H2Iu8+InacQknL0cvdhXkfp/MxsqZl9JTHPT+L2syfXeI2dVzN1Y6Z/gF5PKQfrW4MEz0yvYCodV3pqzwbRl5BHYUu+K9IQtP2kUNXL74TXwX5HeIh9HSF5KYTAezPh3dU1hDcNOnjFBAXjCNlx3sxUFoc9kZAtaSPwNxIp0ciQFossqcQy1PvvgDmEt3KWEZNRVKrfpbEunwL/kujfGng4zvMD4HoqJZCoNK/ypBxxvLsJb4dsIrxlMyAx7BDgVcLD+KuJaeeoXvq9/dLLJcvisGcT3jzZGNv16FzrkpDNZzuwJ7blxizLOIOQYWhWbNNngU7J+lUafikx6UNsz0die84nvFyQqz2ztdFEKibCeCoueylhGxuS6HdWXHebCK8D/iiWdyE8TF72Hvmf2ZeAZCkh+cm4Sm1yS+VlzNSWsXwA8KdY9inhve+Osd9jVEzDdgP7Z0LqQUjWsp6QbenyxDwnkiNdXpbt8nuEzE+fAj9PLOtY6jY93lgqpm7cbx0SXspIpun7W2LbKptOdeJKtv12GOGlmrI31f6jqvhWm09VgbOIENDuJAStVsApiUYuJrwS2DZuSGU7cNlCPhrHa52lrGdccWfFRjszdneN08mWFmskOXa+xDBfiNM9OjbmeZXqd3+sxxcJb6GUpUObRNipOhF2kvdzzY/9g+e6uCKbE3aeqbFfO8KbFtfFtmxHzKJE9dLvVU4vV7nsmLjBnRDX3aWEgNCyinU5lsSGnyN4lhCSiBxIeGvkN9UMnpMIO9pBQC/CGzEZ27OKNppIxeD5T7F/2T+euYl+K4ER8ftB7Mtb+hPCK5EHxM8I9j11kqxzhTZJLmMVbTmQsB23jOvxTeCuTO1SaVssC55vEt7EakV4k2ktcHpi+TOmy8uxXb5O2I77EPLEJoNdXabHK2+vmqzDDMGzOnEl2347E7gkfm9LzLCWr+B5Ulx5++XcBF4Drkp0H074j9I8sZCHZthIkmU3ljVMouxlwk6fKy3WSKoInhnGuQu4s1JdeiX6zwIuit8XA6MS/cbnmh/7B88HEv3OIrzhAiEl2Jws06gq/V6m9HKVy+4lBtxE2ULC66a51uVYqhc8JyW6B8f5F2VaH1QMRBVyXgLfztaeVbTRRLKnl+sY10PZUconwHeIqdMSw91KOGoemGEayTpXaBMqBs+sbZlhmucll4ccwZPwj3oP0C7R/yfAw4nlz5guL8d2mdyOrwJeSyxfXabHK2+vmq5DKgbP6sSVbPvtm4SzhIwJyOv6U9U1z97Ax575ZwQypYsryyNYJlP6rGRZX+CCSim+TiEEzqxpsarDzE4ws9fNbK2ZlRLSzVVO4ZYtfVyFVGDsn4auKtmm25sQJDOpKv1epvRylcv6AtdVas/ecTq51mV1VW6TA6g6LR7s35650qrlaqNyMSnHJDP7KL4TvjT2KqvP1wn/uD42szdikhAIp67FwCtmttjMbqpG/TPVMWNbxiQzU82sJNbrN1SvjWBfSr1NibKqUh62quJ6ZeV11iNLP6i79HjVWodZVCeuZNu/xhESkS8ws9lmdnbKOlRLVcFzGdAny8rJlC5uNxXzUXqG8ZJlywhHnsl0XAe6+yRyp8XKNN3K/otw7ai3u3cgnKpVlcKtTIVUYIRlqwvLyJ75qKr0e1W1Zdn0b6/Unm3c/Qlyr8vqtCfs3ya7CNedKqRGi4/ddE0Mu5Jwup5pOpXlaqOkbxBOM79CSMzRr2z2AO4+293HEC6DPEO4Voi7b3L369z9UOBc4IdmdkY15le5jtna8g5Ce37BQ+rAb1Fxu8vV1isI23y7RFltUx5WXme5tqnapMerPJ1s67Cqba06cSXzhN0XufvFhHX+U+BpC5nS6kVVwXMWodEmmdmBZtbKzIbHfk8APzCz/hbyQt5BSM9fkyOb3wDnmNlX45FEq/gYUi/PnRarOqnE2hH+i283s2GEna26phHShB1kZr0I14XqwvNAdzO71sxamlk7Mzsh9quL9Hv3A1fEo26L6+zv4s6Ya12uBnqZWYsqpv8tMxtsIV/mrcDTHh6x+ZBwBPR3ZnYA4VpZy8R4yfbsSciclU2uNkpqR7jetY4QuO8o62EhHd03zayDu+8i3EDYG/udbWYD485fSjhNrk56vaRcbdmOcDOkNC7r9ZXGzZo60ENmqbeBn8RpHk04mqrNs63Xx3bvTUghVzkbV1Jt0uMl5VqHq4F+VjHHa1LquGJm3zKzrh6SaG+MxTVdt9WWM3jGHeMcwkXwTwjXOP4h9n6QcPfwTUJ6ru3UMMjEjWUM4U5cWUq66xP1ypgWy6uXSuwq4FYz20QIRNNqULVbCKcLSwg5ER+ryXJlE0/HziS06SpgEfDl2LvW6ffc/V3Cb9L8inC3tJhwLaqqdfknwp3bVWaWK73eY4RruqsINwK+F6ddSmjvBwhHSVvi9MvcGruXEH6U7Wmyp4nL1UZJjxLWUQnhrvo7lfpfAiyNp85XEFLVQUjr9kdCgJsJ3OPur+dY5kx1zNWWtxByXZYSAk7llG0/IfyT3GhmP8ow+YsJR9ErCL9B9GMPqd3SepaQrm5urM+UbAN6LdLjVZpOrnVY9vv168wsU9rB2sSVUcA8M9tMyFV7kYf0eWUvsoyo5nSqRe+2S4MzsysJG/Zp+a5LU2ZmTvjlgeJ816UpKsg3jKSwmFl3MxtuIU3c4YRHWH6f73qJ1IbeKpCG0ILw42Nlv4o5lfAso0jB0mm7iEgKOm0XEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAsAGY2yswWmlmxmd2U7/pI9ZnZg2a2xszez3ddpG4peDZyZlYE3A2MBgYDF5vZ4PzWSmrgYWBUvishdU/Bs/EbBhS7+2J33wlMBcbkuU5STe7+JrA+3/WQuqfg2fj1BJYlupfHMhHJIwVPEZEUFDwbvxKgd6K7VywTkTxS8Gz8ZgODzKy/mbUALgKm57lOIp97Cp6NnLvvBq4BXgbmA9PcfV5+ayXVZWZPADOBw81suZmNy3edpG6Yu+e7DiIiBUdHniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4FggzG5/vOkh6Wn9Nj4Jn4dDOV9i0/poYBU8RkRSazEPybS98uGksSBa7Fs/ggENH5rsa9WLU6KPzXYV6t3jG7zh05NfyXY168/Rlx1qa8arabzdPG5tqug1BR54FoqkGzs+Lphw4P68UPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFJGCZWYPmtkaM3s/UTbRzErMbG78nJXoN8HMis1soZl9NVE+KpYVm9lN1Zm3gqeIFLKHgVEZyu9096Hx8wKAmQ0GLgKGxHHuMbMiMysC7gZGA4OBi+OwOTWvowUQEWlw7v6mmfWr5uBjgKnuvgNYYmbFwLDYr9jdFwOY2dQ47Ae5JqYjTxFpiq4xs/fiaf1BsawnsCwxzPJYlq08JwVPEWm0zGy8mb2b+Iyvxmj3AgOAocBK4N/ro246bReRRsvdJwOTazjO6rLvZnY/8HzsLAF6JwbtFcvIUZ6VjjxFpEkxs+6Jzr8Hyu7ETwcuMrOWZtYfGATMAmYDg8ysv5m1INxUml7VfHTkKSIFy8yeAEYCXcxsOfBjYKSZDQUcWAp8B8Dd55nZNMKNoN3A1e6+J07nGuBloAh40N3nVTVvBU8RyZt+h1V5XyYnd784Q/GUHMPfDtyeofwF4IWazFun7SIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAp6SD6P7rlyOKOP7cXa0u0M+9GzADxy7WkM6tEBgA5tWlC6dScn3zCdPl3b8tc7z2PRis8AmL1oLd+/fyYAQ/t35r6rT6FViyJembOc6x+alZ8F+hy5angfvtS7A6Xbd/PDZ+YDcFK/jlw4tDs9O7ZiwnML+WjdVgC6tm3BXX8/mBWl2wFYtHYLk2cuqzC9G884lG7tWpZPSxo/Bc88enxGMfe9NJ/7rx5RXnbpXW+Uf7/jkuP4bOuu8u4lqzZx8g37v3J71+Uncs19bzN70Vp+N+ErnDm0J6/OrTKvgdTC68XreXHBWr47ol952ScbtvPzPy3mOyf32W/41Zt2cP30BRmndULfjmzftbe+qir1pNGctpvZEWZ2o5n9In5uNLMj812v+vTW/NVs2Lwza/+vndSfp95anHMa3Tq2pn3rFsxetBaAJ978iHOO33/nlbo1f/VmNu/YU6GspHQ7Kz7bUaPptGrejLOHHMxv/7aqLqsnDaBRBE8zuxGYChghy8ms+P2J6v6eSFMz/MhurCndxkerNpWX9T24LW/99BxemjiKk484GIAendpQsm5L+TAl67bQvVObBq+v5HZw2xb8/NwjuGX0II7sdmB5+UXHdue591ezY4+OPAtNYzltHwcMcfddyUIz+w9gHjAp00gxMep4gBZfupQDDh1Zv7VsQBcM789Tby0p7161YStHXvU06zfvYGj/zky9/nSOv+6ZPNZQqmvD1l1c8dT7bN6xh0M7t+aGMwbwg99/QLd2LenWriUPzyqha9sW+a6m1FCjOPIE9gI9MpR3j/0ycvfJ7n6cux/XlAJnUTPj3GF9+e3b+4Lnzt17Wb85nBLOXbKOJas3MbB7e1as30rPzvuOZHp2PpCV67c2eJ0lu917vfwUf/G6baz+bAc92rfisK4HMqBLG+45fwi3nXUY3du35JZRg/JcW6muxnLkeS3wmpktYt9vifQBBgLX5K1WefLlL/TgwxWlrEgEwS7tWrJ+8072utPv4LYM6N6Opas3sWHLTj7btpPjB3Vl9qK1XHzqAH79ku7YNibtWzZn887d7PVw+n5I+5as3rSDj9Zt5ZWFnwLhjvyErwzgxy8tynNtpboaRfB095fM7DDCL9mVJfgrAWaXJSttih76/qmMGHwIndu1YuG9F3D7tLk8+voizq90yg4wfPAh3HzhUHbtcfbudb5//0w2bAk3m37wwDvcd1V4VOnVuSW8Mkd32uvbtaf1Y8gh7WjXqjn3XXgUT85ZyeYduxl3Ym/at2rOhDMHsHT9Nm57pZgjD2nLRcd0Z/dex4HJM5exeWeT3aw/N8zd812HOtH2woebxoJ8Do0afXS+qyC19PRlx1qa8Y66+dWc++37t52ZaroNobFc8xQRKSgKniIiKSh4ikjBMrMHzWyNmb2fKPu5mS0ws/fM7Pdm1jHRb4KZFZvZQjP7aqJ8VCwrru6z5QqeIlLIHgZGVSp7FTjK3Y8GPgQmAJjZYMLPCg+J49xjZkVmVgTcDYwGBgMXx2FzUvAUkYLl7m8C6yuVveLuu2PnO0Cv+H0MMNXdd7j7EqCY8ITPMKDY3Re7+07C245jqpq3gqeINGX/BLwYv/dk33PkAMtjWbbynBQ8RaTRMrPxZvZu4jO+BuP+C7AbeLw+6tYoHpIXEcnE3ScDk2s6npmNBc4GzvB9D7OXAL0Tg/WKZeQoz0pHniLSpJjZKOAG4Fx3TyZ6mA5cZGYtzaw/MIiQwW02MMjM+ptZC8JNpf0T51aiI08RKVhm9gQwEuhiZsuBHxPurrcEXjUzgHfc/Qp3n2dm04APCKfzV5e9/m1m1wAvA0XAg+4+r6p5K3iKSN4cMaBzrcZ394szFE/JMfztwO0Zyl8AXqjJvHXaLiKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIpIwTKz75vZ+2Y2z8yujWWdzOxVM1sU/x4Uy83MfmFmxWb2npkdW5t5K3iKSEEys6OAy4FhwBeBs81sIHAT8Jq7DwJei90Aowk/NzwIGA/cW5v5K3iKSKE6EviLu291993AG8DXgDHAI3GYR4Dz4vcxwKMevAN0NLPuaWeu4Ckihep9YISZdTazNsBZQG+gm7uvjMOsArrF7z2BZYnxl8eyVPS77SLSaJnZeMIpdpnJ7j4ZwN3nm9lPgVeALcBcYE9yfHd3M/P6qJuCp4g0WjFQTs7RfwowBcDM7iAcTa42s+7uvjKelq+Jg5cQjkzL9IplqTSZ4Dlq9NH5roKk9NKL7+W7ClJbl9XqxnVqZnawu68xsz6E650nAv2BS4FJ8e+zcfDpwDVmNhU4AShNnN7XWJMJniJSeIb161DbSfzWzDoDu4Cr3X2jmU0CppnZOOBj4MI47AuE66LFwFbgstrMWMFTRAqWu4/IULYOOCNDuQNX19W8dbddRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFJGCZWYdzexpM1tgZvPN7CQz62Rmr5rZovj3oDismdkvzKzYzN4zs1r95KeCp4gUsv8EXnL3I4AvAvOBm4DX3H0Q8FrsBhgNDIqf8cC9tZmxgqeIFCQz6wCcCkwBcPed7r4RGAM8Egd7BDgvfh8DPOrBO0BHM+uedv4KniJSqPoDa4GHzGyOmT1gZgcC3dx9ZRxmFdAtfu8JLEuMvzyWpaLgKSKNlpmNN7N3E5/xid7NgWOBe939GGAL+07RgfLfavf6qFvz+pioiEhdcPfJwOQsvZcDy939L7H7aULwXG1m3d19ZTwtXxP7lwC9E+P3imWpKHiKSN4cc0iH1OO6+yozW2Zmh7v7QuAM4IP4uRSYFP8+G0eZDlxjZlOBE4DSxOl9jSl4ikgh+y7wuJm1ABYDlxEuR04zs3HAx8CFcdgXgLOAYmBrHDY1BU8RKVjuPhc4LkOvMzIM68DVdTVv3TASEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAU9qpRHVw3vw5d6d6B0+25++Mx8AE7q15ELh3anZ8dWTHhuIR+t2wpA25ZF/OjLhzKgSxtmFK9jyjvLy6dzaOfWXD2iHy2KjDnLP+PBvyzPOD+pO/dcOZzRx/Zibel2hv0oPIP9yLWnMahHeOi7Q5sWlG7dyck3TOdLA7rwy++cDIABdzw1l+dmf0LPzm24/+oRHNyxNe7OQ3/8kHtenJ+vRZIaUvDMo9eL1/PigrV8d0S/8rJPNmzn552TiQgAAAXVSURBVH9azHdO7lNh2F17nKn/s4I+B7Wm90GtKvS7/KQ+/Pqtj1m0div/cuYAjunZnjklnzXEInxuPT6jmPtems/9V48oL7v0rjfKv99xyXF8tnUXAB8s28CIm55jz16nW8fWvPPzc3nhr8vYvceZ8Nhs/rZkPW1bNefPk87hT++tYEFJaYMvj9ScTtvzaP7qzWzesadCWUnpdlZ8tmO/YXfs3suCNVvYuWdvhfKOrZvT5oAiFq0NR6gzitdzfN/0r7xJ9bw1fzUbNu/M2v9rJ/XnqbcWA7Bt5x727A25KVodUITHNBWrN27jb0vWA7B5+24WlpTSvVOb+q241JlGf+RpZpe5+0P5rkdj1blNC9Zt3bcTr9+6k85tWuSxRjL8yG6sKd3GR6s2lZcdN7AL9145nN5d23L5L/9cHkzL9Onali/278S7xZ82dHUlpUI48rwlW49kuqrFM37XkHUSyeqC4f156q0lFcreLf6U4697ltMmPM91f/8FWh5QVN7vwJbNefy6kdz48Cw2bdvV0NWVlBrFkaeZvZetF/sSme4nma7q/If+p15y9jV26yodaXaqdCQqDauomXHusL6cctNzGfsvLClly/bdDO7dkTmL19G8yHj8ui/z5J8XM33WJw1cW6mNRhE8CQHyq8CGSuUGvN3w1SkcG7ftZuuuPQzq2oZFa7cycmAnXpi/Nt/V+tz68hd68OGKUlas31pe1rdrW5av28KevU7vLgdyWI8OfLJ2MwD3XDGchSWl/OoPH+SrypJSYwmezwNtY4aUCsxsRsNXp2Fce1o/hhzSjnatmnPfhUfx5JyVbN6xm3En9qZ9q+ZMOHMAS9dv47ZXigG45/whtG5RRPNmxrA+HfnXl4tZXrqdB2Yu4+oRfWlR1Iw5JaXMWa477fXtoe+fyojBh9C5XSsW3nsBt0+by6OvL+L8DKfsJx1xMNed9wV27XH27nV+MOUd1m3awUmHH8w3ThvI+x+v5+2fnQvAxCf+yitzUufnlQZk7k3jbPfzetreFLz0YrarNlIoNk8ba2nGe3X+pzn32zOP7JJqug2hEG4YiYg0OgqeIiIpKHiKiKSg4CkikoKCp4gUJDNrZWazzOxvZjbPzG6J5f3N7C9mVmxmT8Yfh8PMWsbu4ti/X23mr+ApIoVqB3C6u38RGAqMMrMTgZ8Cd7r7QMKz4+Pi8OOADbH8zjhcagqeIlKQPNgcOw+IHwdOB56O5Y8A58XvY2I3sf8ZZpb6USgFTxEpWGZWZGZzgTXAq8BHwEZ33x0HWQ70jN97AssAYv9SoHPaeSt4ikijlUz+Ez/jk/3dfY+7DwV6AcOAIxqqbo3l9UwR+Rzq3/nAnP2TyX+qGG6jmb0OnAR0NLPm8eiyF1D2vmsJ0BtYbmbNgQ7AurR115GniBQkM+tqZh3j99bAmcB84HXg/DjYpcCz8fv02E3s/yevxfvpOvIUkULVHXjEzIoIB4LT3P15M/sAmGpmtwFzgClx+CnAY2ZWDKwHLqrNzBU8RaQguft7wDEZyhcTrn9WLt8OXFBX89dpu4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkiBcvMRpnZQjMrNrObGnLeCp4iUpDir2beDYwGBgMXm9nghpq/gqeIFKphQLG7L3b3ncBUYExDzVzBU0QKVU9gWaJ7eSxrEE3md9ufvuxYy3cdJKXLjs13DSRPBh7cOud+a2bjgfGJosnuPrl+a1U9TSZ4ikjTEwNltmBZAvROdPeKZQ1Cp+0iUqhmA4PMrL+ZtQAuAqY31Mx15CkiBcndd5vZNcDLQBHwoLvPa6j5m7s31LxERJoMnbaLiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIp/H/c3WfCs9bTgwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.DataFrame(\n",
    "    data=metrics.confusion_matrix(y_test, np.rint(y_pred_oneknn)), \n",
    "    \n",
    ")\n",
    "\n",
    "confusion_figure, confusion_axes = matplotlib.pyplot.subplots()\n",
    "confusion_figure.set_size_inches(5, 3)\n",
    "confusion_axes.set_title(\n",
    "    'Confusion matrix showing the frequency of \\n'\n",
    "    'correct and incorrect bug classification predictions.'\n",
    "    '\\n\\n'  # hack to avoid overlap with x-axis labels below\n",
    ")\n",
    "confusion_axes.xaxis.tick_top()  # move x-axis labels to top of matrix\n",
    "_ = sns.heatmap(\n",
    "    confusion_matrix,\n",
    "    annot=True,\n",
    "    fmt=\"d\",\n",
    "    cmap=sns.color_palette(\"Blues\"),\n",
    "    vmin=0,\n",
    "    ax=confusion_axes,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "tn, fp, fn, tp = metrics.confusion_matrix(y_test, np.rint(y_pred_oneknn)).flatten().tolist()\n",
    "\n",
    "fpr_onenn = fp/(fp+tp)\n",
    "fnr_onenn = fn/(fn+tn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2NN returns the same as one"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "                     metric_params=None, n_jobs=None, n_neighbors=3, p=2,\n",
       "                     weights='distance')"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier \n",
    "\n",
    "threeknn = KNeighborsClassifier(n_neighbors=3, weights = 'distance')\n",
    "threeknn.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_threeknn = pd.DataFrame(threeknn.predict(x_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_threeknn = y_pred_threeknn.set_index(y_test.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "threeknn_score=threeknn.score(x_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6197354681517577"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "threeknn_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAEFCAYAAABjDmX4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZwU1bn/8c/DwLAIDJuybwou4IoGo4iSGH+CUfEmavQmRgwJWTQJMcZobn4JGjUkudmjXjVGcYmEeKMS4xojYox7NAoCYQSUHWQZQWR/7h/nNNY03T0zNcP09PB9v17zmq5zajl1qurpququp83dERGRumlR7AaIiJQiBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRSabfA0s7Zm9mczqzKzP9ZjPp82s8casm3FYmYjzWxeA89zkpnd1ZDzzLGMh83swj25jMSyGmx9GmofbK7MbISZzTezjWZ2VrHbU1dFD55m9p9m9lLswOXxQDmhAWZ9NtAd6Oru56Sdibvf7e7/rwHas0eZmZvZoELjuPvT7n5QY7Wpobj7GHef0tDzNbNRZrakoeeb0CD7YDN2NfAbd2/v7vcXuzF1VdTgaWaXAr8AriPsZP2AG4CxDTD7/sC/3X17A8yr5JlZy2K3YS9UcB/UNqE/MLvYjUjN3YvyB1QAG4FzCozTmhBcl8W/XwCtY90oYAnwTWAVsBy4KNZdBWwFtsVljAcmAXcl5j0AcKBlHB4HLAA2AAuBTyfK/56Y7njgRaAq/j8+UTcD+AHwTJzPY0C3POuWaf/lifafBZwG/BtYC3wnMf5w4FlgfRz3N0B5rJsZ1+W9uL6fSsz/28AK4M5MWZzmgLiMYXG4F7AaGJWnvd8Glsb1mgecHMsnAdOAO2LdbOCYxHSHxH5ZH+vOjOUDY1mLOHwLsCox3Z3AxES/fj65PYD/BtbFbTUmMd3A2B8bgL8C1ye3e2K8fYD3gZ2xzzbGPqhpfXoB/xv7aiHwtTz9lWsfHBf3jZ8Da4Br4rifA+bE9XkU6J+YzynAXML+9hvgqURfTKLwPl0B3ErYX5YC1wBltezHLsBthONuHXB/LJ8FnJEYrxXwDnBUnn74AlBJ2NemA71i+Zux79+P/dO6WLEodQwr2oJhNLA9s6HzjHM18BywH7Av8A/gB7FuVJz+6rgBTwM2AZ3z7Fh5d7R4IL0LHBTregJDkztZYodaB1wQpzs/DndNHORvAgcCbePw5Dzrlmn/92L7v0A4IH8PdACGxh1rYBz/aODDcbkDCAfbxMT8HBiUY/4/IrwJtSURPBM79htAO8JB+9952noQsDix4w8ADkj06+bY/2XAD4HnEgdWJfAdoBz4KCEgZfr5beDo+Hoe4c3rkETdUYl+TQbPbbHtZcCXCQe4xfpnCQGhHDghbtfdgmeij5ZklRVanxbAy3GblQP7xzafmmf+k6i+z42L2+SrcTu2JVxlVRLeZFoC3wX+EcfvFvvr7NiX34jT1zZ43gfcRNi/9wNeAL5Yy378C/AHoHNc9kmx/HLgD4lljgVez7P+HyUE1mGEffDXwMxE/SLgY8WKQfWOYUVbMHwaWFHDOG8CpyWGTwUWJXb890kEX8IZ3Ifz7Fh5d7S4c60HPgm0zWrDOD4InhcAL2TVPwuMi69nAN9N1H0FeCTPumXanzkT6BDbc2xinJeBs/JMPxG4LzGcK3huBdpklWUHi+nA68Br5Hn3BwbFvv0Y0CqrbhLw18TwEOD9+Hok4ay3RaL+HmBSfH0ncCnQgxA8fwx8id3PSmdQPXhWJubXLq57D8Jtn+1Au0T9XdQ9eOZbn2OBt7PGvxK4Lc/8s/e5cTmmfxgYnxhuQTgJ6A98lhi4Y50RriZqDJ6E22BbSOzPhDf7J2vRjz0JZ4Wdc6xTL0JA7xiH7wUuz7P+twI/Tgy3JwTsAXF4ESUcPIt5z3MN0K2G+z69gLcSw2/Fsl3z8Or3kzYRNlCduPt7hEvdLwHLzewvZnZwLdqTaVPvxPCKOrRnjbvviK/fj/9XJurfz0xvZgea2YNmtsLM3iXcJ+5WYN4Aq919cw3j3AIcCvza3bfkGsHdKwnBehKwysymmllyO2Svc5u4XXsBi919Z6I+2V9PEQLYiYRL7RnASfHv6azpknYtz903xZft4/LWJsognDHXVb716Q/0MrP1mT/CWXX3Osw7uz39gV8m5reWECR7E/svM6KHiFPb9elPOGNcnpj3TYQz0Ix8/diX0I/rsmfq7ssItx4+aWadgDHA3XnaUO14cfeNhOO+d57xS0oxg+ezhHfGQl9RWEbYCTL6xbI03iO8u2b0SFa6+6PufgrhXXcuIajU1J5Mm5ambFNd3Eho12B370g4aK2GabxQpZm1J9xHvhWYZGZd8s7I/ffufgJh/Z1wO6Amy4C+Zpbcz5L99RTh7HRUfP13YAQheD5Vi/lnWw50MbPkdu5bYPyC/ZPDYmChu3dK/HVw99PqMI/sZS4mXEon59nW3f9BWJ9d7Tczo/r6FNqnFxOOr26J+XZ096G1XM8uMTjmMgX4DHAO8Ky759v/qx0vZrYP0JXGOV72uKIFT3evItw7ut7MzjKzdmbWyszGmNmP42j3AN81s33NrFscP+138F4FTjSzfmZWQbjcAsDMupvZ2LhxtxBuYOc663kIODB+vaqlmX2KcFn3YMo21UUHwv27jfGs+MtZ9SsJ9+Dq4pfAS+7+ecI9rv/JNZKZHWRmHzWz1oT7gZkPWmryPOHM7fK4bUcBZwBTAdx9fpzXZ4Cn3P3duB6fJEXwdPe3gJcIbwTlZnZcXF4+K4GucX+ojReADWb27fgdzjIzO9TMPlTXtib8D3ClmQ0FMLMKM8t8rekvwFAz+0Q88/0a1QNk3n3a3ZcTPrD8qZl1NLMWZnaAmZ1UU4PitA8DN5hZ57jtTkyMcj/hPubXCR+s5XMPcJGZHRn3neuA5919UU1tKAVF/aqSu/+UcM/ru4QPSxYDlxA2DoRPB18i3I97HfhnLEuzrMcJN8BfI9xLTAa8FrEdywiXTSexe3DC3dcApxM+4V9DuHl+uru/k6ZNdXQZ8J+E+023ENYlaRIwJV6inVvTzMxsLOFDu8x6XgoMM7NP5xi9NTCZcPN/BeHS78oc41Xj7lsJwWtMnPYG4LPuPjcx2lOE2xeLE8NG2NZpfBo4jvhpNqGf8t2OmEs4wBfEfuuVa7zE+DsI2/9IwqfT7wC/JXyqnYq730c4i58ab8fMIvQXcb86h9D3a4DBhEvmzLSF9mkI90zLCR8KriPcn+xZy6ZdQLg/OZdwv3tiYrnvE75xMBD4U4F1+yvw/+O4ywnf8Divlstv8jKfrIk0S2b2B2Cuu3+/2G1pCGY2g/Ah0W+L3I7vAQe6+2eK2Y5iKvoTRiINycw+FC9PW5jZaMJXaUru6ZWmLN4bHw/cXOy2FJOCpzQ3PQif2m8EfgV82d1fKWqLmhEz+wLh9trD7j6z2O0pJl22i4ikoDNPEZEUFDxroabsOzEjVF2/JlRSzGyGmX2+yG3Yo+nvzGx2/DoVFtxmZuvM7AXbA+n84nL6xf2nrKHnvacl9wmrR+pGa8SUgw2pJINnI6QSqxMPKbUWFLsdaZnZODP7e7HbUWzuPtTdZ8TBEwhJOfq4+3BvoHR+ZrbIzD6WWObbcf/ZUWi6ps5rmbox1xug76GUg3taowTPXI9gKh1XeurPRtGfkEfhvWI3pDFo/0mhpoffCY+D/YnwJfY1hOSlEALvdwnPrq4iPGlQ4dUTFIwnZMeZmassjvthQrak9cC/SKREI0daLPKkEsvR7o8DrxCeyllMTEaR1b4LY1veAf4rUd8WuD0u8w3gW2QlkMha1q6kHHG66wlPh2wgPGVzQGLcocDjhC/jrySmnaN26fd2Sy+XLIvjnk548mR97NfDC21LQjafzcCO2Jfr86zjDEKGoRdinz4AdEm2L2v8RcSkD7E/p8T+nEN4uKBQf+bro0lUT4Txx7juVYR9bGii7rS47TYQHge8LJZ3I3yZPPMc+dN8kIBkESH5yfisPrkqex1z9WUsPwD4Wyx7h/Dcd6dYdyfV07Bdzu6ZkHoRkrWsJWRb+kJimZMokC4vz375NULmp3eAnyTWdRwNmx5vHNVTN+62DQkPZSTT9P0rsW9l5lObuJLvuB1OeKgm86Taz2qKb/X5qylwlhEC2s8JQasNcEKikysJjwS2jztS5gDOrOQdcbq2ecp6xw13Wuy0U+LwvnE++dJijaLAwZcY57A438NjZ56V1b5bYjuOIDyFkkmHNplwUHUhHCSzCi2P3YPnmrghWxIOnqmxrgPhSYtvxr7sQMyiRO3S72Wnl8suOyrucMfGbXchISC0rmFbjiOx4xcInksJSUT2ITw1clctg+dkwoHWGehDeCImZ3/W0EeTqB48PxfrM288rybqlgMj4+vOfJC39IeERyJbxb+RfPCtk2Sbq/VJch1r6MtBhP24ddyOM4Ff5OqXrH0xEzxnEp7EakN4kmk18NHE+udMl1dgv3ySsB/3I+SJTQa7hkyPt6u/6rINcwTP2sSVfMfts8AF8XV7Yoa1YgXP4+LG2y3nJvAE8JXE8EGEd5SWiZXcP8dOkiz7dqZjEmWPEg76QmmxRlFD8MwxzS+An2e1pU+i/gXgvPh6ATA6UTeh0PLYPXj+NlF3GuEJFwgpwV7JM4+a0u/lSi+XXXYjMeAmyuYRHjcttC3HUbvgOTkxPCQuvyzX9qB6IKqW8xL4fL7+rKGPJpE/vVynuB0yZylvA18kpk5LjHc14ax5UI55JNtcrU+oHjzz9mWOeZ6VXB8KBE/CG/UOoEOi/ofA7Yn1z5kur8B+mdyPvwI8kVi/hkyPt6u/6roNqR48axNX8h23MwlXCTkTkDf0X033PPsCb3nunxHIlS4uk0cwI1f6rGRZf+CcrBRfJxACZ960WLVhZsea2ZNmttrMqgjp5rJTuOVLH1ctFRi7p6GrSb759iUEyVxqSr+XK71cdll/4JtZ/dk3zqfQtqyt7D5pRc1p8WD3/iyUVq1QH+0Sk3JMNrM34zPhi2JVpj2fJLxxvWVmT8UkIRAuXSuBx8xsgZldUYv252pjzr6MSWammtnS2K67qF0fwQcp9TYkympKedimhvuV2dusV546aLj0eLXahnnUJq7kO77GExKRzzWzF83s9JRtqJWagudioF+ejZMrXdx2quej9BzTJcsWE848k+m49nH3yRROi5Vrvtl+T7h31NfdKwiXajWlcMuolgqMsG4NYTH5Mx/VlH6vpr7MzP/arP5s5+73UHhb1qY/Yfc+2Ua471QtNVr82s2+iXGXEy7Xc80nW6E+SvpPwmXmxwiJOQZkFg/g7i+6+1jCbZD7CfcKcfcN7v5Nd98fOBO41MxOrsXystuYry+vI/TnYR5SB36G6vtdob5eRtjnOyTK6pvyMHubFdqn6pMeL3s++bZhTftabeJK7hm7z3f38wnb/EfAvRYype0RNQXPFwidNtnM9jGzNmY2ItbdA3zDzAZayAt5HSE9f13ObO4CzjCzU+OZRJv4NaQ+XjgtVm1SiXUgvItvNrPhhIOttqYR0oR1NrM+hPtCDeFBoKeZTTSz1mbWwcyOjXUNkX7vFuBL8azb4jb7eDwYC23LlUAfMyuvYf6fMbMhFvJlXg3c6+ErNv8mnAF93MxaEe6VtU5Ml+zP3oTMWfkU6qOkDoT7XWsIgfu6TIWFdHSfNrMKd99G+ABhZ6w73cwGxYO/inCZXJv0ekmF+rID4cOQqriu38qaNm/qQA+Zpf4B/DDO83DC2VR9vtv6rdjvfQkp5LKzcSXVJz1eUqFtuBIYYNVzvCaljitm9hkz29dDEu31sbiu27bWCgbPeGCcQbgJ/jbhHsenYvXvCJ8eziSk59pMHYNM3FnGEj6Jy6Sk+1aiXTnTYnntUol9BbjazDYQAtG0OjTtKsLlwkJCTsQ767Je+cTLsVMIfboCmA98JFbXO/2eu79E+E2a3xA+La0k3IuqaVv+jfDJ7QozK5Re707CPd0VhA8CvhbnXUXo798SzpLei/PPuDoOLyT8KNu95E8TV6iPku4gbKOlhE/Vn8uqvwBYFC+dv0RIVQchrdtfCQHuWeAGd3+ywDrnamOhvryKkOuyihBwslO2/ZDwJrnezC7LMfvzCWfRywi/QfR9D6nd0nqAkK7u1dieW/ON6PVIj5c1n0LbMPP79WvMLFfawfrEldHAbDPbSMhVe56H9HmZB1lG1nI+taJn26XRmdmXCTv2ScVuS3NmZk745YHKYrelOSrJJ4yktJhZTzMbYSFN3EGEr7DcV+x2idSHniqQxlBO+PGxzK9iTiV8l1GkZOmyXUQkBV22i4ikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4lgAzG21m88ys0syuKHZ7pPbM7HdmtsrMZhW7LdKwFDybODMrA64HxgBDgPPNbEhxWyV1cDswutiNkIan4Nn0DQcq3X2Bu28FpgJji9wmqSV3nwmsLXY7pOEpeDZ9vYHFieElsUxEikjBU0QkBQXPpm8p0Dcx3CeWiUgRKXg2fS8Cg81soJmVA+cB04vcJpG9noJnE+fu24FLgEeBOcA0d59d3FZJbZnZPcCzwEFmtsTMxhe7TdIwzN2L3QYRkZKjM08RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEuEmU0odhskPW2/5kfBs3To4Ctt2n7NjIKniEgKzeZL8u3Pvb15rEge2xbMoNX+o4rdjD1i9JjDi92EPW7BjD+x/6hPFLsZe8y9Fw2zNNPVdNxunDYu1Xwbg848S0RzDZx7i+YcOPdWCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIpIyTKz35nZKjOblSibZGZLzezV+Hdaou5KM6s0s3lmdmqifHQsqzSzK2qzbAVPESlltwOjc5T/3N2PjH8PAZjZEOA8YGic5gYzKzOzMuB6YAwwBDg/jltQywZaARGRRufuM81sQC1HHwtMdfctwEIzqwSGx7pKd18AYGZT47hvFJqZzjxFpMkyswlm9lLib0ItJ73EzF6Ll/WdY1lvYHFinCWxLF95QTrzFJGiGXBg4Rg1y/1m4OY6zvZG4AeAx/8/BT6Xpn2FKHiKSLPi7iszr83sFuDBOLgU6JsYtU8so0B5XrpsF5Fmxcx6Jgb/A8h8Ej8dOM/MWpvZQGAw8ALwIjDYzAaaWTnhQ6XpNS1HZ54iUrLM7B5gFNDNzJYA3wdGmdmRhMv2RcAXAdx9tplNI3wQtB242N13xPlcAjwKlAG/c/fZNS1bwVNESpa7n5+j+NYC418LXJuj/CHgobosW5ftIiIpKHiKiKSg4CkikoKCp4hICvrAqIhu+PIIxgzrw+qqzQy/7AEApkw8icG9KgCoaFdO1aatHH/5dFqVteBXE45j2AHd2LnTufz2F3j6jRW0LS/jzktHsX/3juzYuZOHXl7C93//cjFXa6/wlRH9OLpvBVWbt3Pp/XMAOG5AJ849sie9O7Xhyj/P4801mwBo37qMyz6yPwd0a8eMyjXc+twSAMrLjG9+ZH96dGjNTndeWlzF3S8vK9o6Sd0oeBbR3TMquemROdxy8chdZRf+4qldr6+74Bje3bQNgIs+diAAx172APt2bMOfvvMxTrwyfPf3V3+ezczZK2hV1oK/fO9UTjmyN4+/WuN3fKUenqxcy8NzV/PVkQN2lb29bjM/+dsCvnh8v2rjbtvhTP3nMvp1bkvfzm2q1U2ftZLZKzbSsoXx/VMHc1Tvjryy9N3GWAWppyYTPM3sYMLD+JnntZYC0919TvFatWc9M2cl/fZtn7f+E8cN5ONXPwLAwX0qeGrWcgBWv7uZqve2Mmz/brz85jvMnL0CgG07dvLqwjX07tpuzzd+Lzdn5Ub2bV9erWxp1eac427ZvpO5q96jR8fW1cq37nBmr9gIwPadzoK1m+i6T6s902BpcE3inqeZfRuYChjhG/8vxNf31Da3XnMz4pDurKp6nzdXbADg9UXr+Pgx/ShrYfTftz1H7t+NPt32qTZNRbtyxhzdlxmvLy9Gk6Ue2pWXcUzfCl5btqHYTZFaaipnnuOBoe6+LVloZj8DZgOTc00UM6xMACg/+kJa7T9qz7ayEZ0zYiB/fGbhruE7npzPQX0qeHryGby9eiPPz1vFjp2+q76shXHb10/kxofnsGjVxmI0WVJqYfCNkwbw0BurWLVxa7GbI7XUVILnTqAX8FZWec9Yl5MnMq60P/d2zzdeqSlrYZw5vD8nXPHnXWU7djpXTHlx1/Bff3Aalcuqdg3/+ovH8+aKd7nhoYIpCKUJ+tLx/Vj+7hb+8sbqYjdF6qCpBM+JwBNmNp8P8ur1AwYBlxStVUXykcN68e9lVSxbu2lXWdvyMsyMTVu285HDerJjx07mLg3B83ufOoqKdq24+H+eKVaTJaXzhvWkXXkZNz7zdrGbInXUJIKnuz9iZgcSsjonPzB6MfPgfnN029dPZOSQHnTt0IZ5N57DtdNe5Y4n53N21iU7wL4Vbbn/v07BdzrL1m7i8795GoBeXdpx+SePYN6S9TzzozMBuOmROUz52/xGX5+9ycSTBjC0Rwc6tGnJTeceyh9eWc7GLdsZ/+G+dGzTkitPOYBFa9/nmscqAbjh7KG0LS+jZQtjeL9O/ODRSjZt28HZR/RkyfrN/PjMgwF4ZM5qnpi/ppirJrVk7s3jarc5XbbvbUaPObzYTZB6uveiYZZmukO/+3jB43bWNaekmm9jaBKftouIlBoFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxEpWfF32VeZ2axE2U/MbG783fb7zKxTou5KM6s0s3lmdmqifHQsq6ztI+EKniJSym4HRmeVPQ4c6u6HA/8GrgQwsyGEX8YcGqe5wczKzKwMuB4YAwwBzo/jFqTgKSIly91nAmuzyh5z9+1x8DnC77BDyNo21d23uPtCoJLwYM5woNLdF7j7VkKSorE1LVvBU0Sas88BD8fXvfng8W+AJbEsX3lBCp4i0mSZ2QQzeynxN6EO0/4X4ffZ794TbWsSz7aLyN7p4AO6FqyflcicVhdmNg44HTjZP3gGfSnQNzFan1hGgfK8dOYpIs2KmY0GLgfOdPdNiarpwHlm1trMBgKDCYnXXwQGm9lAMysnfKg0vabl6MxTREqWmd0DjAK6mdkS4PuET9dbA4+bGcBz7v4ld59tZtOANwiX8xdnsraZ2SXAo0AZ8Dt3n13TshU8RaRkufv5OYpvLTD+tcC1OcofAh6qy7J12S4ikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIlKyzOzrZjbLzGab2cRY1sXMHjez+fF/51huZvYrM6s0s9fMbFh9lq3gKSIlycwOBb4ADAeOAE43s0HAFcAT7j4YeCIOA4wh/GLmYGACcGN9lq/gKSKl6hDgeXff5O7bgaeATwBjgSlxnCnAWfH1WOAOD54DOplZz7QLV/AUkVI1CxhpZl3NrB1wGtAX6O7uy+M4K4Du8XVvYHFi+iWxLJVm89PDo8ccXuwmSEqPPPxasZsg9XVRvW4f5mVmEwiX2Bk3u/vNAO4+x8x+BDwGvAe8CuxITu/ubma+J9rWbIKniJSe4QMqCtbfGwLlzfnq3f1W4u+0m9l1hLPJlWbW092Xx8vyVXH0pYQz04w+sSwVXbaLSMkys/3i/36E+52/B6YDF8ZRLgQeiK+nA5+Nn7p/GKhKXN7Xmc48RaSU/a+ZdQW2ARe7+3ozmwxMM7PxwFvAuXHchwj3RSuBTcBF9VmwgqeIlCx3H5mjbA1wco5yBy5uqGXrsl1EJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUkZJlZp3M7F4zm2tmc8zsODPrYmaPm9n8+L9zHNfM7FdmVmlmr5lZvX5sXsFTRErZL4FH3P1g4AhgDnAF8IS7DwaeiMMAY4DB8W8CcGN9FqzgKSIlycwqgBOBWwHcfau7rwfGAlPiaFOAs+LrscAdHjwHdDKznmmXr+ApIk2WmU0ws5cSfxMS1QOB1cBtZvaKmf3WzPYBurv78jjOCqB7fN0bWJyYfkksS0W/2y4iRXNUj4qC9e5+M3BznuqWwDDgq+7+vJn9kg8u0TPTu5l5Q7Q1m848RaRULQGWuPvzcfheQjBdmbkcj/9XxfqlQN/E9H1iWSoKniJSktx9BbDYzA6KRScDbwDTgQtj2YXAA/H1dOCz8VP3DwNVicv7OtNlu4iUsq8Cd5tZObAAuIhwUjjNzMYDbwHnxnEfAk4DKoFNcdzUFDxFpGS5+6vAMTmqTs4xrgMXN9SyddkuIpKCgqeISAoKniIiKSh4ioikoA+MiugrI/pxdN8KqjZv59L75wDQvryMb4wayH4dylm1YSs/m7GQ97bu4MxD92Pk/l0AKGth9K5ow/h7XmPLjp1cPeZAWpUZZWY8u2g9015N/e0LqaUbvjyCMcP6sLpqM8MvC9+EmTLxJAb3Cl/6rmhXTtWmrRx/+XQAhvbrzK8mHEfHtq3Y6XDilQ+yZdsOPnncAL71icMpa2E8/M8lfO/ul4u2TlI3Cp5F9GTlWh6eu4Bq1gsAAAZhSURBVJqvjhywq+ysw3vw+vIN3P/YSs46rDv/cXh37nppGdNnrWL6rPBd36P7VnD60P3YuHUHAFc9Mp/N23dSZnDNxw/ilaVVzF+9qRirtNe4e0YlNz0yh1suHrmr7MJfPLXr9XUXHMO7m7YB4c3u1q+O5PO/eZpZb62jS/vWbNu+ky7tW3PNBccw8tt/5p0NW7jp4hMYdWhPZszSm18p0GV7Ec1ZuZGNW3ZUK/tQvwpmVK4BYEblGj7Ur9Nu050wsDPPLFi7a3jz9p1AOEjLWhjskYfRJOmZOStZt3Fr3vpPHDeQPz6zAICTj+jFrLfXMeutdQCs3biFne4M6N6eN5e/yzsbtgDw5GvLGHts/z3feGkQTT54mlm9vshaajq1acn697cDsP797XRqU/3ioLzMOLJPR55btH5XWQuDn5x5MLeefzivLXuX+e/orLOYRhzSnVVV7/Pmig0ADOpZgTvc/51T+PvkM5h45qEALFixgcG9Kui3b3vKWhhnDO9H7277FLPpUgdNPngCV+WrSGZcWTDjT43ZpkaTfRJ5TL9OzFv53q5LdoCdDt+aPpcvTpvFoG770LdTm8ZtpFRzzoiB/PGZhbuGW5YZxx28H+N/PZNTvvcQZwzvx6hDe7L+va1M/O2zTJl4Eo9dPYa3Vm1kx05dNpSKJnHP08xey1fFB+mkdpPMuHL2bf9sFnvd+s3b6dQ2nH12atuSqs3bq9WPGNiZvy9cm3PaTVt3MGv5Bo7q05HF6zc3RnMlS1kL48zh/Tnhij/vKlu2ZhPPzFnJmnh5/tgrSzhiYBdmzFrOwy8v4eGXlwBw0ckHKniWkKZy5tkd+CxwRo6/NUVsV6N76e0qRg3qCsCoQV158e2qXXXtWrVgSI/21co6tm5Ju/IyIFzSH9GrI0sVOIvmI4f14t/Lqli29oNbJ3/911KG9u1M2/IyyloYJxzSg7lLwjbct2O4Sui0TzlfOPVgpvxtflHaLXXXJM48gQeB9vE51WrMbEbjN6dxTDxpAEN7dKBDm5bcdO6h/OGV5dz3+gq+OWogJx/YldUbt/KzJz+4/BvevxOvLX2XLfEDIoDO7Vpxycj+tDDDDP6xcB0vL3m3GKuzV7nt6ycyckgPunZow7wbz+Haaa9yx5PzOTvrkh1g/Xtb+fVfZjPzh6fjDo++soRHXwlnmz++aDiH9Q9fQZt877+oXK5tVyosPCtf+prLZfve6JGH8921kVKxcdo4SzPd43PeKXjcnnJIt1TzbQxN5bJdRKSkKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKSEkyszZm9oKZ/cvMZpvZVbF8oJk9b2aVZvaH+PtGmFnrOFwZ6wfUZ/kKniJSqrYAH3X3I4AjgdHxVzF/BPzc3QcB64DxcfzxwLpY/vM4XmoKniJSkjzYGAdbxT8HPkr4DXeAKcBZ8fXYOEysP9nMUn+PVMFTRJqsZPKf+Dchq77MzF4FVgGPA28C6909kxRiCdA7vu4NLAaI9VVA17RtayqPZ4rIXmhg18Ip+JLJf/LU7wCONLNOwH3AwQ3awAJ05ikiJc/d1wNPAscBncwsc2LYB1gaXy8F+gLE+grqkXhIwVNESpKZ7RvPODGztsApwBxCED07jnYh8EB8PT0OE+v/5vVI7qHLdhEpVT2BKWZWRjgRnObuD5rZG8BUM7sGeAW4NY5/K3CnmVUCa4Hz6rNwBU8RKUnu/hpwVI7yBcDwHOWbgXMaavm6bBcRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPESlZZjbazOaZWaWZXdGYy1bwFJGSFH/47XpgDDAEON/MhjTW8hU8RaRUDQcq3X2Bu28FpgJjG2vhCp4iUqp6A4sTw0tiWaNoNj89fO9Fw6zYbZCULhpW7BZIkQzar23B49bMJgATEkU3u/vNe7ZVtdNsgqeIND8xUOYLlkuBvonhPrGsUeiyXURK1YvAYDMbaGblwHnA9MZauM48RaQkuft2M7sEeBQoA37n7rMba/nm7o21LBGRZkOX7SIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAr/BzlLcuHZUV8KAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.DataFrame(\n",
    "    data=metrics.confusion_matrix(y_test, np.rint(y_pred_threeknn)), \n",
    "    \n",
    ")\n",
    "\n",
    "confusion_figure, confusion_axes = matplotlib.pyplot.subplots()\n",
    "confusion_figure.set_size_inches(5, 3)\n",
    "confusion_axes.set_title(\n",
    "    'Confusion matrix showing the frequency of \\n'\n",
    "    'correct and incorrect bug classification predictions.'\n",
    "    '\\n\\n'  # hack to avoid overlap with x-axis labels below\n",
    ")\n",
    "confusion_axes.xaxis.tick_top()  # move x-axis labels to top of matrix\n",
    "_ = sns.heatmap(\n",
    "    confusion_matrix,\n",
    "    annot=True,\n",
    "    fmt=\"d\",\n",
    "    cmap=sns.color_palette(\"Blues\"),\n",
    "    vmin=0,\n",
    "    ax=confusion_axes,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "tn, fp, fn, tp = metrics.confusion_matrix(y_test, np.rint(y_pred_threeknn)).flatten().tolist()\n",
    "\n",
    "fpr_threenn = fp/(fp+tp)\n",
    "fnr_threenn = fn/(fn+tn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gaussian Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "gnb = GaussianNB()\n",
    "y_pred_gnb = gnb.fit(x_train, y_train).predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of mislabeled points out of a total 5746 points : 2132\n",
      "Accuracy 0.6289592760180995\n"
     ]
    }
   ],
   "source": [
    "total_error = (y_test != y_pred_gnb).sum()\n",
    "\n",
    "print(\"Number of mislabeled points out of a total %d points : %d\"\n",
    "       % (x_test.shape[0],total_error))\n",
    "print(\"Accuracy \" + str(1- (total_error/x_test.shape[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "gnb_score = 1- (total_error/x_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAEFCAYAAABjDmX4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3de7xVdZ3/8ddb7gqigldAxGtKZdmENmaalxFv4W9SR9MSQxmtpjE1b/VLstHMmrFp1Jm85TXRTI1Mx7BE7ecFMEtFQU94AURQERQFFPj8/vh+D65z2HufcxaHs8/B9/PxOI+z13d911rf9V1rf/a67PXZigjMzKxt1qt3A8zMuiIHTzOzEhw8zcxKcPA0MyvBwdPMrAQHTzOzEtbZ4Cmpj6TfSlok6VdrMJ9jJf2+PdtWL5L2kjSjnec5TtKN7TnPCsu4R9Lxa3MZhWW12/q01z64rpK0p6TnJS2WdHi929NWdQ+ekr4kaWruwLn5jfLZdpj1EcDmwICIOLLsTCLipoj4h3Zoz1olKSRtX6tORDwUETt1VJvaS0QcFBHXtfd8Je0jaXZ7z7egXfbBddj5wKUR0Tci7qx3Y9qqrsFT0mnAT4ELSTvZ1sDlwKh2mP1Q4LmIWN4O8+ryJHWvdxs+hGrug94mDAWm1bsRpUVEXf6A/sBi4MgadXqRgusr+e+nQK88bh9gNnA6MB+YC5yQx30feA94Py9jDDAOuLEw722AALrn4dHATOBt4AXg2EL5nwrT/T0wBViU//99Ydwk4AfA/8vz+T0wsMq6Nbb/zEL7DwcOBp4DFgDnFuqPAB4BFua6lwI987gH87q8k9f3nwrzPwt4FbihsSxPs11exm55eCvgNWCfKu09C5iT12sGsF8uHwfcClyfx00D/q4w3c65XxbmcV/I5cNy2Xp5+EpgfmG6G4BTC/16YnF7AD8B3szb6qDCdMNyf7wN3AdcVtzuhXobAEuAlbnPFuc+aGl9tgJ+nfvqBeCbVfqr0j44Ou8blwBvAP+W634VeDavz73A0MJ8DgCmk/a3S4EHCn0xjtr7dH/gatL+Mgf4N6BbK/txE+AXpPfdm8Cdufxp4LBCvR7A68Anq/TDSUADaV+bAGyVy/+W+35J7p9e9YpFpWNY3RYMI4HljRu6Sp3zgUeBzYBNgYeBH+Rx++Tpz88b8GDgXWDjKjtW1R0tv5HeAnbK47YEhhd3ssIO9Sbw5TzdMXl4QOFN/jdgR6BPHr6oyro1tv97uf0nkd6QvwT6AcPzjjUs1/8UsEde7jakN9uphfkFsH2F+f+I9CHUh0LwLOzYzwDrk960P6nS1p2AWYUdfxtgu0K/Ls393w34IfBo4Y3VAJwL9AT2JQWkxn5+GfhUfj2D9OG1c2HcJwv9Wgye7+e2dwNOIb3Blcc/QgoIPYHP5u26WvAs9NHsZmW11mc94PG8zXoC2+Y2H1hl/uNous+NztvkX/J27EM6y2ogfch0B74LPJzrD8z9dUTuy2/l6VsbPO8Afk7avzcDJgP/3Mp+/B1wC7BxXvbeufxM4JbCMkcBT1VZ/31JgXU30j74X8CDhfEvAvvXKwatcQyr24LhWODVFur8DTi4MHwg8GJhx19CIfiSjuD2qLJjVd3R8s61EPgi0KdZG0bzQfD8MjC52fhHgNH59STgu4VxXwP+t8q6Nba/8UigX27P7oU6jwOHV5n+VOCOwnCl4Pke0LtZWfNgMQF4CniSKp/+wPa5b/cHejQbNw64rzC8C7Akv96LdNS7XmH8zcC4/PoG4DRgC1LwvBg4mdWPSifRNHg2FOa3fl73LUiXfZYD6xfG30jbg2e19dkdeLlZ/XOAX1SZf/N9bnSF6e8BxhSG1yMdBAwFvkIO3HmcSGcTLQZP0mWwZRT2Z9KH/f2t6MctSUeFG1dYp61IAX3DPHwbcGaV9b8auLgw3JcUsLfJwy/ShYNnPa95vgEMbOG6z1bAS4Xhl3LZqnlE0+tJ75I2UJtExDukU92TgbmSfifpI61oT2ObBhWGX21De96IiBX59ZL8f15h/JLG6SXtKOkuSa9Keot0nXhgjXkDvBYRS1uocyXwUeC/ImJZpQoR0UAK1uOA+ZLGSypuh+br3Dtv162AWRGxsjC+2F8PkALY50in2pOAvfPfQ82mK1q1vIh4N7/sm5e3oFAG6Yi5raqtz1BgK0kLG/9IR9Wbt2HezdszFPjPwvwWkILkIHL/NVaMFHFauz5DSUeMcwvz/jnpCLRRtX4cQurHN5vPNCJeIV16+KKkjYCDgJuqtKHJ+yUiFpPe94Oq1O9S6hk8HyF9Mtb6isIrpJ2g0da5rIx3SJ+ujbYojoyIeyPiANKn7nRSUGmpPY1tmlOyTW3x36R27RARG5LetGphmqg1UlJf0nXkq4FxkjapOqOIX0bEZ0nrH6TLAS15BRgiqbifFfvrAdLR6T759Z+APUnB84FWzL+5ucAmkorbeUiN+jX7p4JZwAsRsVHhr19EHNyGeTRf5izSqXRxnn0i4mHS+qxqvyTRdH1q7dOzSO+vgYX5bhgRw1u5npvk4FjJdcBxwJHAIxFRbf9v8n6RtAEwgI55v6x1dQueEbGIdO3oMkmHS1pfUg9JB0m6OFe7GfiupE0lDcz1y34H7y/A5yRtLak/6XQLAEmbSxqVN+4y0gXsSkc9dwM75q9XdZf0T6TTurtKtqkt+pGu3y3OR8WnNBs/j3QNri3+E5gaESeSrnH9T6VKknaStK+kXqTrgY03WlryGOnI7cy8bfcBDgPGA0TE83lexwEPRMRbeT2+SIngGREvAVNJHwQ9JX0mL6+aecCAvD+0xmTgbUln5e9wdpP0UUmfbmtbC/4HOEfScABJ/SU1fq3pd8BwSf+Yj3y/SdMAWXWfjoi5pBuW/y5pQ0nrSdpO0t4tNShPew9wuaSN87b7XKHKnaTrmP9KurFWzc3ACZI+kfedC4HHIuLFltrQFdT1q0oR8e+ka17fJd0smQV8g7RxIN0dnEq6HvcU8OdcVmZZE0kXwJ8kXUssBrz1cjteIZ027c3qwYmIeAM4lHSH/w3SxfNDI+L1Mm1qozOAL5GuN11JWpeiccB1+RTtqJZmJmkU6aZd43qeBuwm6dgK1XsBF5Eu/r9KOvU7p0K9JiLiPVLwOihPeznwlYiYXqj2AOnyxazCsEjbuoxjgc+Q72aT+qna5YjppDf4zNxvW1WqV6i/grT9P0G6O/06cBXprnYpEXEH6Sh+fL4c8zSpv8j71ZGkvn8D2IF0ytw4ba19GtI1056km4Jvkq5PbtnKpn2ZdH1yOul696mF5S4hfeNgGHB7jXW7D/i/ue5c0jc8jm7l8ju9xjtrZuskSbcA0yPivHq3pT1ImkS6SXRVndvxPWDHiDiunu2op7o/YWTWniR9Op+eridpJOmrNF3u6ZXOLF8bHwNcUe+21JODp61rtiDdtV8M/Aw4JSKeqGuL1iGSTiJdXrsnIh6sd3vqyaftZmYl+MjTzKwEB89WaCn7Ts4I1davCXUpkiZJOrHObVir6e8kTctfp0LJLyS9KWmy1kI6v7ycrfP+06295722FfcJrUHqRnVgysH21CWDZwekEmuTSCm1Zta7HWVJGi3pT/VuR71FxPCImJQHP0tKyjE4IkZEO6Xzk/SipP0Ly3w57z8rak3X2UUrUzdW+gCMtZRycG3rkOBZ6RFMp+Mqz/3ZIYaS8ii8U++GdATvPyW09PA76XGw20lfYn+DlLwUUuD9LunZ1fmkJw36R9MEBWNI2XEerFSW6+5Bypa0EPgrhZRoVEiLRZVUYhXafQjwBOmpnFnkZBTN2nd8bsvrwHcK4/sA1+ZlPgN8m2YJJJota1VSjjzdZaSnQ94mPWWzXaHucGAi6cv488hp52hd+r3V0ssVy3LdQ0lPnizM/frxWtuSlM1nKbAi9+XCKus4iZRhaHLu098AmxTb16z+i+SkD7k/r8v9+Szp4YJa/Vmtj8bRNBHGr/K6LyLtY8ML4w7O2+5t0uOAZ+TygaQvkzc+R/4QHyQgeZGU/GRMsz75fvN1rNSXuXw74I+57HXSc98b5XE30DQN25msnglpK1KylgWkbEsnFZY5jhrp8qrsl98kZX56HfhxYV1H077p8UbTNHXjatuQ9FBGMU3fXwv7VuN8WhNXqr1vR5Aeqml8Uu0/Wopva/LXUuDsRgpol5CCVm/gs4VObiA9Etg370iNb+DGlbw+T9enStmgvOEOzp12QB7eNM+nWlqsfajx5ivU+Vie78dzZx7erH1X5nbsSnoKpTEd2kWkN9UmpDfJ07WWx+rB8428IbuT3jzj87h+pCctTs992Y+cRYnWpd9rnl6uedkn8w63e952x5MCQq8WtuVoCjt+jeA5h5REZAPSUyM3tjJ4XkR6o20MDCY9EVOxP1voo3E0DZ5fzeMbP3j+Uhg3F9grv96YD/KW/pD0SGSP/LcXH3zrpNjmJn1SXMcW+nJ70n7cK2/HB4GfVuqXZvtiY/B8kPQkVm/Sk0yvAfsW1r9iurwa++X9pP14a1Ke2GKwa8/0eKv6qy3bsELwbE1cqfa+fQT4cn7dl5xhrV7B8zN5462WcxP4A/C1wvBOpE+U7oWV3LbCTlIsO6uxYwpl95Le9LXSYu1DC8GzwjQ/BS5p1pbBhfGTgaPz65nAyMK4sbWWx+rB86rCuINJT7hASgn2RJV5tJR+r1J6ueZl/00OuIWyGaTHTWtty9G0LnheVBjeJS+/W6XtQdNA1CTnJXBitf5soY/GUT293EZ5OzQepbwM/DM5dVqh3vmko+btK8yj2OYmfULT4Fm1LyvM8/Di+lAjeJI+qFcA/QrjfwhcW1j/iunyauyXxf34a8AfCuvXnunxVvVXW7chTYNna+JKtfftg6SzhIoJyNv7r6VrnkOAl6LyzwhUShfXmEewUaX0WcWyocCRzVJ8fZYUOKumxWoNSbtLul/Sa5IWkdLNNU/hVi19XJNUYKyehq4l1eY7hBQkK2kp/V6l9HLNy4YCpzfrzyF5PrW2ZWs175MetJwWD1bvz1pp1Wr10So5KcdFkv6Wnwl/MY9qbM8XSR9cL0l6ICcJgXTq2gD8XtJMSWe3ov2V2lixL3OSmfGS5uR23Ujr+gg+SKn3dqGspZSHvVu4Xtl8m21VZRy0X3q8Vm3DKloTV6q9v8aQEpFPlzRF0qEl29AqLQXPWcDWVTZOpXRxy2majzIqTFcsm0U68iym49ogIi6idlqsSvNt7peka0dDIqI/6VStpRRujZqkAiOtW3uYRfXMRy2l32upLxvnf0Gz/lw/Im6m9rZsTX/C6n3yPum6U5PUaPlrN5sW6s4lna5Xmk9ztfqo6Euk08z9SYk5tmlcPEBETImIUaTLIHeSrhUSEW9HxOkRsS3wBeA0Sfu1YnnN21itLy8k9efHIqUOPI6m+12tvn6FtM/3K5StacrD5tus1j61Junxms+n2jZsaV9rTVypPOOI5yPiGNI2/xFwm1KmtLWipeA5mdRpF0naQFJvSXvmcTcD35I0TCkv5IWk9PxtObK5EThM0oH5SKJ3/hrS4KidFqs1qcT6kT7Fl0oaQXqztdatpDRhG0saTLou1B7uAraUdKqkXpL6Sdo9j2uP9HtXAifno27lbXZIfjPW2pbzgMGSerYw/+Mk7aKUL/N84LZIX7F5jnQEdIikHqRrZb0K0xX7cxApc1Y1tfqoqB/petcbpMB9YeMIpXR0x0rqHxHvk24grMzjDpW0fX7zLyKdJrcmvV5Rrb7sR7oZsiiv67ebTVs1dWCkzFIPAz/M8/w46WhqTb7b+u3c70NIKeSaZ+MqWpP0eEW1tuE8YBs1zfFaVDquSDpO0qaRkmgvzMVt3batVjN45jfGYaSL4C+TrnH8Ux59Denu4YOk9FxLaWOQyTvLKNKduMaUdN8utKtiWqxoXSqxrwHnS3qbFIhubUPTvk86XXiBlBPxhrasVzX5dOwAUp++CjwPfD6PXuP0exExlfSbNJeS7pY2kK5FtbQt/0i6c/uqpFrp9W4gXdN9lXQj4Jt53otI/X0V6SjpnTz/Rufn4RdIP8p2G9XTxNXqo6LrSdtoDumu+qPNxn8ZeDGfOp9MSlUHKa3bfaQA9whweUTcX2OdK7WxVl9+n5TrchEp4DRP2fZD0ofkQklnVJj9MaSj6FdIv0F0XqTUbmX9hpSu7i+5PVdXqxhrkB6v2XxqbcPG369/Q1KltINrEldGAtMkLSblqj06Uvq8xgdZ9mrlfFrFz7Zbh5N0CmnH3rvebVmXSQrSLw801Lst66Iu+YSRdS2StpS0p1KauJ1IX2G5o97tMlsTfqrAOkJP0o+PNf4q5njSdxnNuiyftpuZleDTdjOzEhw8zcxKcPA0MyvBwdPMrAQHTzOzEhw8zcxKcPA0MyvBwdPMrAQHTzOzEhw8zcxKcPA0MyvBwdPMrAQHTzOzEhw8zcxKcPA0MyvBwdPMrAQHTzOzEhw8zcxKcPA0MyvBwdPMrAQHTzOzEhw8zcxKcPA0MyvBwdPMrAQHTzOzEhw8zcxKcPA0MyvBwdPMrAQHTzOzEhw8zcxKcPA0MyvBwdPMrAQHzy5A0khJMyQ1SDq73u2x1pN0jaT5kp6ud1usfTl4dnKSugGXAQcBuwDHSNqlvq2yNrgWGFnvRlj7c/Ds/EYADRExMyLeA8YDo+rcJmuliHgQWFDvdlj7c/Ds/AYBswrDs3OZmdWRg6eZWQkOnp3fHGBIYXhwLjOzOnLw7PymADtIGiapJ3A0MKHObTL70HPw7OQiYjnwDeBe4Fng1oiYVt9WWWtJuhl4BNhJ0mxJY+rdJmsfioh6t8HMrMvxkaeZWQkOnmZmJTh4mpmV4OBpZlaCg6eZWQkOnl2EpLH1boOV5+237nHw7Dr85uvavP3WMQ6eZmYlrDNfku971LXrxopU8f7MSfTYdp96N2OtuOP7h9a7CWvd7269nkOO+kq9m7HWHLDzQJWZrqX37eJbR5eab0fwkWcXsa4Gzg+LdTlwflg5eJqZleDgaWZWgoOnmVkJDp5mZiU4eJqZleDgaWZWgoOnmVkJDp5mZiU4eJqZleDgaWZdkqQhku6X9IykaZL+NZdvImmipOfz/41zuST9TFKDpCcl7VaY1/G5/vOSjm/N8h08zayrWg6cHhG7AHsAX5e0C3A28IeI2AH4Qx4GOAjYIf+NBf4bUrAFzgN2B0YA5zUG3FocPM2sS4qIuRHx5/z6bdJPcw8CRgHX5WrXAYfn16OA6yN5FNhI0pbAgcDEiFgQEW8CE4GRLS3fwdPMOi1JYyVNLfxVzIsqaRvgk8BjwOYRMTePehXYPL8eBMwqTDY7l1Urr6l7G9bDzKxDRcQVwBW16kjqC/waODUi3pI+yGIXESFpraSr9JGnmXVZknqQAudNEXF7Lp6XT8fJ/+fn8jnAkMLkg3NZtfKaHDzNrEtSOsS8Gng2Iv6jMGoC0HjH/HjgN4Xyr+S77nsAi/Lp/b3AP0jaON8o+odcVpNP282sq9oT+DLwlKS/5LJzgYuAWyWNAV4Cjsrj7gYOBhqAd4ETACJigaQfAFNyvfMjYkFLC3fwNLMuKSL+BFT7mY79KtQP4OtV5nUNcE1blu/TdjOzEhw8zcxKcPA0MyvBwdPMrAQHTzOzEhw8zcxKcPA0MyvB3/M0s7rZZscW8290Wj7yNDMrwcHTzKwEB08zsxIcPM3MSnDwNDMrwcHTzKwEB08zsxIcPM3MSvCX5Otk0ID1ufLre7HZRn2ICH5x33Ncfs+z/J89hnLukZ9gp0Ebsfe5d/HEzDcA6NFtPX429jPstt1AVq4Mzrx2Mg898yoAd5x7AFts1Ifu3cTD0+fzraseZWWsld+8sgrmzXmJa378vVXDb8x7hUOOOZH+Azbl7vFXM2/2S5zx4ysZuv3OACx+axFXX/wdXmqYzh77HsRRY0+vV9NtDTh41snyFcE5N0zhry8soG/v7jx00WH88clXeGbWQr70k/v52di/b1L/hP13BGD3M37Dphv25vZz9+dz59xFBHzlkkm8veR9AG46fR/+8TPbcNvDL3T4On1YbT5oKOf8NP1M+MoVK/jOmMPZdY+9eW/ZUk46+0JuvvzHTer36NmTQ790Eq+8PJO5L8+sR5OtHXSa4CnpI6QfpW98XmsOMCEinq1fq9aeeQuXMG/hEgAWL13OjDmL2HKT9bn/qbkV639kcH8eeDqNe+2tpSx65z1223Ygj//t9VWBs3s30aN7N8JHnXUz48mpbLrFIDbZbIuqdXr17sN2u+zKa3Nnd2DLrL11imueks4CxpN+j2Ry/hNws6Sz69m2jrD1pn3ZddgmTG14vWqdp158k0P+bmu6rSeGbtqXT2w7kMEDN1g1/s5zD+CFK49m8ZL3uePRlzqi2VbB43/6A5/aa/96N8M6QKcInsAY4NMRcVFE3Jj/LgJG5HEVSRoraaqkqe/PnNRRbW1XG/Tqzk2n78NZ105edQRZyfX3P8+cBe/w0EWH8aPRI3hsxnxWrPzgCPPwCyey/T/fSq8e67H3R6sf9djas/z993lq8p/45J771rspHxqSrpE0X9LTzcr/RdJ0SdMkXVwoP0dSg6QZkg4slI/MZQ2tPWDrLKftK4GtSD8TWrRlHldRRFwBXAHQ96hru9y5avdu4qbTP88tD81kwuSXa9ZdsTI4+7opq4bv+8HBNLyyqEmdZe+v4K4pszj001tXPf23teeZPz/KkG13ZMONNql3Uz5MrgUuBa5vLJD0edIlwF0jYpmkzXL5LsDRwHBSvLlP0o55ssuAA4DZwBRJEyLimVoL7izB81TgD5KeB2blsq2B7YFv1K1Va9nlJ+/JjDmLuPR3NbcRAH16dkMS7y5bzuc/tiUrVqxk+pxFbNCrO3379GDewiV0W0+M3G0wDz87rwNab81NfWgin/rcAfVuxodKRDwoaZtmxacAF0XEslxnfi4fBYzP5S9IaiCd3QI0RMRMAEnjc93OHzwj4n/zJ8AImt4wmhIRK+rXsrXnMzttxpf23p6nX1rAwxd/AYBxNz9Or+7d+MlXd2fghr359dn78+SLCzj8wols2r8Pd37nAGJl8MqCdznx0ocA2KB3d249cz969ViP9SQenPYqV02cUc9V+1BatnQJ0/86hWNOOXNV2V8ffYBfXXkJixct5H9+8G0GDduBb4y7BIDvnfRFli55h+XLl/PkYw/x9XGXsOWQYfVq/rpmR2AvSRcAS4EzImIKKbY8Wqg3mw/izaxm5bu3tJBOETwBImIlTVdsnfbIjPn0PeraiuN+O2X1U/iXX1vMbqfesVr5/EVL2fvcu9q7edZGvXr34eIb7mlStusee7PrHntXrH/+lb/uiGZ1eZLGAmMLRVfky3W1dAc2AfYAPg3cKmnb9m5bpwmeZmbNFe9rtMFs4PZI39mbLGklMJB0NjukUG9wLqNGeVWd5W67mVl7uRP4PEC+HNgTeB2YABwtqZekYcAOpK9FTgF2kDRMUk/STaUJLS3ER55m1mVJuhnYBxgoaTZwHnANcE3++tJ7wPH5KHSapFtJN4KWA19vvKci6RvAvUA34JqImNbSsh08zazLiohjqow6rkr9C4ALKpTfDdzdlmX7tN3MrAQHTzOzEhw8zcxKcPA0MyvBwdPMrAQHTzOzEvxVJTOrm49sN6DeTSjNR55mZiU4eJqZleDgaWZWgoOnmVkJDp5mZiU4eJqZleDgaWZWgoOnmVkJDp5mZiU4eJqZleDgaWZWgoOnmVkJDp5mZiU4eJqZleDgaWZdlqRrJM3PPzPcWPZjSdMlPSnpDkkbFcadI6lB0gxJBxbKR+ayBklnt2bZDp5m1pVdC4xsVjYR+GhEfBx4DjgHQNIuwNHA8DzN5ZK6SeoGXAYcBOwCHJPr1uTgaWZdVkQ8CCxoVvb7iFieBx8FBufXo4DxEbEsIl4AGoAR+a8hImZGxHvA+Fy3JgdPM1uXfRW4J78eBMwqjJudy6qV1+TgaWadlqSxkqYW/sa2YdrvAMuBm9ZG2/wbRmbWaUXEFcAVbZ1O0mjgUGC/iIhcPAcYUqg2OJdRo7wqH3ma2TpF0kjgTOALEfFuYdQE4GhJvSQNA3YAJgNTgB0kDZPUk3RTaUJLy/GRp5l1WZJuBvYBBkqaDZxHurveC5goCeDRiDg5IqZJuhV4hnQ6//WIWJHn8w3gXqAbcE1ETGtp2Q6eZtZlRcQxFYqvrlH/AuCCCuV3A3e3Zdk+bTczK8HB08yshHXmtH3kQR+vdxOspJ8/+nK9m2Br6ICdB9a7CR1unQmeZtb1jNimf72bUJpP283MSnDwNDMrwcHTzKwEB08zsxIcPM3MSnDwNDMrwcHTzKwEB08zsxIcPM3MSnDwNDMrwcHTzKwEB08zsxIcPM3MSnDwNDMrwcHTzKwEB08zsxIcPM3MSnDwNLMuS9K3JE2T9LSkmyX1zr+//pikBkm35N9iJ/9e+y25/DFJ26zJsh08zaxLkjQI+CbwdxHxUdJvrh8N/Ai4JCK2B94ExuRJxgBv5vJLcr3SHDzNrCvrDvSR1B1YH5gL7AvclsdfBxyeX4/Kw+Tx+0lS2QU7eJpZpyVprKSphb+xjeMiYg7wE+BlUtBcBDwOLIyI5bnabGBQfj0ImJWnXZ7rDyjbNv96ppl1WhFxBXBFpXGSNiYdTQ4DFgK/AkZ2VNt85GlmXdX+wAsR8VpEvA/cDuwJbJRP4wEGA3Py6znAEIA8vj/wRtmFO3iaWVf1MrCHpPXztcv9gGeA+4Ejcp3jgd/k1xPyMHn8HyMiyi7cwdPMuqSIeIx04+fPwFOkeHYFcBZwmqQG0jXNq/MkVwMDcvlpwNlrsnxf8zSzLisizgPOa1Y8ExhRoe5S4Mj2WraDp5nVzSe36F/vJpTm03YzsxIcPM3MSnDwNDMrwcHTzKwEB08zsxIcPM3MSnDwNDMrwcHTzKwEf0m+jr6259Z8akh/Fi1dzml3Pttk3GHDN+P4EYM54Zd/5e1lK9hr2405/GNbgGDp+yu44uFZvPTmkhbnY2uHt535yLOO7m9YwL9NbFitfMAGPdh10Ia8tnjZqrL5i0QoHhIAAAQnSURBVN/je/c8x+l3Psttf3mVk/fcusX52NrjbWcOnnX07LzFLF62YrXy0SMGc8OUORTzvcyY/w7vvJfqPvfaO2yyfo8W52Nrj7eddfrgKemEerehI3166/4sePf9Vad1ley34wCemPNWB7bKWsPb7sOl0wdP4PvVRhRT9M+cdHtHtmmt6NlN/OPHt+CWP79Stc7wLfqy7w4DuXHqnKp1rON52334dIobRpKerDYK2LzadMUU/Uf84s+lk5p2Flts2IvN+vbkJ6N2BmDABj25+As7c85d01m4ZDlDN+7DKXsO5YKJDT7V62S87T58OkXwJAXIA0k/E1ok4OGOb059vPzmUsaMf2rV8OVHDOes307n7WUrGLhBD87Ydxj/9dCLzH1rWY25WD142334dJbgeRfQNyL+0nyEpEkd35yOcere2zB8i370692dnx/1UW55Yi5/fL7yT6oc8Ykt6derOyfuMQSAlRGc9dsZbZ6PtQ9vO9Ma/IRHp7IunLabdVW3nbBbqd8/n/js6zXftwfsPLD076qvbV3hhpGZWafj4GlmVoKDp5lZCQ6eZtalSeom6QlJd+XhYZIek9Qg6RZJPXN5rzzckMdvsybLdfA0s67uX4FiVpUfAZdExPakrz+OyeVjgDdz+SW5XmkOnmbWZUkaDBwCXJWHBewL3JarXAccnl+PysPk8fvl+qU4eJpZp1V8BDv/jW1W5afAmcDKPDwAWBgRy/PwbGBQfj0ImAWQxy/K9UvpLF+SNzNbTfER7OYkHQrMj4jHJe3ToQ3DwdPMuq49gS9IOhjoDWwI/CewkaTu+ehyMNCYiWUOMASYLak70B8o/TiXg6eZ1c2wARuUnjYizgHOAchHnmdExLGSfgUcAYwHjgd+kyeZkIcfyeP/GGvwiKWveZrZuuYs4DRJDaRrmlfn8quBAbn8NODsNVmIjzzNrMuLiEnApPx6JjCiQp2lwJHttUwfeZqZleDgaWZWgoOnmVkJDp5mZiU4eJqZleDgaWZWgoOnmVkJDp5mZiU4eJqZleDgaWZWgoOnmVkJDp5mZiU4eJqZleDgaWZWgoOnmVkJDp5mZiU4eJqZleDgaWZWgoOnmVkJDp5mZiU4eJpZlyVppKQZkhokrdGvYbaVg6eZdUmSugGXAQcBuwDHSNqlo5bv4GlmXdUIoCEiZkbEe8B4YFRHLdzB08y6qkHArMLw7FzWIbp31ILWtttO2E31boOZtc32m/Wp+b6VNBYYWyi6IiKuWLutap11Jnia2bonB8pqwXIOMKQwPDiXdQiftptZVzUF2EHSMEk9gaOBCR21cB95mlmXFBHLJX0DuBfoBlwTEdM6avmKiI5alpnZOsOn7WZmJTh4mpmV4OBpZlaCg6eZWQkOnmZmJTh4mpmV4OBpZlaCg6eZWQn/H93xUZ8hKRbRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.DataFrame(\n",
    "    data=metrics.confusion_matrix(y_test, np.rint(y_pred_gnb)), \n",
    "    \n",
    ")\n",
    "\n",
    "confusion_figure, confusion_axes = matplotlib.pyplot.subplots()\n",
    "confusion_figure.set_size_inches(5, 3)\n",
    "confusion_axes.set_title(\n",
    "    'Confusion matrix showing the frequency of \\n'\n",
    "    'correct and incorrect bug classification predictions.'\n",
    "    '\\n\\n'  # hack to avoid overlap with x-axis labels below\n",
    ")\n",
    "confusion_axes.xaxis.tick_top()  # move x-axis labels to top of matrix\n",
    "_ = sns.heatmap(\n",
    "    confusion_matrix,\n",
    "    annot=True,\n",
    "    fmt=\"d\",\n",
    "    cmap=sns.color_palette(\"Blues\"),\n",
    "    vmin=0,\n",
    "    ax=confusion_axes,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "tn, fp, fn, tp = metrics.confusion_matrix(y_test, np.rint(y_pred_gnb)).flatten().tolist()\n",
    "\n",
    "fpr_gnb = fp/(fp+tp)\n",
    "fnr_gnb = fn/(fn+tn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVM\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import svm\n",
    "clf_svm = svm.SVC(kernel = 'poly', gamma='scale', degree = 3)\n",
    "clf_svm.fit(x_train, y_train)  \n",
    "\n",
    "y_pred_svm = clf_svm.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of mislabeled points out of a total 5746 points : 2022\n",
      "Accuracy 0.6481030281935259\n"
     ]
    }
   ],
   "source": [
    "total_error = (y_test != y_pred_svm).sum()\n",
    "\n",
    "print(\"Number of mislabeled points out of a total %d points : %d\"\n",
    "       % (x_test.shape[0],total_error))\n",
    "print(\"Accuracy \" + str(1- (total_error/x_test.shape[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "svm_score = 1- (total_error/x_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAEFCAYAAABjDmX4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZgV1ZnH8e/bDc2+CSqyg6gR0BiNuACKGkcwKkzUREeNGBISzaLRBHXGUWJioslMTEzUcV8TCHGiwbigUZGIC2B0FES0BZQd2VqafXnnj3Maqy/33u4umr7d7e/zPP30rVPbqVNV7631vebuiIhIzRQVugIiIg2RgqeISAoKniIiKSh4ioikoOApIpKCgqeISAqNNniaWQsze9zMyszsz7sxnfPM7JnarFuhmNkQM5tby9McZ2YP1+Y0s8zjKTO7cE/OIzGvWlue2toGGyszG2Rm75tZuZmNLHR9aqrgwdPM/s3MZsYGXBp3lMG1MOmzgH2Bju5+dtqJuPsf3P1faqE+e5SZuZn1zTeMu//D3Q+qqzrVFncf7u4P1PZ0zWyomS2q7ekm1Mo22IhdD/ze3Vu7+2OFrkxNFTR4mtnlwG+AnxM2sh7AbcCIWph8T+A9d99WC9Nq8MysSaHr8BmUdxvUOqEnMLvQlUjN3QvyB7QDyoGz8wzTjBBcl8S/3wDNYr+hwCLgCmAFsBS4KPb7CbAF2BrnMRoYBzycmHYvwIEmsXsUMA9YB8wHzkuUv5QY71hgBlAW/x+b6DcF+CkwLU7nGaBTjmWrqP/YRP1HAqcC7wGrgX9PDD8QeAVYG4f9PVAS+02Ny7I+Lu/XEtO/ElgGPFRRFsfZP87j8NjdBfgYGJqjvlcCi+NyzQVOiuXjgInAg7HfbOCLifEOju2yNvY7I5b3jmVFsfsuYEVivIeAyxLt+s3k+gD+C1gT19XwxHi9Y3usA/4O3Jpc74nhWgEbgR2xzcpjG1S1PF2A/41tNR/4QY72yrYNjorbxs3AKuBncdhvAHPi8kwGeiamczLwLmF7+z3wYqItxpF/m24H3EPYXhYDPwOKq9mOewH3Efa7NcBjsXwWcHpiuKbASuALOdrhW0ApYVubBHSJ5R/Ett8Y26dZoWJR6hhWsBnDMGBbxYrOMcz1wKvAPsDewMvAT2O/oXH86+MKPBXYAHTIsWHl3NDijvQJcFDstx/QP7mRJTaoNcAFcbxzY3fHxE7+AXAg0CJ235hj2Srqf22s/7cIO+QfgTZA/7hh9Y7DHwEcHefbi7CzXZaYngN9s0z/JsKXUAsSwTOxYb8DtCTstP+Vo64HAQsTG34vYP9Eu26K7V8M/AJ4NbFjlQL/DpQAJxICUkU7fwQcET/PJXx5HZzo94VEuyaD59ZY92LgYsIObrH/K4SAUAIMjut1l+CZaKNFGWX5lqcIeD2usxKgT6zzKTmmP47K29youE6+H9djC8JZVinhS6YJcA3wchy+U2yvs2Jb/jCOX93g+ShwB2H73geYDny7mu34BPAnoEOc9/GxfCzwp8Q8RwBv51j+EwmB9XDCNvg7YGqi/wLgS4WKQbsdwwo2YzgPWFbFMB8Apya6TwEWJDb8jSSCL+EI7ugcG1bODS1uXGuBM4EWGXUYxafB8wJgekb/V4BR8fMU4JpEv0uAp3MsW0X9K44E2sT6HJUY5nVgZI7xLwMeTXRnC55bgOYZZZnBYhLwNvAWOb79gb6xbb8ENM3oNw74e6K7H7Axfh5COOotSvQfD4yLnx8CLgc6E4LnL4HvsOtR6RQqB8/SxPRaxmXvTLjssw1omej/MDUPnrmW5yjgo4zhrwbuyzH9zG1uVJbxnwJGJ7qLCAcBPYGvEwN37GeEs4kqgyfhMthmEtsz4cv+hWq0436Eo8IOWZapCyGgt43djwBjcyz/PcAvE92tCQG7V+xeQAMOnoW85rkK6FTFdZ8uwIeJ7g9j2c5peOXrSRsIK6hG3H094VT3O8BSM3vCzD5XjfpU1KlrontZDeqzyt23x88b4//lif4bK8Y3swPN7G9mtszMPiFcJ+6UZ9oAH7v7piqGuQsYAPzO3TdnG8DdSwnBehywwswmmFlyPWQuc/O4XrsAC919R6J/sr1eJASw4win2lOA4+PfPzLGS9o5P3ffED+2jvNbnSiDcMRcU7mWpyfQxczWVvwRjqr3rcG0M+vTE/htYnqrCUGyK7H9Kgb0EHGquzw9CUeMSxPTvoNwBFohVzt2J7TjmsyJuvsSwqWHM82sPTAc+EOOOlTaX9y9nLDfd80xfINSyOD5CuGbMd8jCksIG0GFHrEsjfWEb9cKnZM93X2yu59M+NZ9lxBUqqpPRZ0Wp6xTTdxOqNcB7t6WsNNaFeN4vp5m1ppwHfkeYJyZ7ZVzQu5/dPfBhOV3wuWAqiwBuptZcjtLtteLhKPTofHzS8AgQvB8sRrTz7QU2MvMkuu5e57h87ZPFguB+e7ePvHXxt1PrcE0Mue5kHAqnZxmC3d/mbA8O+tvZkbl5cm3TS8k7F+dEtNt6+79q7mce8XgmM0DwPnA2cAr7p5r+6+0v5hZK6AjdbO/7HEFC57uXka4dnSrmY00s5Zm1tTMhpvZL+Ng44FrzGxvM+sUh0/7DN6bwHFm1sPM2hFOtwAws33NbERcuZsJF7CzHfU8CRwYH69qYmZfI5zW/S1lnWqiDeH6XXk8Kr44o/9ywjW4mvgtMNPdv0m4xvU/2QYys4PM7EQza0a4Hlhxo6UqrxGO3MbGdTsUOB2YAODu78dpnQ+86O6fxOU4kxTB090/BGYSvghKzOyYOL9clgMd4/ZQHdOBdWZ2ZXyGs9jMBpjZkTWta8L/AFebWX8AM2tnZhWPNT0B9Dezr8Qj3x9QOUDm3KbdfSnhhuV/m1lbMysys/3N7PiqKhTHfQq4zcw6xHV3XGKQxwjXMS8l3FjLZTxwkZkdFrednwOvufuCqurQEBT0USV3/2/CNa9rCDdLFgLfI6wcCHcHZxKux70N/DOWpZnXs4QL4G8RriUmA15RrMcSwmnT8ewanHD3VcBphDv8qwgXz09z95Vp6lRDPwL+jXC96S7CsiSNAx6Ip2hfrWpiZjaCcNOuYjkvBw43s/OyDN4MuJFw8X8Z4dTv6izDVeLuWwjBa3gc9zbg6+7+bmKwFwmXLxYmuo2wrtM4DziGeDeb0E65Lke8S9jB58V265JtuMTw2wnr/zDC3emVwN2Eu9qpuPujhKP4CfFyzCxCexG3q7MJbb8KOIBwylwxbr5tGsI10xLCTcE1hOuT+1WzahcQrk++S7jefVlivhsJTxz0Bv6SZ9n+DvxnHHYp4QmPc6o5/3qv4s6aSKNkZn8C3nX36wpdl9pgZlMIN4nuLnA9rgUOdPfzC1mPQir4G0YitcnMjoynp0VmNozwKE2De3ulPovXxkcDdxa6LoWk4CmNTWfCXfty4BbgYnd/o6A1akTM7FuEy2tPufvUQtenkHTaLiKSgo48RURSUPCshqqy78SMUDV9TKhBMbMpZvbNAtdhj6a/M7PZ8XEqLLjPzNaY2XTbA+n84nx6xO2nuLanvacltwnbjdSNVocpB2tTgwyedZBKrEY8pNSaV+h6pGVmo8zspULXo9Dcvb+7T4mdgwlJObq5+0CvpXR+ZrbAzL6UmOdHcfvZnm+8+s6rmbox2xeg76GUg3tanQTPbK9gKh1XemrPOtGTkEdhfaErUhe0/aRQ1cvvhNfB/kJ4iH0VIXkphMB7DeHd1RWENw3aeeUEBaMJ2XGmZiuLwx5NyJa0Fvg/EinRyJIWixypxLLU+8vAG4S3chYSk1Fk1O/CWJeVwH8k+rcA7o/zfAf4MRkJJDLmtTMpRxzvVsLbIesIb9nsnxi2P/As4WH85cS0c1Qv/d4u6eWSZXHY0whvnqyN7XpovnVJyOazCdge23JtjmWcQsgwND226V+BvZL1yxh+ATHpQ2zPB2J7ziG8XJCvPXO10TgqJ8L4c1z2MsI21j/R79S47tYRXgf8USzvRHiYvOI98n/waQKSBYTkJ6Mz2uQnmcuYrS1j+f7A87FsJeG97/ax30NUTsM2ll0zIXUhJGtZTci29K3EPMeRJ11eju3yB4TMTyuBXyWWdRS1mx5vFJVTN+6yDgkvZSTT9P1fYtuqmE514kqu/XYg4aWaijfVfl1VfNudv6oCZzEhoN1MCFrNgcGJRi4lvBLYOm5IFTtwxUI+GMdrkaOsa1xxp8ZGOzl27x2nkyst1lDy7HyJYQ6J0z00NubIjPrdFevxecJbKBXp0G4k7FR7EXaSWfnmx67Bc1VckU0IO8+E2K8N4U2LK2JbtiFmUaJ66fcy08tlln0hbnBHxXV3ISEgNKtiXY4iseHnCZ6LCUlEWhHeGnm4msHzRsKO1gHoRngjJmt7VtFG46gcPL8R+1d88byZ6LcUGBI/d+DTvKW/ILwS2TT+DeHTp06Sda7UJsllrKIt+xK242ZxPU4FfpOtXTK2xYrgOZXwJlZzwptMHwMnJpY/a7q8PNvlC4TtuAchT2wy2NVmeryd7VWTdZgleFYnruTab18BLoifWxMzrBUqeB4TV94uOTeB54BLEt0HEb5RmiQWsk+WjSRZdmVFwyTKJhN2+nxpsYZSRfDMMs5vgJsz6tIt0X86cE78PA8Ylug3Jt/82DV43p3odyrhDRcIKcHeyDGNqtLvZUsvl1l2OzHgJsrmEl43zbcuR1G94HljortfnH9xtvVB5UBUKecl8M1c7VlFG40jd3q59nE9VBylfAR8m5g6LTHc9YSj5r5ZppGsc6U2oXLwzNmWWaY5Mrk85AmehC/q7UCbRP9fAPcnlj9rurw822VyO74EeC6xfLWZHm9ne9V0HVI5eFYnruTab6cSzhKyJiCv7b+qrnl2Bz707D8jkC1dXEUewQrZ0mcly3oCZ2ek+BpMCJw502JVh5kdZWYvmNnHZlZGSDeXmcItV/q4SqnA2DUNXVVyTbc7IUhmU1X6vWzp5TLLegJXZLRn9zidfOuyujLbpClVp8WDXdszX1q1fG20U0zKcaOZfRDfCV8Qe1XU50zCF9eHZvZiTBIC4dS1FHjGzOaZ2VXVqH+2OmZty5hkZoKZLY71epjqtRF8mlJvXaKsqpSHzau4Xpm5zrrk6Ae1lx6vWuswh+rElVz712hCIvJ3zWyGmZ2Wsg7VUlXwXAj0yLFysqWL20blfJSeZbxk2ULCkWcyHVcrd7+R/Gmxsk030x8J1466u3s7wqlaVSncKlRKBUZYttqwkNyZj6pKv1dVW1ZM/4aM9mzp7uPJvy6r056wa5tsJVx3qpQaLT52s3di2KWE0/Vs08mUr42S/o1wmvklQmKOXhWzB3D3Ge4+gnAZ5DHCtULcfZ27X+HufYAzgMvN7KRqzC+zjrna8ueE9jzEQ+rA86m83eVr6yWEbb5Nomx3Ux5mrrN829TupMfLnE6udVjVtladuJJ9wu7vu/u5hHV+E/CIhUxpe0RVwXM6odFuNLNWZtbczAbFfuOBH5pZbwt5IX9OSM9fkyObh4HTzeyUeCTRPD6G1M3zp8WqTiqxNoRv8U1mNpCws1XXREKasA5m1o1wXag2/A3Yz8wuM7NmZtbGzI6K/Woj/d5dwHfiUbfFdfbluDPmW5fLgW5mVlLF9M83s34W8mVeDzzi4RGb9whHQF82s6aEa2XNEuMl27MrIXNWLvnaKKkN4XrXKkLg/nlFDwvp6M4zs3buvpVwA2FH7HeamfWNO38Z4TS5Oun1kvK1ZRvCzZCyuKw/zhg3Z+pAD5mlXgZ+Ead5KOFoaneebf1xbPfuhBRymdm4knYnPV5SvnW4HOhllXO8JqWOK2Z2vpnt7SGJ9tpYXNN1W215g2fcMU4nXAT/iHCN42ux972Eu4dTCem5NlHDIBM3lhGEO3EVKel+nKhX1rRYXr1UYpcA15vZOkIgmliDqv2EcLown5AT8aGaLFcu8XTsZEKbLgPeB06IvXc7/Z67zyT8Js3vCXdLSwnXoqpal88T7twuM7N86fUeIlzTXUa4EfCDOO0yQnvfTThKWh+nX+H62D2f8KNsj5A7TVy+Nkp6kLCOFhPuqr+a0f8CYEE8df4OIVUdhLRufycEuFeA29z9hTzLnK2O+dryJ4Rcl2WEgJOZsu0XhC/JtWb2oyyTP5dwFL2E8BtE13lI7ZbWXwnp6t6M9bkn14C+G+nxMqaTbx1W/H79KjPLlnZwd+LKMGC2mZUTctWe4yF9XsWLLEOqOZ1q0bvtUufM7GLChn18oevSmJmZE355oLTQdWmMGuQbRtKwmNl+ZjbIQpq4gwiPsDxa6HqJ7A69VSB1oYTw42MVv4o5gfAso0iDpdN2EZEUdNouIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoODZAJjZMDOba2alZnZVoesj1Wdm95rZCjObVei6SO1S8KznzKwYuBUYDvQDzjWzfoWtldTA/cCwQldCap+CZ/03ECh193nuvgWYAIwocJ2kmtx9KrC60PWQ2qfgWf91BRYmuhfFMhEpIAVPEZEUFDzrv8VA90R3t1gmIgWk4Fn/zQAOMLPeZlYCnANMKnCdRD7zFDzrOXffBnwPmAzMASa6++zC1kqqy8zGA68AB5nZIjMbXeg6Se0wdy90HUREGhwdeYqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDg2UCY2ZhC10HS0/prfBQ8Gw7tfA2b1l8jo+ApIpJCo3lIvvVX728cC5LD1nlTaNpnaKGrsUdce/GQQldhj3vtbxM46rRzCl2NPWbsCftbmvGq2m/LJ45KNd26oCPPBqKxBs7PisYcOD+rFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFpMEys3vNbIWZzcoo/76ZvWtms83sl4nyq82s1MzmmtkpifJhsazUzK6qzryb1N5iiIjUTK8Du+7uJO4Hfg88WFFgZicAI4DPu/tmM9snlvcDzgH6A12Av5vZgXG0W4GTgUXADDOb5O7v5JuxgqeINFjuPtXMemUUXwzc6O6b4zArYvkIYEIsn29mpcDA2K/U3ecBmNmEOGze4KnTdhGpt8xsjJnNTPyNqcZoBwJDzOw1M3vRzI6M5V2BhYnhFsWyXOV56chTROotd78TuLOGozUB9gKOBo4EJppZn9qum4KniDQ2i4C/uLsD081sB9AJWAx0TwzXLZaRpzwnnbaLSGPzGHACQLwhVAKsBCYB55hZMzPrDRwATAdmAAeYWW8zKyHcVJpU1Ux05CkiDZaZjQeGAp3MbBFwHXAvcG98fGkLcGE8Cp1tZhMJN4K2Ad919+1xOt8DJgPFwL3uPruqeSt4ikiD5e7n5uh1fo7hbwBuyFL+JPBkTeat03YRkRR05FlAt108iOGHd+Pjsk0M/NFfARjQswO//dYxtG7elA8/Lmf0LVNZt3ErPfZuzes3j+T9JZ8AMOP9j7n0rldoUVLMQ5cPpc++bdm+YwdPvr6I6/74eiEX6zNp1nOPMXfaZHDnoMHDGHDSSP75+MPMfWkyzdu0A+CLIy6k+yFHsvidfzLjsfvZsW0rRU2aMvAr36DL5w4r8BJITSl4FtAfppRyx9NzuOu7Q3aW3frtQfzHQzN4ac5yLjihL5edMYCf/ukNAOYvW8exY3e9jn3L47OZOnsZTYuLeOLaUzj5sK48+2aVNwullqxevIC50yYz4qqbKSpuyuTf/Sc9DgnPXg84aSSH/MuZlYZv1rodJ19yHa3ad2T14gVMvuU/OfemhwpRddkN9ea03cw+Z2ZXmtkt8e9KMzu40PXak6bNWc6a8i2Vyvp2actLc5YD8PxbSxhxVM+809i4ZTtTZy8DYOv2Hbw5fxVdO7bcMxWWrMqWLWSfXgfRpKQ5RcXFdD5gAAvemJZz+E499qdV+44AdOjSk21bN7N969a6qq7UknoRPM3sSmACYIRHB6bHz+Or+5J+YzFn4VpOO7IHAP96dC+6dmy1s1/PfVoz7abTeXrcMI793D67jNuuZQnDj+jOlLeX1ll9JQTAZaWz2FT+Cdu2bGLhrJmsX7MSgHemPM5ffnoJUx+8mc3r1+0y7oJ/TqNTj74UN21a19WW3VRfTttHA/3dvdLXr5n9GpgN3JhtpPiq1hiAkiMupGmfoXu2lnXgktun8auLBnLlmYfy5MyFbNm2HYBlazZw8CWPsLp8M4f17siEH5/IkVc8xrqNocmKi4z7Lj2O25+aw4IV5YVchM+c9vv14NBTzubpW66hSUkzOnbvgxUVcfDxX+awL5+LYbw+6SFe+9+7Oe7rP9w53polHzLj0XsZdukuN3+lAagvwXMHIcvJhxnl+8V+WSVf3Wr91ft9j9WuDr23pIwRNzwLQN/92nLK4d0A2LJtB6vLNwPw5vxVzF++jr77teWNeasA+N23j+WDZZ9w25N5cxnIHnLQoFM4aFDIcDbzsftp2b4TLdp2+LT/4GE8c9u4nd3r16zk7//zU44fdQVt996vrqsrtaBenLYDlwHPmdlTZnZn/HsaeA64tMB1q1N7t20OgBmM/cqh3PPsXAA6tWlGkRkAvfZpzf77tWHB8nAaeO3XvkC7lk0Ze//0wlRa2PjJWgDKV69gwRsvs//AoWwoW72z/4dvvkyHLuH69eYN5Tzz++s48l8vYt++/QtSX9l99eLI092fjq9RDeTTbCaLgRkVbwA0RvddehxD+nWmY5vmzL39bG6Y+CatmzfhW6d8DoBJ0z/ioRdKARjUrzPXfPUwtm53duxwLr3rFdas30KXvVoy9szPM3fRWqbddAYAdzw9hweef79gy/VZ9NydN7C5/BOKiptw7LmX0Kxla6bc9ytWL5wHZrTpuC+Dzvs+EK6DfvLxEt54YjxvPDEegGE/+Bkt2rYv5CJIDVl4a6nhayyn7Z9F1148pOqBpF4be8L+lma8Adc8m3e/nfWzk1NNty7Ul9N2EZEGRcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCSFevFuu4h8Nn1u/46FrkJqOvIUEUlBwVNEJAUFTxGRFBQ8RaTBMrN7zWyFmc1KlP3KzN41s7fM7FEza5/od7WZlZrZXDM7JVE+LJaVVvd30xQ8RaQhux8YllH2LDDA3Q8F3gOuBjCzfsA5QP84zm1mVmxmxcCtwHCgH3BuHDYvBU8RabDcfSqwOqPsGXffFjtfBbrFzyOACe6+2d3nA6WEX68YCJS6+zx330L4Jd8RVc1bwVNEGrNvAE/Fz12BhYl+i2JZrvK8FDxFpN4yszFmNjPxN6YG4/4HsA34w56omx6SF5F6K/nz4jVhZqOA04CT/NMfalsMdE8M1i2Wkac8Jx15ikijYmbDgLHAGe6+IdFrEnCOmTUzs97AAcB0YAZwgJn1NrMSwk2lSVXNR0eeItJgmdl4YCjQycwWAdcR7q43A541M4BX3f077j7bzCYC7xBO579b8dPmZvY9YDJQDNzr7rOrmreCp4g0WO5+bpbie/IMfwNwQ5byJ4EnazJvnbaLiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKTQaLIqDRt+aKGrICldf/s/Cl0F2U1jT9i/0FWoc40meIpIwzOwV7tCVyE1nbaLiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkiDZaZ/dDMZpvZLDMbb2bNzay3mb1mZqVm9iczK4nDNovdpbF/r92Zt4KniDRIZtYV+AHwRXcfABQD5wA3ATe7e19gDTA6jjIaWBPLb47DpabgKSINWROghZk1AVoCS4ETgUdi/weAkfHziNhN7H+SmVnaGSt4ikiD5O6Lgf8CPiIEzTLgdWCtu2+Lgy0CusbPXYGFcdxtcfiOaeev4Cki9ZaZjTGzmYm/MYl+HQhHk72BLkArYFhd1U0p6USk3nL3O4E7c/T+EjDf3T8GMLO/AIOA9mbWJB5ddgMWx+EXA92BRfE0vx2wKm3ddOQpIg3VR8DRZtYyXrs8CXgHeAE4Kw5zIfDX+HlS7Cb2f97dPe3MFTxFpEFy99cIN37+CbxNiGd3AlcCl5tZKeGa5j1xlHuAjrH8cuCq3Zm/TttFpMFy9+uA6zKK5wEDswy7CTi7tuatI08RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFPSQvIgXzhc7tCl2F1HTkKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKCH5AvokkE9OKJ7O8o2bePyx+YA0LqkmB8O7c0+bUpYsW4Lv54yn/VbtnPGgH0Y0mcvAIqLjK7tmjN6/FuUb9nOYV3bctFR3SgyeO69VTz29vJCLtZnwm0XD2L44d34uGwTA38UfiLngcuO54Au4aHvdi1LKNuwhWPHTuKrg/tw2RkDdo47oEcHBl35OKVLy3jo8qH02bct23fs4MnXF3HdH18vyPJIzSl4FtALpat56t2P+f6QXjvLRh7ambeXruOxZ5Yz8pB9+ddD9+XhmUuYNGsFk2atAOCI7u04rf8+lG/ZTpHBN4/uzvWT32f1hq3cePpBzPyojEVlmwq0VJ8Nf5hSyh1Pz+Gu7w7ZWXbhb17c+fnnF3yRTzZsBWDiS/OY+NI8APp3b8/4H5/I2x+upkVJMbc8Ppups5fRtLiIJ649hXTryuQAAAV8SURBVJMP68qzby5G6j+dthfQnOXllG/eXqnsyB7tmFIafg11SukqjuzRfpfxBvfuwLR5qwHo26kVy9ZtZkX5FrbtcKbNW8ORPRruK28NxbQ5y1lTviVn/68c05s/T5u3S/lZg/vwvy/PB2Djlu1Mnb0MgK3bd/Dm/FV07dhyz1RYal29D55mdlGh61CX2jdvwtqN2wBYu3Eb7ZtXPjkoKTYO69aWVxesBWCvlk1Zuf7TnXjVhq3s1app3VVYdjHo4H1ZUbaRD5at26Xfmcf04s/T5u9S3q5lCcOP6M6Ut5fWRRWlFtT74An8JFcPMxtjZjPNbOa8KX+pyzrVmcwflf5ij/bMXb6e8i3bsw4vhXf2oN5ZA+QX+3Zi45btvLNwbaXy4iLjvkuP4/an5rBgRXldVVN2U7245mlmb+XqBeybazx3v5PwO82cdd8/U/94fX2ydtM22rcIR5/tWzShbNO2Sv0H9e7AS/NX7+xevWErnVqV7Ozu2LIpq9dvrbP6SmXFRcYZA3sy+KrHd+l31qDsp/K/+/axfLDsE2578p26qKLUkvpy5Lkv8HXg9Cx/qwpYrzo386MyhvbtCMDQvh2Z8VHZzn4tmxbRr3PrSmWlK9ezX9tm7NO6hCZFxqA+HZixsGyX6UrdOOGQLry3pIwlqzdUKjeDrxzTi0cyjkiv/doXaNeyKWPvn16X1ZRaUC+OPIG/Aa3d/c3MHmY2pe6rUzcuO74X/Tu3oU3zJtzx1QH86Y2lPPr2Mq4Y2puTDuzIx+Vb+PULn+5sA3u2563Fn7B5246dZTsc7n51Idf8S1+KzHj+/VUsWqs77XvafZcex5B+nenYpjlzbz+bGya+yYMvvB+PLnc9ZR98cGcWrdxQ6bS8y14tGXvm55m7aC3TbjoDgDuensMDz79fZ8sh6Zl7ozjbbTSn7Z9FTz+V66qNNBTlE0dZmvGenbMy73578sGdqpyumRUDM4HF7n6amfUGJgAdgdeBC9x9i5k1Ax4EjiCc0X7N3RekqTfUn9N2EZG0LgXmJLpvAm52977AGmB0LB8NrInlN8fhUlPwFJEGy8y6AV8G7o7dBpwIPBIHeQAYGT+PiN3E/ifF4VNR8BSReiv5OGL8G5MxyG+AsUDFjYCOwFp3r3hMZRHQNX7uCiwEiP3L4vCp1JcbRiIiu0g+jpjJzE4DVrj762Y2tE4rhoKniDRcg4AzzOxUoDnQFvgt0N7MmsSjy25ARbKAxUB3YJGZNQHasRuPQuq0XUQaJHe/2t27uXsv4BzgeXc/D3gBOCsOdiHw1/h5Uuwm9n/ed+NxIwVPEWlsrgQuN7NSwjXNe2L5PUDHWH45cNXuzESn7SLS4Ln7FGBK/DwPGJhlmE3A2bU1Tx15ioikoCNPESmY3h1bFboKqenIU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFpMEys2FmNtfMSs3sqrqct4KniDRIZlYM3AoMB/oB55pZv7qav4KniDRUA4FSd5/n7luACcCIupq5gqeINFRdgYWJ7kWxrE40qasZ7WmPXHS4FboOktJFhxe6BlIgffdpkXe/NbMxwJhE0Z3ufueerVX1NJrgKSKNTwyUuYLlYqB7ortbLKsTOm0XkYZqBnCAmfU2sxLgHGBSXc1cR54i0iC5+zYz+x4wGSgG7nX32XU1f3P3upqXiEijodN2EZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0Qkhf8HZsQL9InYS2EAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.DataFrame(\n",
    "    data=metrics.confusion_matrix(y_test, np.rint(y_pred_svm)), \n",
    "    \n",
    ")\n",
    "\n",
    "confusion_figure, confusion_axes = matplotlib.pyplot.subplots()\n",
    "confusion_figure.set_size_inches(5, 3)\n",
    "confusion_axes.set_title(\n",
    "    'Confusion matrix showing the frequency of \\n'\n",
    "    'correct and incorrect bug classification predictions.'\n",
    "    '\\n\\n'  # hack to avoid overlap with x-axis labels below\n",
    ")\n",
    "confusion_axes.xaxis.tick_top()  # move x-axis labels to top of matrix\n",
    "_ = sns.heatmap(\n",
    "    confusion_matrix,\n",
    "    annot=True,\n",
    "    fmt=\"d\",\n",
    "    cmap=sns.color_palette(\"Blues\"),\n",
    "    vmin=0,\n",
    "    ax=confusion_axes,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "tn, fp, fn, tp = metrics.confusion_matrix(y_test, np.rint(y_pred_svm)).flatten().tolist()\n",
    "\n",
    "fpr_svm = fp/(fp+tp)\n",
    "fnr_svm = fn/(fn+tn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='warn', n_jobs=None, penalty='l1',\n",
       "                   random_state=None, solver='warn', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logreg = LogisticRegression(penalty = 'l1')\n",
    "logreg.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy 0.6451444483118691\n"
     ]
    }
   ],
   "source": [
    "y_pred_lr = logreg.predict(x_test)\n",
    "print('Accuracy ' + str(logreg.score(x_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg_accuracy = logreg.score(x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAEFCAYAAABjDmX4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZxVdf3H8deHGRhAVsGNHQFRIH9JhhaSqJlgJlpqmvqTfubaZmguZYmWS7ZY/lILw1wTyUzJn4qmIO6IaQoBOiKyKjuyw8Dn98f3O3jmcu+dmcMwd+74fj4ePLj3e7bv+Z5z3/cs93zH3B0REamdJoWugIhIMVJ4ioikoPAUEUlB4SkikoLCU0QkBYWniEgKjTY8zayFmf3DzFab2V93Yj6nm9mTdVm3QjGzIWY2u47nOdrM7q3LeWZZxuNmdtauXEZiWXW2PnW1DzZWZjbYzN4xs7VmdkKh61NbBQ9PM/uGmU2LDbg4flAOq4NZnwTsBXRw95PTzsTd73P3L9VBfXYpM3Mz651vHHd/zt371led6oq7D3f3u+p6vmY21MwW1PV8E+pkH2zErgF+7+6t3P3hQlemtgoanmY2CvgtcB1hJ+sG3AqMqIPZdwfedveKOphX0TOz0kLX4RMo7z6obUJ3YEahK5GauxfkH9AWWAucnGecMkK4Lor/fguUxWFDgQXAxcASYDHwzTjsamAzsCUu42xgNHBvYt49AAdK4/uRwBxgDfAecHqi/PnEdJ8HXgVWx/8/nxg2GfgZ8EKcz5NAxxzrVln/SxP1PwE4FngbWAH8KDH+IOAlYFUc9/dAszhsSlyXdXF9v56Y/2XAB8A9lWVxml5xGQPj+07AUmBojvpeBiyM6zUbOCqWjwbGA3fHYTOAgxPTHRDbZVUcdnws7xnLmsT3twNLEtPdA1yUaNdvJbcH8CtgZdxWwxPT9YztsQb4J3BLcrsnxtsN2ABsi222NrZBdevTCfhbbKv3gO/laK9s++DIuG/cBCwHfh7H/R9gZlyfiUD3xHyOBmYR9rffA88m2mI0+ffptsBYwv6yEPg5UFLDdtwd+DPhc7cSeDiWTwe+khivKbAMOChHO5wDlBP2tQlAp1j+bmz7DbF9ygqVRakzrGALhmFAReWGzjHONcDLwJ7AHsCLwM/isKFx+mviBjwWWA+0z7Fj5dzR4gfpI6BvHLYP0D+5kyV2qJXAmXG60+L7DokP+bvAfkCL+P6GHOtWWf+fxvqfQ/hA/gVoDfSPO1bPOP5ngEPjcnsQPmwXJebnQO8s8/8F4UuoBYnwTOzY/wFaEj60v8pR177A/MSO3wPolWjXjbH9S4DrgZcTH6xy4EdAM+BIQiBVtvM84DPx9WzCl9cBiWEHJdo1GZ5bYt1LgAsIH3CLw18iBEIz4LC4XXcIz0QbLcgoy7c+TYDX4jZrBuwb63xMjvmPpuo+NzJuk+/G7diCcJZVTviSKQWuBF6M43eM7XVSbMsfxOlrGp5/B/5I2L/3BKYC59WwHf8PeABoH5d9eCy/FHggscwRwFs51v9IQrAOJOyD/wtMSQyfC3yxUBm00xlWsAXD6cAH1YzzLnBs4v0xwNzEjr+BRPgSjuAOzbFj5dzR4s61Cvga0CKjDiP5ODzPBKZmDH8JGBlfTwauTAy7EHgix7pV1r/ySKB1rM8hiXFeA07IMf1FwN8T77OF52ageUZZZlhMAN4C3iTHtz/QO7btF4GmGcNGA/9MvO8HbIivhxCOepskht8PjI6v7wFGAXsTwvNG4Hx2PCqdTNXwLE/Mr2Vc970Jl30qgJaJ4fdS+/DMtT6HAPMyxr8C+HOO+WfucyOzTP84cHbifRPCQUB34L+JwR2HGeFsotrwJFwG20RifyZ82U+qQTvuQzgqbJ9lnToRAr1NfP8gcGmO9R8L3Jh434oQ2D3i+7kUcXgW8prncqBjNdd9OgHvJ96/H8u2z8OrXk9aT9hAteLu6winuucDi83s/8xs/xrUp7JOnRPvP6hFfZa7+9b4ekP8/8PE8A2V05vZfmb2qJl9YGYfEa4Td8wzb4Cl7r6xmnFuBwYA/+vum7KN4O7lhLAeDSwxs3FmltwOmevcPG7XTsB8d9+WGJ5sr2cJAfYFwqn2ZODw+O+5jOmSti/P3dfHl63i8lYkyiAcMddWrvXpDnQys1WV/whH1XvVYt6Z9ekO/C4xvxWEkOxMbL/KET0kTk3XpzvhiHFxYt5/JByBVsrVjl0J7bgyc6buvohw6eFrZtYOGA7cl6MOVT4v7r6W8LnvnGP8olLI8HyJ8M2Y7ycKiwg7QaVusSyNdYRv10p7Jwe6+0R3P5rwrTuLECrV1aeyTgtT1qk2biPUq4+7tyF8aK2aaTzfQDNrRbiOPBYYbWa755yR+1/c/TDC+jvhckB1FgFdzSy5nyXb61nC0enQ+Pp5YDAhPJ+twfwzLQZ2N7Pkdu6aZ/y87ZPFfOA9d2+X+Nfa3Y+txTwylzmfcCqdnGcLd3+RsD7b629mRtX1ybdPzyd8vjom5tvG3fvXcD13j+GYzV3AGcDJwEvunmv/r/J5MbPdgA7Uz+dllytYeLr7asK1o1vM7AQza2lmTc1suJndGEe7H7jSzPYws45x/LS/wXsD+IKZdTOztoTTLQDMbC8zGxE37ibCBexsRz2PAfvFn1eVmtnXCad1j6asU220Jly/WxuPii/IGP4h4RpcbfwOmObu3yJc4/pDtpHMrK+ZHWlmZYTrgZU3WqrzCuHI7dK4bYcCXwHGAbj7O3FeZwDPuvtHcT2+RorwdPf3gWmEL4JmZva5uLxcPgQ6xP2hJqYCa8zssvgbzhIzG2Bmn61tXRP+AFxhZv0BzKytmVX+rOn/gP5m9tV45Ps9qgZkzn3a3RcTblj+2szamFkTM+tlZodXV6E47ePArWbWPm67LyRGeZhwHfP7hBtrudwPfNPMPh33neuAV9x9bnV1KAYF/amSu/+acM3rSsLNkvnAdwgbB8LdwWmE63FvAf+KZWmW9RThAvibhGuJycBrEuuxiHDadDg7hhPuvhw4jnCHfznh4vlx7r4sTZ1q6RLgG4TrTbcT1iVpNHBXPEU7pbqZmdkIwk27yvUcBQw0s9OzjF4G3EC4+P8B4dTviizjVeHumwnhNTxOeyvw3+4+KzHas4TLF/MT742wrdM4Hfgc8W42oZ1yXY6YRfiAz4nt1inbeInxtxK2/6cJd6eXAX8i3NVOxd3/TjiKHxcvx0wntBdxvzqZ0PbLgT6EU+bKafPt0xCumTYj3BRcSbg+uU8Nq3Ym4frkLML17osSy91A+MVBT+ChPOv2T+AncdzFhF94nFrD5Td4lXfWRBolM3sAmOXuVxW6LnXBzCYTbhL9qcD1+Cmwn7ufUch6FFLBnzASqUtm9tl4etrEzIYRfkpTdE+vNGTx2vjZwJhC16WQFJ7S2OxNuGu/FrgZuMDdXy9ojRoRMzuHcHntcXefUuj6FJJO20VEUtCRp4hICgrPGqiu953YI1RtfyZUVMxsspl9q8B12KXd35nZjPhzKiz4s5mtNLOptgu684vL6Rb3n5K6nveultwnbCe6brR67HKwLhVleNZDV2K14qFLrTmFrkdaZjbSzJ4vdD0Kzd37u/vk+PYwQqccXdx9kNdRd35mNtfMvphY5ry4/2zNN11D5zXsujHbF6Dvoi4Hd7V6Cc9sj2CqO6701J71ojuhH4V1ha5IfdD+k0J1D78THgd7iPAj9uWEzkshBO+VhGdXlxCeNGjrVTsoOJvQO86UbGVx3EMJvSWtAv5Noks0snSLRY6uxLLU+8vA64SncuYTO6PIqN9ZsS7LgB8nhrcA7ozL/A/wQzI6kMhY1vZOOeJ0txCeDllDeMqmV2Lc/sBThB/jf0jsdo6adb+3Q/dyybI47nGEJ09WxXY9MN+2JPTmsxHYGttyVY51nEzoYWhqbNNHgN2T9csYfy6x04fYnnfF9pxJeLggX3vmaqPRVO0I469x3VcT9rH+iWHHxm23hvA44CWxvCPhx+SVz5E/x8cdkMwldH5ydkabXJ25jtnaMpb3Ap6JZcsIz323i8PuoWo3bJeyY09InQidtawg9LZ0TmKZo8nTXV6O/fJ7hJ6flgG/TKzrSOq2e7yRVO26cYdtSHgoI9lN378T+1blfGqSK7k+t4MID9VUPqn2m+rybWf+VRecJYRAu4kQWs2BwxKNXE54JLBV3JEqP8CVK3l3nK5FjrLOccMdGxvt6Ph+jzifXN1iDSXPhy8xzqfifA+MjXlCRv1uj/X4L8JTKJXdod1A+FDtTviQTM+3PHYMz+VxQ5YSPjzj4rDWhCctLo5t2ZrYixI1634vs3u5zLKD4g53SNx2ZxECoayabTmSxI6fJzwXEjoR2Y3w1Mi9NQzPGwgftPZAF8ITMVnbs5o2Gk3V8PyfOLzyi+eNxLDFwJD4uj0f91t6PeGRyKbx3xA+/tVJss5V2iS5jtW0ZW/CflwWt+MU4LfZ2iVjX6wMzymEJ7GaE55kWgocmVj/rN3l5dkvJxH2426EfmKTYVeX3eNtb6/abMMs4VmTXMn1uX0JODO+bkXsYa1Q4fm5uPF26HMTeBq4MPG+L+EbpTSxkvtm2UmSZZdVNkyibCLhQ5+vW6yhVBOeWab5LXBTRl26JIZPBU6Nr+cAwxLDzs23PHYMzz8lhh1LeMIFQpdgr+eYR3Xd72XrXi6z7DZi4CbKZhMeN823LUdSs/C8IfG+X1x+SbbtQdUgqtLnJfCtXO1ZTRuNJnf3cu3idqg8SpkHnEfsOi0x3jWEo+beWeaRrHOVNqFqeOZsyyzzPCG5PuQJT8IX9VagdWL49cCdifXP2l1env0yuR9fCDydWL+67B5ve3vVdhtSNTxrkiu5PrdTCGcJWTsgr+t/1V3z7Aq879n/jEC27uIq+xGslK37rGRZd+DkjC6+DiMEZ85usWrCzA4xs0lmttTMVhO6m8vswi1X93FVugJjx27oqpNrvl0JIZlNdd3vZeteLrOsO3BxRnt2jfPJty1rKrNNmlJ9t3iwY3vm61YtXxttFzvluMHM3o3PhM+Ngyrr8zXCF9f7ZvZs7CQEwqlrOfCkmc0xs8trUP9sdczalrGTmXFmtjDW615q1kbwcZd6axJl1XV52Lya65WZ26xTjmFQd93j1Wgb5lCTXMn1+Tqb0BH5LDN71cyOS1mHGqkuPOcD3XJsnGzdxVVQtT9KzzJdsmw+4cgz2R3Xbu5+A/m7xco230x/IVw76urubQmnatV14VapSldghHWrC/PJ3fNRdd3vVdeWlfO/NqM9W7r7/eTfljVpT9ixTbYQrjtV6Rot/uxmj8S4iwmn69nmkylfGyV9g3Ca+UVCxxw9KhcP4O6vuvsIwmWQhwnXCnH3Ne5+sbvvCxwPjDKzo2qwvMw65mrL6wjt+SkPXQeeQdX9Ll9bLyLs860TZTvb5WHmNsu3T+1M93iZ88m1Davb12qSK9ln7P6Ou59G2Oa/AB600FPaLlFdeE4lNNoNZrabmTU3s8Fx2P3AD8ysp4V+Ia8jdM9fmyObe4GvmNkx8UiiefwZUhfP3y1WTboSa034Ft9oZoMIH7aaGk/oJqy9mXUhXBeqC48C+5jZRWZWZmatzeyQOKwuut+7HTg/HnVb3GZfjh/GfNvyQ6CLmTWrZv5nmFk/C/1lXgM86OEnNm8TjoC+bGZNCdfKyhLTJduzM6HnrFzytVFSa8L1ruWE4L6ucoCF7uhON7O27r6FcANhWxx2nJn1jh/+1YTT5Jp0r5eUry1bE26GrI7r+sOMaXN2HeihZ6kXgevjPA8kHE3tzG9bfxjbvSuhC7nM3riSdqZ7vKR82/BDoIdV7eM1KXWumNkZZraHh060V8Xi2m7bGssbnvGD8RXCRfB5hGscX4+D7yDcPZxC6J5rI7UMmbizjCDciavsku6HiXpl7RbLa9aV2IXANWa2hhBE42tRtasJpwvvEfpEvKc265VLPB07mtCmHwDvAEfEwTvd/Z67TyP8TZrfE+6WlhOuRVW3LZ8h3Ln9wMzyda93D+Ga7geEGwHfi/NeTWjvPxGOktbF+Ve6Jr5/j/BH2R4kdzdx+doo6W7CNlpIuKv+csbwM4G58dT5fEJXdRC6dfsnIeBeAm5190l51jlbHfO15dWEvi5XEwIns8u26wlfkqvM7JIssz+NcBS9iPA3iK7y0LVbWo8Quqt7I9ZnbK4RfSe6x8uYT75tWPn365ebWbZuB3cmV4YBM8xsLaGv2lM9dJ9X+SDLkBrOp0b0bLvUOzO7gLBjH17oujRmZuaEvzxQXui6NEZF+YSRFBcz28fMBlvoJq4v4Scsfy90vUR2hp4qkPrQjPDHxyr/KuY4wm8ZRYqWTttFRFLQabuISAoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIikoPEVEUlB4ioikoPAUEUlB4SkikoLCU0QkBYWniEgKCk8RkRQUniIiKSg8RURSUHiKiKSg8BQRSUHhKSKSgsJTRCQFhaeISAoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpKCwlNEJAWFZxEws2FmNtvMys3s8kLXR2rOzO4wsyVmNr3QdZG6pfBs4MysBLgFGA70A04zs36FrZXUwp3AsEJXQuqewrPhGwSUu/scd98MjANGFLhOUkPuPgVYUeh6SN1TeDZ8nYH5ifcLYpmIFJDCU0QkBYVnw7cQ6Jp43yWWiUgBKTwbvleBPmbW08yaAacCEwpcJ5FPPIVnA+fuFcB3gInATGC8u88obK2kpszsfuAloK+ZLTCzswtdJ6kb5u6FroOISNHRkaeISAoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwLBJmdm6h6yDpafs1PgrP4qEPX3HT9mtkFJ4iIik0mh/JtzrlzsaxIjlsmTOZpvsOLXQ1dolhww8sdBV2uTmTH2LfoV8tdDV2mQe/OdDSTFfd53bt+JGp5lsfdORZJBprcH5SNObg/KRSeIqIpKDwFBFJQeEpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIikoPEVEUlB4ioikoPAUEUlB4SkikoLCU0QkBYWniEgKCk8RkRQUniIiKSg8RURSUHiKiKSg8BQRSUHhKSKSgsJTRCQFhaeISAoKTxEpWmZ2h5ktMbPpGeXfNbNZZjbDzG5MlF9hZuVmNtvMjkmUD4tl5WZ2eU2WXVp3qyEiUu/uBH4P3F1ZYGZHACOA/3L3TWa2ZyzvB5wK9Ac6Af80s/3iZLcARwMLgFfNbIK7/yffghWeIlK03H2KmfXIKL4AuMHdN8VxlsTyEcC4WP6emZUDg+KwcnefA2Bm4+K4ecNTp+0i0tjsBwwxs1fM7Fkz+2ws7wzMT4y3IJblKs9LR54iUjA99sufUWZ2LnBuomiMu4+pZralwO7AocBngfFmtu/O1DPXQkREGqQYlNWFZaYFwEPu7sBUM9sGdAQWAl0T43WJZeQpz0mn7SLS2DwMHAEQbwg1A5YBE4BTzazMzHoCfYCpwKtAHzPraWbNCDeVJlS3EB15ikjRMrP7gaFARzNbAFwF3AHcEX++tBk4Kx6FzjCz8YQbQRXAt919a5zPd4CJQAlwh7vPqG7ZCk8RKVruflqOQWfkGP9a4Nos5Y8Bj9Vm2TptFxFJQeEpIpKCwlNEJAWFp4hICrphVEC3XjCY4QO7sHT1RgZd8ggAd110OH06tQWgbctmrF6/mc9fOoHSEuOW8wfz6Z4dKG1i/GXKu/z64bcAuHD4AYw8aj/M4M9Pv8Otj+V9qkzqwIWDu/GZrm1ZvbGCUQ/PBOBzPdpxyqf3oXO75lzxj9m8u3w9AK3KSrjkiH3p1bElk8uXM/blBdvnc/WwPrRr2ZTNFdsA+NmT5Xy0saL+V0hqTeFZQPdNLuePT8zk9m8P2V521m+f3f76ujMP5qP1WwA48dAelJWWcMglj9CiWQnTfnMif33hPVo1L2XkUftx+I8eZXPFNh7+0dE88dp85ny4pt7X55NkUvkKHp+1lO8O6bG9bN7KjfzymTmc9/luVcbdstUZ969FdGvfgq7tm+8wr5ufnbs9aKV4NJjTdjPb38wuM7Ob47/LzOyAQtdrV3ph5oesXLs55/Cvfq4nf31hzvb3LZuXUtLEaNGslM0VW1mzfjN9O7fl1fKlbNi8la3bnOdnfsDxh3Svj+p/os38cC1rN22tUrZw9UYWfbRph3E3VWxj1pJ1bN66rb6qJ/WgQYSnmV0GjAOM8Iv/qfH1/TXtW6+xGXzAXixZvYF3PwhHkH9/eS7rN1bw7pivM/PWk7j5HzNYuW4z/5m/is/vvxe7tyqjRbMSvnRQF7p02K3AtZfauHBId355/P6c9F97F7oqUgsN5bT9bKC/u29JFprZb4AZwA3ZJkp2GtDsM2fRdN+hu7aW9ejkwT356wvvbX9/cO892LptG73Pe4D2u5Ux8ZrhTHprEbMXruamR6bzyJVHs35jBW/NXcHWbV7Amktt/G7KXFas30Lz0ib88Mh9ObzX7jz77opCV0tqoEEceQLbCJ2TZtonDsvK3ce4+8HufnBjCs6SJsbxg7rztxc/Ds9TDuvJU28spGKrs/Sjjbw8ewkDe3UE4O5J7zDk8kc5ZvQTrFy3mfLFqwtVdamlFfGa9saKbTw3ZwW992hZ4BpJTTWU8LwIeNrMHjezMfHfE8DTwPcLXLd6d8SnOvH2otUsWvHxTYT5y9Zx+IB9AGhZVsqgPnswe2EIyT3ahJsQXTrsxohB3Rn//Hs7zlQanCYGrctKACgx+EzXtsxfubHAtZKaahCn7e7+ROz9ZBAfd0K6EHi18sH9xujP3/8CQ/rtTYfWzZl928lcO/4N7p70DidlnLIDjHliFn+48DBe/fUIzIx7Jr3DjHkrAbjv4iPYvXUZWyq2MWrsy6xen/smlNSNiw7vQf+9W9O6eSl/PGUAD7y+mLWbKjj70K60aV7KFUf3Yu6KDfz8yXIAbj2pPy2alVDaxBjUrR0/m1jO0nWbufJLfShtYjQxeHPxGv759rICr5nUlIXORopfq1PubBwr8gk0bPiBha6C7KQHvznQ0kw34Mqn8n5up//86FTzrQ8N5bRdRKSoKDxFRFJQeIqIpKDwFBFJQeEpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIikoPEVEUmgQz7aLyCfT/r06FLoKqenIU0QkBYWniBQtM7vDzJaY2fRE2S/NbJaZvWlmfzezdolhV5hZuZnNNrNjEuXDYll5Tf96hcJTRIrZncCwjLKngAHufiDwNnAFgJn1A04F+sdpbjWzEjMrAW4BhgP9gNPiuHkpPEWkaLn7FGBFRtmT7l7595tfBrrE1yOAce6+yd3fA8oJfQgPAsrdfY67byb8PbUR1S1b4Skijdn/AI/H152B+YlhC2JZrvK8FJ4i0mCZ2blmNi3x79xaTPtjoAK4b1fUTT9VEpEGy93HAGNqO52ZjQSOA47yj/9cxkKga2K0LrGMPOU56chTRBoVMxsGXAoc7+7rE4MmAKeaWZmZ9QT6AFOBV4E+ZtbTzJoRbipNqG45OvIUkaJlZvcDQ4GOZrYAuIpwd70MeMrMAF529/PdfYaZjQf+Qzid/3blH5g0s+8AE4ES4A53n1HdshWeIlK03P20LMVj84x/LXBtlvLHgMdqs2ydtouIpKDwFBFJQeEpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIikoPEVEUlB4ioikoPAUEUlB4SkikoLCU0QkBYWniEgKCk8RkRQUniIiKTSa/jx/esGQQldBUrrmtucKXQXZWd8cWOga1LtGE54iUnwG9Whb6CqkptN2EZEUFJ4iIikoPEVEUlB4ioikoPAUEUlB4SkikoLCU0QkBYWniEgKCk8RKVpm9gMzm2Fm083sfjNrbmY9zewVMys3swfMrFkctyy+L4/De+zMshWeIlKUzKwz8D3gYHcfAJQApwK/AG5y997ASuDsOMnZwMpYflMcLzWFp4gUs1KghZmVAi2BxcCRwINx+F3ACfH1iPieOPwoM7O0C1Z4ikhRcveFwK+AeYTQXA28Bqxy94o42gKgc3zdGZgfp62I43dIu3yFp4g0WGZ2rplNS/w7NzGsPeFosifQCdgNGFZfdVOvSiLSYLn7GGBMjsFfBN5z96UAZvYQMBhoZ2al8eiyC7Awjr8Q6AosiKf5bYHlaeumI08RKVbzgEPNrGW8dnkU8B9gEnBSHOcs4JH4ekJ8Txz+jLt72oUrPEWkKLn7K4QbP/8C3iLk2RjgMmCUmZUTrmmOjZOMBTrE8lHA5TuzfJ22i0jRcvergKsyiucAg7KMuxE4ua6WrSNPEZEUFJ4iIikoPEVEUlB4ioikoPAUEUlB4SkikoLCU0QkBYWniEgKCk8RkRT0hJGIFMxBe7ctdBVS05GniEgKCk8RkRQUniIiKSg8RURSUHiKiKSg8BQRSUHhKSKSgsJTRCQF/Ui+gZj+9MPMfmEiuNP3sGEMOOoEpv5tLPPefIUmpaW06bgPQ876AWUtW22fZu2KJfzt6vMZ+OXT+dSXvlbA2n/y3HrBYIYP7MLS1RsZdEn4+2IDurfnd+d8jlbNm/L+0rWcffMU1mzYwu6tyrh31FAG9u7IfZPLufiOV7bP5+TBPbnkxANxdxav3MC3/ncKy9dsKtRqSS3oyLMBWLFwLrNfmMiIy2/ixCtvYf5bU/loySI6HXAQX/3pbXz1J7fSZq/O/PuJ8VWme+Wvt9Ol/8EFqvUn232TyznhuqeqlN1y3mCuuu81DrnkEf4x9X0uOn4AABu3bOVnD7zOj++ZVmX8kibGjSMHcezVT3DoDycw/f0VnDfsgHpbB9k5Cs8GYPUH89mzR19KmzWnSUkJe/cZwNzXX6BLv4E0KSkBYM+e+7N+5bLt08x940Vad9yb9vt0K1S1P9FemPkhK9durlLWu1Mbnp/5IQDPvLmIEYd0B2D9pgpemr2EjZu3VhnfDMyMlmXhBLBNy2YsXrG+HmovdaHBh6eZfbPQddjV2nfqzgfl09m49iMqNm9k/vRprMSmQgAAAAUdSURBVEsEJcDbLz5JlwHhKHPLxg28OfFBDvryNwpRXclh5vxVHPfZ8GV24qE96Nxht7zjV2x1Lrr9JV751QjK/3gK+3duy13PvFMfVZU60ODDE7g61wAzO9fMppnZtFceHVefdapT7fbpxoHHnMwTN1/JEzf/hA5d98WafLxp3nhsHE2alNBr0BEA/OvR+xhw1Ak0bd6iUFWWLC687QXO+VJfnrvhOFq3aMrmiq15xy8tMb71pb4Mvuwf9D5vPNPnreSSEz9VT7WVndUgbhiZ2Zu5BgF75ZrO3ccQ/sg9N05613dB1epN38HH0HfwMQBMe/hOWrbrCMDbLz7FvLemcuwPrsPMAFg6dzZz//U8rz50B5s3rAMzSpo2o98RXylY/QXeXrSaEdeG66C992nDMQO75B3/wB67A/Deh2sAeOiluYwaofAsFg0iPAkBeQywMqPcgBfrvzr1b8NHq2jRph1rVyxh7usv8pXLfsOCGdN468kHOfbiGylt1nz7uMdd8svtr//1j3tpWtZCwdkA7NGmOUs/2ogZXPrVAxn71Oy84y9asZ79u7SjY+sylq3ZxJEHdmL2wlX1VFvZWQ0lPB8FWrn7G5kDzGxy/Ven/j095lo2rf2IJiWlfP60Cylr2YoXx93GtootPPG7HwOwZ8++DD79uwWuqQD8+ftfYEi/venQujmzbzuZa8e/QavmpZxzzP4ATJg6j3smlW8ff8bvT6J1y6Y0K23CcZ/txoifP8mshau5/sF/M/Hq4WzZuo15y9Zx/i3PF2qVipaZlQDTgIXufpyZ9QTGAR2A14Az3X2zmZUBdwOfAZYDX3f3uamX617UZ7vbFftp+yfZNbc9V+gqyE5aO36kpZnuqZnL8n5ujz6gY7XzNbNRwMFAmxie44GH3H2cmf0B+Le732ZmFwIHuvv5ZnYqcKK7fz1NvaE4bhiJiGRlZl2ALwN/iu8NOBJ4MI5yF3BCfD0ivicOP8oqbySkoPAUkWL2W+BSYFt83wFY5e4V8f0CoHN83RmYDxCHr47jp6LwFJEGK/lzxPjv3MSw44Al7v5aIerWUG4YiYjsIPlzxCwGA8eb2bFAc6AN8DugnZmVxqPLLsDCOP5CoCuwwMxKgbaEG0ep6MhTRIqSu1/h7l3cvQdwKvCMu58OTAJOiqOdBTwSX0+I74nDn/GduGOu8BSRxuYyYJSZlROuaY6N5WOBDrF8FHD5zixEp+0iUvTcfTIwOb6eAwzKMs5G4OS6WqaOPEVEUlB4ioikoNN2ESmYntV029eQ6chTRCQFhaeISAoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIikoPEVEUlB4ioikoPAUEUlB4SkikoLCU0QkBYWniEgKCk8RkRQUniIiKSg8RURSUHiKiKSg8BSRomVmw8xstpmVm9nl9blshaeIFCUzKwFuAYYD/YDTzKxffS1f4SkixWoQUO7uc9x9MzAOGFFfC1d4ikix6gzMT7xfEMvqRWl9LWhXu/SIXlboOkg6lx7Rq9BVkALpvWeLvJ9bMzsXODdRNMbdx+zaWtVMowlPEWl8YlDmCsuFQNfE+y6xrF7otF1EitWrQB8z62lmzYBTgQn1tXAdeYpIUXL3CjP7DjARKAHucPcZ9bV8c/f6WpaISKOh03YRkRQUniIiKSg8RURSUHiKiKSg8BQRSUHhKSKSgsJTRCQFhaeISAr/D5pf7xCUW1BZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.DataFrame(\n",
    "    data=metrics.confusion_matrix(y_test, np.rint(y_pred_lr)), \n",
    "    \n",
    ")\n",
    "\n",
    "confusion_figure, confusion_axes = matplotlib.pyplot.subplots()\n",
    "confusion_figure.set_size_inches(5, 3)\n",
    "confusion_axes.set_title(\n",
    "    'Confusion matrix showing the frequency of \\n'\n",
    "    'correct and incorrect bug classification predictions.'\n",
    "    '\\n\\n'  # hack to avoid overlap with x-axis labels below\n",
    ")\n",
    "confusion_axes.xaxis.tick_top()  # move x-axis labels to top of matrix\n",
    "_ = sns.heatmap(\n",
    "    confusion_matrix,\n",
    "    annot=True,\n",
    "    fmt=\"d\",\n",
    "    cmap=sns.color_palette(\"Blues\"),\n",
    "    vmin=0,\n",
    "    ax=confusion_axes,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "tn, fp, fn, tp = metrics.confusion_matrix(y_test, np.rint(y_pred_lr)).flatten().tolist()\n",
    "\n",
    "fpr_logreg = fp/(fp+tp)\n",
    "fnr_logreg = fn/(fn+tn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AdaBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.datasets import make_classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostClassifier(algorithm='SAMME.R', base_estimator=None, learning_rate=1.0,\n",
       "                   n_estimators=100, random_state=0)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_ab = AdaBoostClassifier(n_estimators=100, random_state=0)\n",
    "clf_ab.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_ab = clf_ab.predict(x_test)\n",
    "# y_pred = y_pred.reshape(-1,1)\n",
    "# y_pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of mislabeled points out of a total 5746 points : 2029\n",
      "Accuracy 0.6468847894187261\n"
     ]
    }
   ],
   "source": [
    "total_error = (y_test != y_pred_ab).sum()\n",
    "\n",
    "print(\"Number of mislabeled points out of a total %d points : %d\"\n",
    "       % (x_test.shape[0],total_error))\n",
    "print(\"Accuracy \" + str(1- (total_error/x_test.shape[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "ab_accuracy = 1-(total_error/x_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAEFCAYAAABjDmX4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZgU1bnH8e/LDKvAIIOK7Ci4gCGJCybiMmpMwA2TqNGoEaMSNcbrvtx4r8TrFnOvWa5LNG5Eo4RwFUniEk1EEkUBoxER1BFQVtlHEBAG3vvHOYM1TXfPTDFMzwy/z/PMM92nTlWdOlX1di1db5u7IyIiddOi0A0QEWmKFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJotsHTzNqa2R/NrMLM/rAN0znDzP5Sn20rFDM7zMzeredpjjKzR+tzmlnm8YyZnb0955GYV70tT31tg82VmQ0xs/fNbI2ZnVTo9tRVwYOnmX3XzKbFDlwUd5RD62HSJwO7AaXufkraibj779z96/XQnu3KzNzM+uWr4+5/d/e9G6pN9cXdh7n76PqerpmVmdn8+p5uQr1sg83YjcCd7t7e3ccXujF1VdDgaWaXA78AbiFsZL2Au4Hh9TD53sB77l5ZD9Nq8sysuNBt2AHl3Qa1TugNzCh0I1Jz94L8ASXAGuCUPHVaE4Lrwvj3C6B1HFYGzAeuAJYAi4Bz4rCfABuAjXEe5wKjgEcT0+4DOFAc348AZgOrgTnAGYnyfyTGOwSYClTE/4ckhk0E/gt4OU7nL0CXHMtW1f6rE+0/CTgWeA9YAfx7ov5gYDKwKta9E2gVh02Ky/JpXN7vJKZ/DbAYeKSqLI6zZ5zH/vF9N2ApUJajvdcAC+JyvQscHctHAWOB38ZhM4ADE+PtG/tlVRx2YizvG8taxPe/AZYkxnsEuDTRr+cl1wfw38DKuK6GJcbrG/tjNfACcFdyvSfq7QSsAzbHPlsT+6Cm5ekG/F/sqznAJTn6K9s2OCJuGz8HlgM3xbrfB2bG5XkO6J2YzjHALML2difwUqIvRpF/my4BHiBsLwuAm4CiWvZjZ+Ahwn63Ehgfy98GTkjUawksA76cox/OB8oJ29oEoFss/yD2/brYP60LFYtSx7CCzRiGApVVKzpHnRuBV4FdgV2AV4D/isPK4vg3xhV4LLAW2DnHhpVzQ4s70ifA3nHY7sDA5EaW2KBWAmfF8U6P70sTO/kHwF5A2/j+thzLVtX+/4ztP5+wQz4GdAAGxg2rb6x/APCVON8+hJ3t0sT0HOiXZfo/JXwItSURPBMb9jtAO8JO+9852ro3MC+x4fcB9kz06/rY/0XArcCriR2rHPh3oBVwFCEgVfXzR8AB8fW7hA+vfRPDvpzo12Tw3BjbXgRcSNjBLQ6fTAgIrYBD43rdKngm+mh+Rlm+5WkBvB7XWStgj9jmb+SY/iiqb3Mj4jr5UVyPbQlnWeWED5li4HrglVi/S+yvk2NfXhbHr23wfBK4l7B97wpMAX5Qy378M/B7YOc47yNi+dXA7xPzHA5Mz7H8RxEC6/6EbfB/gUmJ4XOBrxUqBm1zDCvYjOEMYHENdT4Ajk28/wYwN7HhryMRfAlHcF/JsWHl3NDixrUK+DbQNqMNI/g8eJ4FTMkYPhkYEV9PBK5PDLsIeDbHslW1v+pIoENsz8GJOq8DJ+UY/1LgycT7bMFzA9AmoywzWEwApgNvkePTH+gX+/ZrQMuMYaOAFxLvBwDr4uvDCEe9LRLDHwdGxdePAJcDXQnB83bgArY+Kp1I9eBZnpheu7jsXQmXfSqBdonhj1L34JlreQ4GPsqofx3wUI7pZ25zI7KM/wxwbuJ9C8JBQG/ge8TAHYcZ4WyixuBJuAz2GYntmfBh/2It+nF3wlHhzlmWqRshoHeM78cBV+dY/geA2xPv2xMCdp/4fi5NOHgW8prncqBLDdd9ugEfJt5/GMu2TMOrX09aS1hBdeLunxJOdS8AFpnZn81sn1q0p6pN3RPvF9ehPcvdfVN8vS7+/zgxfF3V+Ga2l5n9ycwWm9knhOvEXfJMG2Cpu6+voc5vgP2A/3X3z7JVcPdyQrAeBSwxszFmllwPmcvcJq7XbsA8d9+cGJ7sr5cIAexwwqn2ROCI+Pf3jPGStszP3dfGl+3j/FYkyiAcMddVruXpDXQzs1VVf4Sj6t3qMO3M9vQGfpmY3gpCkOxO7L+qih4iTm2XpzfhiHFRYtr3Eo5Aq+Tqx56EflyZOVF3X0i49PBtM+sEDAN+l6MN1fYXd19D2O+756jfpBQyeE4mfDLm+4rCQsJGUKVXLEvjU8Kna5WuyYHu/py7H0P41J1FCCo1taeqTQtStqku7iG0q7+7dyTstFbDOJ5voJm1J1xHfgAYZWadc07I/TF3P5Sw/E64HFCThUBPM0tuZ8n+eolwdFoWX/8DGEIIni/VYvqZFgGdzSy5nnvmqZ+3f7KYB8xx906Jvw7ufmwdppE5z3mEU+nkNNu6+yuE5dnSfjMzqi9Pvm16HmH/6pKYbkd3H1jL5ewcg2M2o4EzgVOAye6ea/uvtr+Y2U5AKQ2zv2x3BQue7l5BuHZ0l5mdZGbtzKylmQ0zs9tjtceB681sFzPrEuun/Q7em8DhZtbLzEoIp1sAmNluZjY8rtzPCBewsx31PA3sFb9eVWxm3yGc1v0pZZvqogPh+t2aeFR8YcbwjwnX4Oril8A0dz+PcI3r19kqmdneZnaUmbUmXA+sutFSk9cIR25Xx3VbBpwAjAFw9/fjtM4EXnL3T+JyfJsUwdPdPwSmET4IWpnZV+P8cvkYKI3bQ21MAVab2TXxO5xFZrafmR1U17Ym/Bq4zswGAphZiZlVfa3pz8BAM/tWPPK9hOoBMuc27e6LCDcs/8fMOppZCzPb08yOqKlBcdxngLvNbOe47g5PVBlPuI75b4Qba7k8DpxjZl+K284twGvuPremNjQFBf2qkrv/D+Ga1/WEmyXzgIsJKwfC3cFphOtx04F/xrI083qecAH8LcK1xGTAaxHbsZBw2nQEWwcn3H05cDzhDv9ywsXz4919WZo21dGVwHcJ15t+Q1iWpFHA6HiKdmpNEzOz4YSbdlXLeTmwv5mdkaV6a+A2wsX/xYRTv+uy1KvG3TcQgtewOO7dwPfcfVai2kuEyxfzEu+NsK7TOAP4KvFuNqGfcl2OmEXYwWfHfuuWrV6i/ibC+v8S4e70MuB+wl3tVNz9ScJR/Jh4OeZtQn8Rt6tTCH2/HOhPOGWuGjffNg3hmmkrwk3BlYTrk7vXsmlnEa5PziJc7740Md91hG8c9AWeyLNsLwD/EesuInzD47Razr/Rq7qzJtIsmdnvgVnufkOh21IfzGwi4SbR/QVux38Ce7n7mYVsRyEV/AkjkfpkZgfF09MWZjaU8FWaJvf0SmMWr42fC9xX6LYUkoKnNDddCXft1wC/Ai509zcK2qJmxMzOJ1xee8bdJxW6PYWk03YRkRR05CkikoKCZy3UlH0nZoSq69eEmhQzm2hm5xW4Dds1/Z2ZzYhfp8KCh8xspZlNse2Qzi/Op1fcforqe9rbW3KbsG1I3WgNmHKwPjXJ4NkAqcTqxENKrdmFbkdaZjbCzP5R6HYUmrsPdPeJ8e2hhKQcPdx9sNdTOj8zm2tmX0vM86O4/WzKN15j57VM3ZjtA9C3U8rB7a1Bgme2RzCVjis99WeD6E3Io/BpoRvSELT9pFDTw++Ex8GeIHyJfTkheSmEwHs94dnVJYQnDUq8eoKCcwnZcSZlK4t1v0LIlrQK+BeJlGhkSYtFjlRiWdp9HPAG4amcecRkFBntOzu2ZRnw48TwtsDDcZ7vAFeRkUAiY15bknLE8e4iPB2ymvCUzZ6JugOB5wlfxv+YmHaO2qXf2yq9XLIs1j2e8OTJqtivg/KtS0I2n/XAptiXq3Is40RChqEpsU+fAjon25dRfy4x6UPsz9GxP2cSHi7I15+5+mgU1RNh/CEuewVhGxuYGHZsXHerCY8DXhnLuxC+TF71HPnf+TwByVxC8pNzM/rkJ5nLmK0vY/mewN9i2TLCc9+d4rBHqJ6G7Wq2zoTUjZCsZQUh29L5iXmOIk+6vBzb5SWEzE/LgJ8llnUE9ZsebwTVUzdutQ4JD2Uk0/T9K7FtVU2nNnEl1347mPBQTdWTanfUFN+25a+mwFlECGg/JwStNsChiU4uJzwS2D5uSFU7cNVC/jaO1zZHWfe44o6NnXZMfL9LnE6utFhl5Nn5EnW+EKc7KHbmSRnt+01sxxcJT6FUpUO7jbBTdSbsJG/nmx9bB8/lcUUWE3aeMXFYB8KTFlfEvuxAzKJE7dLvZaaXyyz7ctzgDo7r7mxCQGhdw7ocQWLDzxM8FxCSiOxEeGrk0VoGz9sIO9rOQA/CEzFZ+7OGPhpF9eD5/Ti86oPnzcSwRcBh8fXOfJ639FbCI5Et499hfP6tk2Sbq/VJchlr6Mt+hO24dVyPk4BfZOuXjG2xKnhOIjyJ1YbwJNNS4KjE8mdNl5dnu3yRsB33IuSJTQa7+kyPt6W/6rIOswTP2sSVXPvtZOCs+Lo9McNaoYLnV+PK2yrnJvBX4KLE+70JnyjFiYXcI8tGkiy7pqpjEmXPEXb6fGmxyqgheGYZ5xfAzzPa0iMxfApwWnw9GxiaGDYy3/zYOnjenxh2LOEJFwgpwd7IMY2a0u9lSy+XWXYPMeAmyt4lPG6ab12OoHbB87bE+wFx/kXZ1gfVA1G1nJfAebn6s4Y+GkXu9HKd4nqoOkr5CPgBMXVaot6NhKPmflmmkWxztT6hevDM2ZdZpnlScnnIEzwJH9SbgA6J4bcCDyeWP2u6vDzbZXI7vgj4a2L56jM93pb+qus6pHrwrE1cybXfTiKcJWRNQF7ffzVd8+wJfOjZf0YgW7q4qjyCVbKlz0qW9QZOyUjxdSghcOZMi1UbZnawmb1oZkvNrIKQbi4zhVuu9HHVUoGxdRq6muSabk9CkMympvR72dLLZZb1Bq7I6M+ecTr51mVtZfZJS2pOiwdb92e+tGr5+miLmJTjNjP7ID4TPjcOqmrPtwkfXB+a2UsxSQiEU9dy4C9mNtvMrq1F+7O1MWtfxiQzY8xsQWzXo9Suj+DzlHqrE2U1pTxsU8P1ysx11i3HMKi/9Hi1Woc51Cau5Nq/ziUkIp9lZlPN7PiUbaiVmoLnPKBXjpWTLV1cJdXzUXqW8ZJl8whHnsl0XDu5+23kT4uVbbqZHiNcO+rp7iWEU7WaUrhVqZYKjLBs9WEeuTMf1ZR+r6a+rJr+zRn92c7dHyf/uqxNf8LWfbKRcN2pWmq0+LWbXRJ1FxFO17NNJ1O+Pkr6LuE082uExBx9qmYP4O5T3X044TLIeMK1Qtx9tbtf4e57ACcCl5vZ0bWYX2Ybc/XlLYT+/IKH1IFnUn27y9fXCwnbfIdE2bamPMxcZ/m2qW1Jj5c5nVzrsKZtrTZxJfuE3d9399MJ6/ynwDgLmdK2i5qC5xRCp91mZjuZWRszGxKHPQ5cZmZ9LeSFvIWQnr8uRzaPAieY2TfikUSb+DWkHp4/LVZtUol1IHyKrzezwYSdrbbGEtKE7WxmPQjXherDn4DdzexSM2ttZh3M7OA4rD7S7/0GuCAedVtcZ8fFnTHfuvwY6GFmrWqY/plmNsBCvswbgXEevmLzHuEI6Dgza0m4VtY6MV6yP7sTMmflkq+PkjoQrnctJwTuW6oGWEhHd4aZlbj7RsINhM1x2PFm1i/u/BWE0+TapNdLyteXHQg3Qyrisl6VMW7O1IEeMku9AtwapzmIcDS1Ld9tvSr2e09CCrnMbFxJ25IeLynfOvwY6GPVc7wmpY4rZnamme3iIYn2qlhc13Vba3mDZ9wxTiBcBP+IcI3jO3Hwg4S7h5MI6bnWU8cgEzeW4YQ7cVUp6a5KtCtrWiyvXSqxi4AbzWw1IRCNrUPTfkI4XZhDyIn4SF2WK5d4OnYMoU8XA+8DR8bB25x+z92nEX6T5k7C3dJywrWomtbl3wh3bhebWb70eo8QrukuJtwIuCROu4LQ3/cTjpI+jdOvcmN8P4fwo2zjyJ0mLl8fJf2WsI4WEO6qv5ox/Cxgbjx1voCQqg5CWrcXCAFuMnC3u7+YZ5mztTFfX/6EkOuyghBwMlO23Ur4kFxlZldmmfzphKPohYTfILrBQ2q3tJ4ipKt7M7bngVwVfRvS42VMJ986rPr9+uVmli3t4LbElaHADDNbQ8hVe5qH9HlVD7IcVsvp1IqebZcGZ2YXEjbsIwrdlubMzJzwywPlhW5Lc9QknzCSpsXMdjezIRbSxO1N+ArLk4Vul8i20FMF0hBaEX58rOpXMccQvsso0mTptF1EJAWdtouIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeDYBZjbUzN41s3Izu7bQ7ZHaM7MHzWyJmb1d6LZI/VLwbOTMrAi4CxgGDABON7MBhW2V1MHDwNBCN0Lqn4Jn4zcYKHf32e6+ARgDDC9wm6SW3H0SsKLQ7ZD6p+DZ+HUH5iXez49lIlJACp4iIikoeDZ+C4Ceifc9YpmIFJCCZ+M3FehvZn3NrBVwGjChwG0S2eEpeDZy7l4JXAw8B8wExrr7jMK2SmrLzB4HJgN7m9l8Mzu30G2S+mHuXug2iIg0OTryFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcGziTCzkYVug6Sn9df8KHg2Hdr5mjatv2ZGwVNEJIVm8yX59qc+3DwWJIeNsyfSco+yQjdjuxg6bFChm7DdzZ74BHuUfavQzdhuxp2zv6UZr6b9ds3YEamm2xB05NlENNfAuaNozoFzR6XgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIpIk2VmD5rZEjN7O6P8R2Y2y8xmmNntifLrzKzczN41s28kyofGsnIzu7Y28y6uv8UQEWlwDwN3Ar+tKjCzI4HhwBfd/TMz2zWWDwBOAwYC3YAXzGyvONpdwDHAfGCqmU1w93fyzVjBU0SaLHefZGZ9MoovBG5z989inSWxfDgwJpbPMbNyYHAcVu7uswHMbEysmzd46rRdRJqbvYDDzOw1M3vJzA6K5d2BeYl682NZrvK8FDxFpNEys5FmNi3xN7IWoxUDnYGvAFcBY83M6rttOm0XkUbL3e8D7qvjaPOBJ9zdgSlmthnoAiwAeibq9Yhl5CnPScFTRAqmz141nh2nMR44Engx3hBqBSwDJgCPmdkdhBtG/YEpgAH9zawvIWieBny3ppkoeIpIk2VmjwNlQBczmw/cADwIPBi/vrQBODsehc4ws7GEG0GVwA/dfVOczsXAc0AR8KC7z6hp3gqeItJkufvpOQadmaP+zcDNWcqfBp6uy7x1w0hEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFPRVpQK6+8IhDNu/B0sr1jP4yqcA+ELvzvzy/K/SplURlZs2c9n9r/L6B8votFMr7rlwCH1368D6jZu46J6XeWfeKgC+9sXu3H7OYIpaGKP/+j53PDW9kIu1Q7hoSC8O6FlCxfpKLh8/E4D2rYq4rKwvu3ZoxZLVG7hj4hw+3bCJbiWt+eGhvdmjtB2P/3MhE95eUm1aLQx+esI+rFi7kVtf+KAQiyMp6MizgH43sZyTbnm+WtlNZx7ArePe5JCrJ3DT2De46cwDAbjym4N4a+4KvnLVBEbe+Q9uHxGSwbQw445zD+ZbtzzPgZeN55Qhfdmne0mDL8uO5sXyFdz0fHm1spMGdWX6otX86P/eYfqi1Xxz0G4ArPlsEw++Nn+roFnl2AG7Mn/V+u3eZqlfjSZ4mtk+ZnaNmf0q/l1jZvsWul3b08szP2blmg3VytyhY9uWAJS0a8WilWsB2KdHCS+9vQiA9xZW0GuX9uxa0oYD+3Vh9uLVzF2yho2bNjPulTkcd1Cvhl2QHdDMj9ew5rNN1coO6lXCxPLlAEwsX85BvToB8Mn6Sj5YtpZNm32r6XRu15IDenTkr+8v2/6NlnrVKIKnmV0DjCE8YzqFz583fby2WZ2bi2tGT+Gmsw5k1t2ncPNZB3LDY68DMP3DlZx4cG8ADtizC712aU+3zjvRrXM75i//dMv4C5Z/SrfO7QrS9h1dpzbFrFpXCcCqdZV0alPzVbFzDu7BI9MW4FvHVWnkGkXwBM4FDnL329z90fh3GyFR6bm5Rkqmq9o4e2JDtXW7Ou/re3Pt6Knsc9EfuHb0VO6+YAgAd4yfTkm7Vrxy+4lcMGxf/jVnRdYjGWk8alo7B/ToSMW6SmYvX9cg7ZH61VhuGG0mZDn5MKN89zgsq2S6qvanPtwsIsl3j+jHVQ9NAeCJyXO58weHALB63UYuvOflLfVm3Hkyc5espm2rInqU7rSlvHvpTixcsbZB2yzBqvWVdGobjj47tS2mYn1l3vp779aeg3qVsH+PjrQsakG7VkVccngffjVpbsM0WLZJYwmelwJ/NbP3+Tyjcy+gH3BxwVpVAItXrOWwAV35+zuLKdtvdz5Y/AkQrn+u/aySjZs2M+Lo/rw8czGr123k9Q+WsefuHem9S3sWrljLyYf05fu/mlTgpdgxTfuogrJ+pYyf/jFl/UqZ+lFF3vqPvb6Qx15fCMDAru05cb/dFDibkEYRPN392Zh3bzCfp79fAEytShnVHD30b4dz2ICulHZow7v3nMLNY9/k4ntf4fZzBlPcogXrN27iR/dOBmDv7iXc+8NDcWDWvFVc9OtwFLpps3PFg68y/sfHUNTCeOTFcmbOX1XApdoxXHpEHwZ27UCHNsXce+p+/P6NRTw5fTFXlPXl6L1KWbpmA3e8OAeATm2L+ekJ+9C2ZRHuznEDduXSJ99h3cacJ1XSBJg3kyvVzeW0fUc0dNigQjdBttG4c/ZP9TMX+13/fN799u2bjqn3n8+oL43lhpGISJOi4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikkKjeLZdRHZM++xZWugmpKYjTxFpsszsQTNbYmZvJ8p+ZmazzOwtM3vSzDolhl1nZuVm9q6ZfSNRPjSWldc2AbuCp4g0ZQ8DQzPKngf2c/dBwHvAdQBmNgA4DRgYx7nbzIrMrAi4CxgGDABOj3XzUvAUkSbL3ScBKzLK/uLuVZmoXwV6xNfDgTHu/pm7zwHKCWkwBwPl7j7b3TcQfhJoeE3zVvAUkUYr+VM78W9kHSfxfeCZ+Lo7nydbB5gfy3KV56UbRiLSaCV/aqeuzOzHQCXwu3ptVKTgKSLNjpmNAI4HjvbPM74vAHomqvWIZeQpz0mn7SLSrJjZUOBq4ER3T/4a4gTgNDNrbWZ9gf6EnzmfCvQ3s75m1opwU2lCTfPRkaeINFlm9jhQBnQxs/nADYS7662B580M4FV3v8DdZ5jZWOAdwun8D6t+I83MLgaeA4qAB919Rk3zVvAUkSbL3U/PUvxAnvo3AzdnKX8aeLou89Zpu4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikkKzSYY8dNigQjdBUnr2mbcK3QTZVufsX+gWNLhmEzxFpOkZ3Kek0E1ITaftIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4g0WWZ2mZnNMLO3zexxM2tjZn3N7DUzKzez35tZq1i3dXxfHof32ZZ5K3iKSJNkZt2BS4AD3X0/oAg4Dfgp8HN37wesBM6No5wLrIzlP4/1UlPwFJGmrBhoa2bFQDtgEXAUMC4OHw2cFF8Pj++Jw482M0s7YwVPEWmS3H0B8N/AR4SgWQG8Dqxy98pYbT7QPb7uDsyL41bG+qVp56/gKSKNlpmNNLNpib+RiWE7E44m+wLdgJ2AoQ3VNiUGEZFGy93vA+7LMfhrwBx3XwpgZk8AQ4BOZlYcjy57AAti/QVAT2B+PM0vAZanbZuOPEWkqfoI+IqZtYvXLo8G3gFeBE6Odc4GnoqvJ8T3xOF/c3dPO3MFTxFpktz9NcKNn38C0wnx7D7gGuByMysnXNN8II7yAFAayy8Hrt2W+eu0XUSaLHe/Abgho3g2MDhL3fXAKfU1bx15ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCHs8UkYL5cteSQjchNR15ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKehL8gV00ZBeHNCzhIr1lVw+fiYA7VsVcVlZX3bt0Iolqzdwx8Q5fLphE91KWvPDQ3uzR2k7Hv/nQia8vWTLdO4+eSDrKjezebOz2Z1r/vhuoRZph3H3hUMYtn8PllasZ/CV4ccZv9C7M788/6u0aVVE5abNXHb/q7z+wTJOPXQPLh++H2bG6nUbufT+ybz94Ur6796R0ZeVbZlmn13bc9PYN7n76XcKtFRSFwqeBfRi+QqembWUHx3WZ0vZSYO6Mn3Rasb/5WNO+sJufHPQbjw6bSFrPtvEg6/NZ3CvTlmnNeqZ91j92aYGarn8bmI59z47k9/88LAtZTedeQC3jnuT599cwNe/3J2bzjyQYT95lg+XrGboqGdZ9ekGjvlSd/535CEc+eM/8/6iTzjk6gkAtDDj/XtP5Y9TPizUIkkd6bS9gGZ+vIY1GQHvoF4lTCxfDsDE8uUcFIPlJ+sr+WDZWjZtTv0z01KPXp75MSvXbKhW5gPNasQAAAVaSURBVA4d27YEoKRdKxatXAvAa+8tZdWnoe7U95fSvbTdVtMr+8LuzF78CfOWfbqdWy71pdEfeZrZOe7+UKHb0VA6tSlm1bpKAFatq6RTm5pXkQP/8Y3+uMPz7y7lhfeWb+dWSjbXjJ7C+B8fw81nHUSLFnD09U9vVed7R/XnL28s2Kr85CF9GffynIZoptSTpnDk+ZNcA8xspJlNM7Npsyc+0ZBtajC1Oc78j6ff4+oJs7j5+XKG7rsL++7Wfru3S7Z23tf35trRU9nnoj9w7eip3H3BkGrDDx/YlbOP7M9//u71auUti1pw3AE9efLVuQ3YWtlWjSJ4mtlbOf6mA7vlGs/d73P3A939wD3KvtWALd5+Vq2vpFPbcLTZqW0xFesraxxnxdqNQDi1n/JhBf132fq0ULa/7x7Rj6deC9csn5g8lwP6ddkybGCvnbnzB4fwnZ/9jRVrPqs23te/3J035yxnScX6hmyubKNGETwJAfJ7wAlZ/naoc9BpH1VQ1q8UgLJ+pUz9qCJv/dbFLWhT3GLL6y9278BHK7UTFsLiFWs5bEBXAMr2250PFn8CQI/SnXjsyiM5/86/U77ok63GO2XIHvxBp+xNTmO55vknoL27v5k5wMwmNnxzGsalR/RhYNcOdGhTzL2n7sfv31jEk9MXc0VZX47eq5SlazZwx4thp+rUtpifnrAPbVsW4e4cN2BXLn3yHTq0Lubqo/cAoMiMv89eyZsLtt5BpX499G+Hc9iArpR2aMO795zCzWPf5OJ7X+H2cwZT3KIF6zdu4kf3Tgbg2pO/SOf2rfn5eV8FoHLTZg6/7k8AtGtdzJGDdueS+14p2LI0dWZWBEwDFrj78WbWFxgDlAKvA2e5+wYzaw38FjiAcFD2HXefm3q+7s3j7u3JD/2zeSzIDujZZ94qdBNkG60ZO8LSjPf8zGV599tj9u1S43TN7HLgQKBjDJ5jgSfcfYyZ/Rr4l7vfY2YXAYPc/QIzOw34prt/J027ofGctouI1JmZ9QCOA+6P7w04ChgXq4wGToqvh8f3xOFHx/qpKHiKSKOV/EZN/BuZUeUXwNXA5vi+FFjl7lV3WucD3ePr7sA8gDi8ItZPpbFc8xQR2Yq73wfcl22YmR0PLHH3182srEEbhoKniDRdQ4ATzexYoA3QEfgl0MnMiuPRZQ+g6qmEBUBPYL6ZFQMlbMO3eXTaLiJNkrtf5+493L0PcBrwN3c/A3gRODlWOxt4Kr6eEN8Th//Nt+GOuYKniDQ31wCXm1k54ZrmA7H8AaA0ll8OXLstM9Fpu4g0ee4+EZgYX88GBmepsx44pb7mqSNPEZEUFDxFRFLQabuIFEzf0p0K3YTUdOQpIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniLSZJnZUDN718zKzezahpy3gqeINElmVgTcBQwDBgCnm9mAhpq/gqeINFWDgXJ3n+3uG4AxwPCGmrmCp4g0Vd2BeYn382NZgyhuqBltb+PO2d8K3QZJ6Zz9C90CKZB+u7bNu9+a2UhgZKLoPne/b/u2qnaaTfAUkeYnBspcwXIB0DPxvkcsaxA6bReRpmoq0N/M+ppZK+A0YEJDzVxHniLSJLl7pZldDDwHFAEPuvuMhpq/uXtDzUtEpNnQabuISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpLC/wNkuCRDQe+fNQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.DataFrame(\n",
    "    data=metrics.confusion_matrix(y_test, np.rint(y_pred_ab)), \n",
    "    \n",
    ")\n",
    "\n",
    "confusion_figure, confusion_axes = matplotlib.pyplot.subplots()\n",
    "confusion_figure.set_size_inches(5, 3)\n",
    "confusion_axes.set_title(\n",
    "    'Confusion matrix showing the frequency of \\n'\n",
    "    'correct and incorrect bug classification predictions.'\n",
    "    '\\n\\n'  # hack to avoid overlap with x-axis labels below\n",
    ")\n",
    "confusion_axes.xaxis.tick_top()  # move x-axis labels to top of matrix\n",
    "_ = sns.heatmap(\n",
    "    confusion_matrix,\n",
    "    annot=True,\n",
    "    fmt=\"d\",\n",
    "    cmap=sns.color_palette(\"Blues\"),\n",
    "    vmin=0,\n",
    "    ax=confusion_axes,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "tn, fp, fn, tp = metrics.confusion_matrix(y_test, np.rint(y_pred_ab)).flatten().tolist()\n",
    "\n",
    "fpr_ab = fp/(fp+tp)\n",
    "fnr_ab = fn/(fn+tn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting xgboost\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c1/24/5fe7237b2eca13ee0cfb100bec8c23f4e69ce9df852a64b0493d49dae4e0/xgboost-0.90-py2.py3-none-manylinux1_x86_64.whl (142.8MB)\n",
      "\u001b[K     |################################| 142.8MB 122kB/s eta 0:00:013    |##                              | 10.6MB 14.1MB/s eta 0:00:10     |##                              | 10.9MB 14.1MB/s eta 0:00:10     |##                              | 13.1MB 14.1MB/s eta 0:00:10     |###                             | 13.5MB 14.1MB/s eta 0:00:10     |###                             | 14.3MB 3.9MB/s eta 0:00:34     |########                        | 38.5MB 6.9MB/s eta 0:00:16     |########                        | 38.9MB 6.9MB/s eta 0:00:16     |#########                       | 44.1MB 8.8MB/s eta 0:00:12     |##########                      | 45.6MB 8.8MB/s eta 0:00:12     |##########                      | 47.1MB 8.8MB/s eta 0:00:11     |###########                     | 49.4MB 7.1MB/s eta 0:00:14     |############                    | 55.3MB 6.1MB/s eta 0:00:15     |############                    | 57.0MB 6.1MB/s eta 0:00:15     |#############                   | 59.8MB 6.1MB/s eta 0:00:14     |#############                   | 60.1MB 6.1MB/s eta 0:00:14     |#############                   | 61.3MB 6.1MB/s eta 0:00:14     |##############                  | 62.8MB 7.8MB/s eta 0:00:11     |##############                  | 65.9MB 54kB/s eta 0:23:40     |##############                  | 66.4MB 54kB/s eta 0:23:31     |###############                 | 67.4MB 54kB/s eta 0:23:13     |###############                 | 71.2MB 3.7MB/s eta 0:00:20     |################                | 73.2MB 3.7MB/s eta 0:00:19     |################                | 73.5MB 3.7MB/s eta 0:00:19     |################                | 73.9MB 3.7MB/s eta 0:00:19     |################                | 74.6MB 3.7MB/s eta 0:00:19     |################                | 75.6MB 3.7MB/s eta 0:00:19     |#################               | 78.2MB 8.0MB/s eta 0:00:09     |##################              | 82.0MB 8.0MB/s eta 0:00:08     |##################              | 82.9MB 8.0MB/s eta 0:00:08     |##################              | 83.3MB 8.0MB/s eta 0:00:08     |##################              | 84.2MB 8.0MB/s eta 0:00:08     |##################              | 84.6MB 6.7MB/s eta 0:00:09     |###################             | 88.9MB 6.7MB/s eta 0:00:09     |####################            | 92.1MB 6.7MB/s eta 0:00:08     |####################            | 93.0MB 8.3MB/s eta 0:00:07     |####################            | 93.4MB 8.3MB/s eta 0:00:06     |#####################           | 95.6MB 8.3MB/s eta 0:00:06     |#######################         | 104.6MB 11.4MB/s eta 0:00:04     |############################    | 125.6MB 9.7MB/s eta 0:00:02     |############################    | 129.3MB 9.7MB/s eta 0:00:02     |############################### | 140.6MB 8.4MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from xgboost) (1.16.4)\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from xgboost) (1.3.0)\n",
      "Installing collected packages: xgboost\n",
      "Successfully installed xgboost-0.90\n",
      "\u001b[33mWARNING: You are using pip version 19.1.1, however version 19.2.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6435781413156979"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "model_xgb=xgb.XGBClassifier(random_state=1,learning_rate=0.01)\n",
    "model_xgb.fit(x_train, y_train)\n",
    "model_xgb.score(x_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_xgb = model_xgb.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of mislabeled points out of a total 5746 points : 2048\n",
      "Accuracy 0.6435781413156978\n"
     ]
    }
   ],
   "source": [
    "total_error = (y_test != y_pred_xgb).sum()\n",
    "\n",
    "print(\"Number of mislabeled points out of a total %d points : %d\"\n",
    "       % (x_test.shape[0],total_error))\n",
    "print(\"Accuracy \" + str(1- (total_error/x_test.shape[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_accuracy = 1- (total_error/x_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAEFCAYAAABjDmX4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXhV1bnH8e9LQpgFAScIAjI4YG3FigNSRxSsim3VatVKL5VWa1urday9oq3Wtre17XVosc5aEb0O6HVWEGfBq1URkAgok8xEEDEE3vvHWsGdcM5Jsgk5OfH3eZ48OXvtae21937PHt9j7o6IiNRPi3xXQESkECl4ioikoOApIpKCgqeISAoKniIiKSh4ioik0GyDp5m1MbNHzKzczO7bgumcamZPNWTd8sXMhpjZzAae5hgzu6shp5lhHo+b2Rlbcx6JeTXY8jTUNthcmdlgM5tlZmvM7Ph816e+8h48zex7ZjY1NuCiuKMc1ACTPgHYAeji7iemnYi73+3uRzZAfbYqM3Mz65trGHd/wd13baw6NRR3H+7utzf0dM3sEDOb39DTTWiQbbAZuxK4zt3bu/tD+a5MfeU1eJrZecBfgKsJG9nOwA3AiAaYfE/gfXevbIBpFTwzK853Hb6Ecm6DWif0BKbluxKpuXte/oCOwBrgxBzDtCIE14Xx7y9Aq9jvEGA+cD6wBFgE/CD2uwKoANbHeYwCxgB3JabdC3CgOHaPBGYDq4E5wKmJ8hcT4x0ITAHK4/8DE/0mAb8BXorTeQrommXZqup/YaL+xwNHA+8DK4BLE8MPAl4BVsVhrwNKYr/JcVk+jcv73cT0LwI+Bu6sKovj9InzGBi7uwFLgUOy1PciYEFcrpnA4bF8DDAeuCP2mwZ8PTHe7rFdVsV+x8Xy3rGsRey+CViSGO9O4NxEu/4wuT6A/wJWxnU1PDFe79geq4FngOuT6z0xXDvgM2BjbLM1sQ1qW55uwP/EtpoD/CxLe2XaBkfGbeNaYDnw2zjsfwDT4/I8CfRMTGcoMIOwvV0HPJ9oizHk3qY7AjcTtpcFwG+Bojq2Y2fgVsJ+txJ4KJa/CxybGK4lsAzYO0s7nAmUEba1CUC3WP5BbPvPYvu0ylcsSh3D8jZjGAZUVq3oLMNcCbwKbA9sB7wM/Cb2OySOf2VcgUcDa4Fts2xYWTe0uCN9Auwa++0EDEhuZIkNaiVwehzvlNjdJbGTfwD0B9rE7muyLFtV/f8z1v9Mwg75L6ADMCBuWL3j8PsA+8f59iLsbOcmpudA3wzT/z3hS6gNieCZ2LDfA9oSdtr/ylLXXYF5iQ2/F9An0a7rYvsXAb8DXk3sWGXApUAJcBghIFW180fAPvHzTMKX1+6Jfnsn2jUZPNfHuhcBZxF2cIv9XyEEhBLgoLheNwueiTaaX6Ms1/K0AN6I66wE2CXW+ags0x9D9W1uZFwnP43rsQ3hLKuM8CVTDFwGvByH7xrb64TYlr+I49c1eD4I/IOwfW8PvA78qI7t+L/AvcC2cd4Hx/ILgXsT8xwBvJNl+Q8jBNaBhG3wv4HJif5zgSPyFYO2OIblbcZwKvBxLcN8AByd6D4KmJvY8D8jEXwJR3D7Z9mwsm5oceNaBXwHaFOjDiP5InieDrxeo/8rwMj4eRJwWaLf2cATWZatqv5VRwIdYn32SwzzBnB8lvHPBR5MdGcKnhVA6xplNYPFBOAd4G2yfPsDfWPbHgG0rNFvDPBMonsP4LP4eQjhqLdFov89wJj4+U7gPGBHQvD8A/BjNj8qnUT14FmWmF7buOw7Ei77VAJtE/3vov7BM9vy7Ad8VGP4S4Bbs0y/5jY3MsP4jwOjEt0tCAcBPYHvEwN37GeEs4lagyfhMtjnJLZnwpf9xDq0406Eo8JtMyxTN0JA3yZ23w9cmGX5bwb+kOhuTwjYvWL3XAo4eObzmudyoGst1326AR8muj+MZZum4dWvJ60lrKB6cfdPCae6PwYWmdn/mtludahPVZ26J7o/rkd9lrv7hvj5s/h/caL/Z1Xjm1l/M3vUzD42s08I14m75pg2wFJ3X1fLMDcBewL/7e6fZxrA3csIwXoMsMTMxplZcj3UXObWcb12A+a5+8ZE/2R7PU8IYN8gnGpPAg6Ofy/UGC9p0/zcfW382D7Ob0WiDMIRc31lW56eQDczW1X1Rziq3qEe065Zn57AXxPTW0EIkt2J7Vc1oIeIU9fl6Uk4YlyUmPY/CEegVbK1Yw9CO66sOVF3X0i49PAdM+sEDAfuzlKHavuLu68h7PfdswxfUPIZPF8hfDPmekRhIWEjqLJzLEvjU8K3a5Udkz3d/Ul3H0r41p1BCCq11aeqTgtS1qk+biTUq5+7b0PYaa2WcTxXTzNrT7iOfDMwxsw6Z52Q+7/c/SDC8jvhckBtFgI9zCy5nSXb63nC0ekh8fOLwGBC8Hy+DtOvaRHQ2cyS67lHjuFztk8G84A57t4p8dfB3Y+uxzRqznMe4VQ6Oc027v4yYXk21d/MjOrLk2ubnkfYv7ompruNuw+o43J2jsExk9uB04ATgVfcPdv2X21/MbN2QBcaZ3/Z6vIWPN29nHDt6HozO97M2ppZSzMbbmZ/iIPdA1xmZtuZWdc4fNpn8N4CvmFmO5tZR8LpFgBmtoOZjYgr93PCBexMRz2PAf3j41XFZvZdwmndoynrVB8dCNfv1sSj4rNq9F9MuAZXH38Fprr7DwnXuP6eaSAz29XMDjOzVoTrgVU3WmrzGuHI7cK4bg8BjgXGAbj7rDit04Dn3f2TuBzfIUXwdPcPgamEL4ISMzsgzi+bxUCXuD3UxevAajO7KD7DWWRme5rZvvWta8LfgUvMbACAmXU0s6rHmv4XGGBm345Hvj+jeoDMuk27+yLCDcs/mdk2ZtbCzPqY2cG1VSiO+zhwg5ltG9fdNxKDPES4jvlzwo21bO4BfmBmX4vbztXAa+4+t7Y6FIK8Pqrk7n8iXPO6jHCzZB5wDmHlQLg7OJVwPe4d4P9iWZp5PU24AP424VpiMuC1iPVYSDhtOpjNgxPuvhw4hnCHfznh4vkx7r4sTZ3q6ZfA9wjXm24iLEvSGOD2eIp2Um0TM7MRhJt2Vct5HjDQzE7NMHgr4BrCxf+PCad+l2QYrhp3ryAEr+Fx3BuA77v7jMRgzxMuX8xLdBthXadxKnAA8W42oZ2yXY6YQdjBZ8d265ZpuMTwGwjr/2uEu9PLgH8S7mqn4u4PEo7ix8XLMe8S2ou4XZ1IaPvlQD/CKXPVuLm2aQjXTEsINwVXEq5P7lTHqp1OuD45g3C9+9zEfD8jPHHQG3ggx7I9A/w6DruI8ITHyXWcf5NXdWdNpFkys3uBGe5+eb7r0hDMbBLhJtE/81yP/wT6u/tp+axHPuX9DSORhmRm+8bT0xZmNozwKE3Bvb3SlMVr46OAsfmuSz4peEpzsyPhrv0a4G/AWe7+Zl5r1IyY2ZmEy2uPu/vkfNcnn3TaLiKSgo48RURSUPCsg9qy78SMUPV9TKigmNkkM/thnuuwVdPfmdm0+DgVFtxqZivN7HXbCun84nx2jttPUUNPe2tLbhO2BakbrRFTDjakggyejZBKrF48pNSane96pGVmI83sxXzXI9/cfYC7T4qdBxGScpS6+yBvoHR+ZjbXzI5IzPOjuP1syDVeU+d1TN2Y6QvQt1LKwa2tUYJnplcwlY4rPbVno+hJyKPwab4r0hi0/aRQ28vvhNfBHiA8xL6ckLwUQuC9jPDu6hLCmwYdvXqCglGE7DiTM5XFYfcnZEtaBfybREo0MqTFIksqsQz1/ibwJuGtnHnEZBQ16ndGrMsy4FeJ/m2A2+I83wMuoEYCiRrz2pSUI453PeHtkNWEt2z6JIYdADxNeBh/MTHtHHVLv7dZerlkWRz2GMKbJ6tiu+6Va10SsvmsAzbEtlyVZRknETIMvR7b9GGgc7J+NYafS0z6ENvz9tie0wkvF+Rqz2xtNIbqiTDui8teTtjGBiT6HR3X3WrC64C/jOVdCQ+TV71H/gJfJCCZS0h+MqpGm1xRcxkztWUs7wM8F8uWEd777hT73Un1NGwXsnkmpG6EZC0rCNmWzkzMcww50uVl2S5/Rsj8tAz4Y2JZR9Kw6fFGUj1142brkPBSRjJN378T21bVdOoSV7Ltt4MIL9VUvan259ri25b81RY4iwgB7VpC0GoNHJRo5DLCK4Ht44ZUtQNXLeQdcbw2Wcq6xxV3dGy0obF7uzidbGmxDiHHzpcY5itxunvFxjy+Rv1uivX4KuEtlKp0aNcQdqrOhJ3k3VzzY/PguTyuyGLCzjMu9utAeNPi/NiWHYhZlKhb+r2a6eVqlu0dN7j94ro7gxAQWtWyLkeS2PBzBM8FhCQi7QhvjdxVx+B5DWFH2xYoJbwRk7E9a2mjMVQPnv8R+1d98byV6LcIGBI/b8sXeUt/R3glsmX8G8IXT50k61ytTZLLWEtb9iVsx63iepwM/CVTu9TYFquC52TCm1itCW8yLQUOSyx/xnR5ObbLiYTteGdCnthksGvI9Hib2qs+6zBD8KxLXMm2374CnB4/tydmWMtX8DwgrrzNcm4CzwJnJ7p3JXyjFCcWcpcMG0my7KKqhkmUPUnY6XOlxTqEWoJnhnH+Alxboy6lif6vAyfHz7OBYYl+o3PNj82D5z8T/Y4mvOECISXYm1mmUVv6vUzp5WqW3UgMuImymYTXTXOty5HULXhek+jeI86/KNP6oHogqpbzEvhhtvaspY3GkD29XKe4HqqOUj4CfkRMnZYY7krCUXPfDNNI1rlam1A9eGZtywzTPD65POQInoQv6g1Ah0T/3wG3JZY/Y7q8HNtlcjs+G3g2sXwNmR5vU3vVdx1SPXjWJa5k228nE84SMiYgb+i/2q559gA+9Mw/I5ApXVxVHsEqmdJnJct6AifWSPF1ECFwZk2LVRdmtp+ZTTSzpWZWTkg3VzOFW7b0cdVSgbF5GrraZJtuD0KQzKS29HuZ0svVLOsJnF+jPXvE6eRal3VVs01aUntaPNi8PXOlVcvVRpvEpBzXmNkH8Z3wubFXVX2+Q/ji+tDMno9JQiCcupYBT5nZbDO7uA71z1THjG0Zk8yMM7MFsV53Ubc2gi9S6q1OlNWW8rB1Ldcra66zbln6QcOlx6vTOsyiLnEl2/41ipCIfIaZTTGzY1LWoU5qC57zgJ2zrJxM6eIqqZ6P0jOMlyybRzjyTKbjaufu15A7LVam6db0L8K1ox7u3pFwqlZbCrcq1VKBEZatIcwje+aj2tLv1daWVdO/qkZ7tnX3e8i9LuvSnrB5m6wnXHeqlhotPnazXWLYRYTT9UzTqSlXGyV9j3CaeQQhMUevqtkDuPsUdx9BuAzyEOFaIe6+2t3Pd/ddgOOA88zs8DrMr2Yds7Xl1YT2/IqH1IGnUX27y9XWCwnbfIdE2ZamPKy5znJtU1uSHq/mdLKtw9q2tbrElcwTdp/l7qcQ1vnvgfstZErbKmoLnq8TGu0aM2tnZq3NbHDsdw/wCzPrbSEv5NWE9Pz1ObK5CzjWzI6KRxKt42NIpZ47LVZdUol1IHyLrzOzQYSdra7GE9KEbWtmpYTrQg3hUWAnMzvXzFqZWQcz2y/2a4j0ezcBP45H3RbX2TfjzphrXS4GSs2spJbpn2Zme1jIl3klcL+HR2zeJxwBfdPMWhKulbVKjJdsz+6EzFnZ5GqjpA6E613LCYH76qoeFtLRnWpmHd19PeEGwsbY7xgz6xt3/nLCaXJd0usl5WrLDoSbIeVxWS+oMW7W1IEeMku9DPwuTnMvwtHUljzbekFs9x6EFHI1s3ElbUl6vKRc63Ax0Muq53hNSh1XzOw0M9vOQxLtVbG4vuu2znIGz7hjHEu4CP4R4RrHd2PvWwh3DycT0nOto55BJm4sIwh34qpS0l2QqFfGtFhet1RiZwNXmtlqQiAaX4+qXUE4XZhDyIl4Z32WK5t4OjaU0KYfA7OAQ2PvLU6/5+5TCb9Jcx3hbmkZ4VpUbevyOcKd24/NLFd6vTsJ13Q/JtwI+Fmcdjmhvf9JOEr6NE6/ypWxew7hR9nuJ3uauFxtlHQHYR0tINxVf7VG/9OBufHU+ceEVHUQ0ro9QwhwrwA3uPvEHMucqY652vIKQq7LckLAqZmy7XeEL8lVZvbLDJM/hXAUvZDwG0SXe0jtltbDhHR1b8X63JxtQN+C9Hg1ppNrHVb9fv1yM8uUdnBL4sowYJqZrSHkqj3ZQ/q8qhdZhtRxOnWid9ul0ZnZWYQN++B816U5MzMn/PJAWb7r0hwV5BtGUljMbCczG2whTdyuhEdYHsx3vUS2hN4qkMZQQvjxsapfxRxHeJZRpGDptF1EJAWdtouIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeBYAMxtmZjPNrMzMLs53faTuzOwWM1tiZu/muy7SsBQ8mzgzKwKuB4YDewCnmNke+a2V1MNtwLB8V0IanoJn0zcIKHP32e5eAYwDRuS5TlJH7j4ZWJHvekjDU/Bs+roD8xLd82OZiOSRgqeISAoKnk3fAqBHors0lolIHil4Nn1TgH5m1tvMSoCTgQl5rpPIl56CZxPn7pXAOcCTwHRgvLtPy2+tpK7M7B7gFWBXM5tvZqPyXSdpGObu+a6DiEjB0ZGniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4FwsxG57sOkp7WX/Oj4Fk4tPMVNq2/ZkbBU0QkhWbzkHz7k25rHguSxfrZk2i5yyH5rsZWMWz4XvmuwlY3e9ID7HLIt/Ndja3m/h8MtDTj1bbfrhk/MtV0G4OOPAtEcw2cXxbNOXB+WSl4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikUJzvCojIl1ev/t3zXYXUdOQpIgXLzG4xsyVm9m6N8p+a2Qwzm2Zmf0iUX2JmZWY208yOSpQPi2VlZnZxXeatI08RKWS3AdcBd1QVmNmhwAjgq+7+uZltH8v3AE4GBgDdgGfMrH8c7XpgKDAfmGJmE9z9vVwzVvAUkYLl7pPNrFeN4rOAa9z98zjMklg+AhgXy+eYWRkwKPYrc/fZAGY2Lg6bM3jqtF1Empv+wBAze83MnjezfWN5d2BeYrj5sSxbeU4KniLSZJnZaDObmvgbXYfRioHOwP7ABcB4M7OGrptO20WkyXL3scDYeo42H3jA3R143cw2Al2BBUCPxHClsYwc5VnpyFNEmpuHgEMB4g2hEmAZMAE42cxamVlvoB/wOjAF6Gdmvc2shHBTaUJtM9GRp4gULDO7BzgE6Gpm84HLgVuAW+LjSxXAGfEodJqZjSfcCKoEfuLuG+J0zgGeBIqAW9x9Wm3zVvDMoxvOGszwgaUsLV/HoF8+DMDt5x5Mv24dAejYtoTytRUceGH4Ejz/+K/w/cP6sWGjc8Gtr/HsvxcCcPbw3Rl5eH/M4NZnZ3HDYzlvEkoDOHvwzuzToyPl6yo576HpABzQqxMnfW0nundqzSWPzOSD5WsBKG5hjD5wZ/p0bYu7c+tr85n28RoArhjWj05tW1JRuRGA3zxVxifrKvOzUAXI3U/J0uu0LMNfBVyVofwx4LH6zFvBM4/unlTGP56Yzk0/GbKp7Iy/PL/p89Wnf51P1q4HYLfuHTnhwN7se95D7LRtWx759ZF87ecPsltpR0Ye3p+DL32UisqNPHTpUJ54Yx6zF69u9OX5MplYtoLHZyzlp0N6bSr7aOU6/vjcbH504M7Vhj2ifxcAzn9oOtu0LuZXQ/ty8SMz8Nj/b8/P3RRopXA0mWueZrabmV1kZn+LfxeZ2e75rtfW9NL0xaxcU5G1/7cP6M19L80G4Jv77sz9L8+honIjHy5dw+yPV/P1vl3ZtXtHppQt5bOKDWzY6Lw4/WOO269nYy3Cl9b0xWtY8/mGamULytex8JPPNxu2tFMb3l0Uvsw+WVfJ2opK+nRt2yj1lK2nSQRPM7sIGAcY4QLu6/HzPXV9Vaq5Gbz7Diwp/4wPPg47XbfObZm//NNN/Res+JRundvy3rxVHLjbDnRu34o2JUUcuXcppV3a5avaksHcFWvZt0dHWhhs376EXbq0pUu7kk39zx7Skz8etxsnfHXHPNZS6qupnLaPAga4+/pkoZn9GZgGXJNppPjM12iAkn3OoOUuh2zdWjaiEwf35r6X5tQ63MwF5Vz78Ls8fNlQ1q6r5J25K9iw0WsdTxrPc7OWU9qpNb8/djeWfVrBzKWfstHDOvrr5LmsWLue1sUtuOCwXTi4T2ee/2BFnmssddFUgudGwrumH9Yo3yn2yyj5DFj7k25rNhGjqIVx3KCeHHTxI5vKFq5YW+2IsnvndixcEa6T3TFxFndMnAXA5acMZGHiCFXyb6PDba9/8djgVd/sz6LycHq/Il7TXle5kRdmr6Dvdm0VPAtEkzhtB84FnjWzx81sbPx7AngW+Hme69boDv1KN95fWL4pOAI8NnUeJxzYm5LiFvTcrj19dtqGqWXLANhum9YAlHZpx4hBPRn/Yu1HrNJ4SoqMVsVhV9urWwc2bHTml6+jhUGHVkUAFBns06Mj81auy2dVpR6axJGnuz8RH2YdxBfvlC4AplQ9h9Uc3frzbzBkjx3p0qE1M288kavGv8UdE2dxQoZT9unzV/HAK3OZ+ufjqdzonHfzq5tO/e4+/1A6d2jF+sqNnHfzq5SvzX4TShrGuQf3YsCOHejQuph/nLQn9765iDWfVzJq/x5s07qYS4b2Ye6Kz/jtU2V0bNOSy47sizusWFvB3yaHE6yWRS247Mh+FLcwWhi8vWg1z7y/LM9LJnVl7s3jbLc5nbZ/2Qwbvle+qyBb6P4fDEz17vielz2dc79997dDG/yd9IbSVE7bRUQKioKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKTeLddhH5ctqtT5d8VyE1HXmKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkiBcvMbjGzJWb2bqLsj2Y2w8zeNrMHzaxTot8lZlZmZjPN7KhE+bBYVlbXX+xV8BSRQnYbMKxG2dPAnu6+F/A+cAmAme0BnAwMiOPcYGZFZlYEXA8MB/YATonD5qTgKSIFy90nAytqlD3l7pWx81WgNH4eAYxz98/dfQ5QRvjdtEFAmbvPdvcKYFwcNicFTxFpssxstJlNTfyNruck/gN4PH7uDsxL9Jsfy7KV56Q3jESkyXL3scDYNOOa2a+ASuDuBq1UpOApIs2OmY0EjgEO9y9+IngB0CMxWGksI0d5VjptF5FmxcyGARcCx7n72kSvCcDJZtbKzHoD/YDXgSlAPzPrbWYlhJtKE2qbj448RaRgmdk9wCFAVzObD1xOuLveCnjazABedfcfu/s0MxsPvEc4nf+Ju2+I0zkHeBIoAm5x92m1zVvBU0QKlrufkqH45hzDXwVclaH8MeCx+sxbp+0iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSQrN5t/0/zxqS7ypISlfe+EK+qyBb6gcD812DRtdsgqeIFJ5BvTrmuwqp6bRdRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFJGCZWa/MLNpZvaumd1jZq3NrLeZvWZmZWZ2r5mVxGFbxe6y2L/XlsxbwVNECpKZdQd+Bnzd3fcEioCTgd8D17p7X2AlMCqOMgpYGcuvjcOlpuApIoWsGGhjZsVAW2ARcBhwf+x/O3B8/DwidhP7H25mlnbGCp4i0mSZ2Wgzm5r4G13Vz90XAP8FfEQImuXAG8Aqd6+Mg80HusfP3YF5cdzKOHyXtHVTPk8RabLcfSwwNlM/M9uWcDTZG1gF3AcMa6y66chTRArVEcAcd1/q7uuBB4DBQKd4Gg9QCiyInxcAPQBi/47A8rQzV/AUkUL1EbC/mbWN1y4PB94DJgInxGHOAB6OnyfEbmL/59zd085cwVNECpK7v0a48fN/wDuEeDYWuAg4z8zKCNc0b46j3Ax0ieXnARdvyfx1zVNECpa7Xw5cXqN4NjAow7DrgBMbat468hQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBT2qJCJ5s/eOHfNdhdR05CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSgh+SbiHefeZCZLz0JZnTu1oshZ/yCl+++jkWz3qGkTTsAvnHGL+jSow8fvvUKbzxyJ2YtaNGiBfud9CN27Dsgz0vw5XLDWYMZPrCUpeXrGPTLkKh8z57b8tczD6B965Z8uHQNo/42mdWfrae4yLj+x4P5Wu8uFLcw/jX5A/700DsAnD18d0Ye3h8zuPXZWdzw2Hv5XCypBwXPJuDTlcuYNnEC37n87xSXtOK5sVcze8rzAAz69ih673NQteG77fY1dv7q/pgZK+bP4bmbfscJV2T8jSzZSu6eVMY/npjOTT8Zsqns+h8N5ld3TuHF6Ys5/dC+nHvcnvzm3jf51v69aFVcxH6/fJg2JUVM/fO3uO+lObRvXczIw/tz8KWPUlG5kYcuHcoTb8xj9uLVeVwyqSudtjcRvnEDG9ZXsHHDBirXf07bTtl/EbVl6zZU/dz0+op1kP6npyWll6YvZuWaimplfbttw4vTFwPw3NsLGbFfz0392rYupqiF0aakmMzUeyIAAAXASURBVIrKDaxeW8Gu3TsypWwpn1VsYMNG58XpH3NcYhxp2pp88DSzH+S7Dltbu227sucR32bcpWdwz0WnUtK6HaV7DATgjQm388BvzubV8WPZsH79pnHmvvky918+mqeuu5wh3z83X1WXhOnzVnHMvjsD8K39e9G9S7jc8uCrc1m7rpIPxn6X6TecwN8emcbKTyt4b94qDtxtBzq3b0WbkiKO3LuU0jiONH1NPngCV2TrYWajzWyqmU197dFxjVmnBvX5p6v56O1XOem3t3LK7+9ifcU6yl57jq9/ayTfGTOWERf/lc/Xrubtp+7bNE6vvQ/khCvGcsRZv+b/JtyZx9pLlbNvfIkzj9yVF645hg5tWlJRuQGAr/fdjg0bN9L3R/ey5zn/w0+PHUCv7dszc0E51z78Lg9fNpSHLh3KO3NXsGFj6h9zlEbWJK55mtnb2XoBO2Qbz93HEn4tjz9M/KBgt7qFM96iQ5cdadMhZJjptfdgFn8wnb77HQZAUcuW9D9gKO888z+bjbtTv6/wwrJrWbemnNbtCzdDTXPw/sJyRlz1NAB9d9qGowaWAnDSQb15+q0FVG5wln6yjldnLmFgn67MXbKGOybO4o6JswC4/JSBLFz+ad7qL/XTVI48dwC+Dxyb4S/1j9IXinadt2PJnBlUVqzD3Vk44y067dSDteUrAHB3Pvz3K2zbrRcAnyxZSNXPTS/7qIwN69fTqt02+aq+RNtt0xoIl6Av/PZe3Pz0TADmLfuUg/fcCYC2rYoZ1G87Zi4orzZOaZd2jBjUk/EvzslDzSWNJnHkCTwKtHf3t2r2MLNJjV+dxrV9793oPfAgHrrqZ1hREV167MJuBw3nyet+zbrV5TjQpXQXBn/vHADmvPkSZa8+S4uiYopalnDomRdvuoEkjePWn3+DIXvsSJcOrZl544lcNf4t2rcu5syjdgNgwusfcefEMgDGPjGDv599EFP+NAIz486Js5j20UoA7j7/UDp3aMX6yo2cd/OrlK+tyDpPaVqs6gim0BXyafuX3ZU3vpDvKsgWWjN+ZKpv76enL8u53w7dvWut0zWzImAqsMDdjzGz3sA4oAvwBnC6u1eYWSvgDmAfwhntd919bpp6Q9M5bRcRSevnwPRE9++Ba929L7ASGBXLRwErY/m1cbjUFDxFpGCZWSnwTeCfsduAw4D74yC3A8fHzyNiN7H/4bYF17sUPEWkkP0FuBDYGLu7AKvcvTJ2zwe6x8/dgXkAsX95HD4VBU8RabKSz3LHv9GJfscAS9z9jXzUrancbRcR2UzyWe4MBgPHmdnRQGtgG+CvQCczK45Hl6XAgjj8AqAHMN/MioGObMGjkDryFJGC5O6XuHupu/cCTgaec/dTgYnACXGwM4CH4+cJsZvY/znfgseNFDxFpLm5CDjPzMoI1zRvjuU3A11i+XnAxVsyE522i0jBc/dJwKT4eTYwKMMw64ATG2qeOvIUEUlBR54ikje9CzgFn448RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNECpaZDTOzmWZWZmYXN+a8FTxFpCCZWRFwPTAc2AM4xcz2aKz5K3iKSKEaBJS5+2x3rwDGASMaa+YKniJSqLoD8xLd82NZoyhurBltbRce2sfyXQdJ58JD++S7CpInfbdvk3O/NbPRwOhE0Vh3H7t1a1U3zSZ4ikjzEwNltmC5AOiR6C6NZY1Cp+0iUqimAP3MrLeZlQAnAxMaa+Y68hSRguTulWZ2DvAkUATc4u7TGmv+5u6NNS8RkWZDp+0iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgK/w+AcDw6hOcNRAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.DataFrame(\n",
    "    data=metrics.confusion_matrix(y_test, np.rint(y_pred_xgb)), \n",
    "    \n",
    ")\n",
    "\n",
    "confusion_figure, confusion_axes = matplotlib.pyplot.subplots()\n",
    "confusion_figure.set_size_inches(5, 3)\n",
    "confusion_axes.set_title(\n",
    "    'Confusion matrix showing the frequency of \\n'\n",
    "    'correct and incorrect bug classification predictions.'\n",
    "    '\\n\\n'  # hack to avoid overlap with x-axis labels below\n",
    ")\n",
    "confusion_axes.xaxis.tick_top()  # move x-axis labels to top of matrix\n",
    "_ = sns.heatmap(\n",
    "    confusion_matrix,\n",
    "    annot=True,\n",
    "    fmt=\"d\",\n",
    "    cmap=sns.color_palette(\"Blues\"),\n",
    "    vmin=0,\n",
    "    ax=confusion_axes,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "tn, fp, fn, tp = metrics.confusion_matrix(y_test, np.rint(y_pred_xgb)).flatten().tolist()\n",
    "\n",
    "fpr_xgb = fp/(fp+tp)\n",
    "fnr_xgb = fn/(fn+tn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Table of Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'fpr_nn' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-78-eb6540da6036>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m results = {'Model': ['Neural Network', 'Random Forest', '1 Nearest Neighbour', '3 Nearest Neighbours', 'Gaussian Naive Bayes', 'SVM', 'Logistic Regression', 'AdaBoost', 'XGBoost'], \n\u001b[1;32m      2\u001b[0m            \u001b[0;34m'Accuracy'\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mnn_score\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrf_accuracy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moneknn_score\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mthreeknn_score\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgnb_score\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msvm_score\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogreg_accuracy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mab_accuracy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxgb_accuracy\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m           \u001b[0;34m'FPR'\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfpr_nn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfpr_rf\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfpr_onenn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfpr_threenn\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfpr_gnb\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfpr_svm\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfpr_logreg\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfpr_ab\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfpr_xgb\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m           'FNR' : [fnr_nn,fnr_rf,fnr_onenn,fnr_threenn,fnr_gnb,fnr_svm,fnr_logreg,fnr_ab,fnr_xgb]}\n\u001b[1;32m      5\u001b[0m \u001b[0maccuracy_table\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'fpr_nn' is not defined"
     ]
    }
   ],
   "source": [
    "results = {'Model': ['Neural Network', 'Random Forest', '1 Nearest Neighbour', '3 Nearest Neighbours', 'Gaussian Naive Bayes', 'SVM', 'Logistic Regression', 'AdaBoost', 'XGBoost'], \n",
    "           'Accuracy' : [nn_score, rf_accuracy, oneknn_score, threeknn_score, gnb_score, svm_score, logreg_accuracy, ab_accuracy, xgb_accuracy],\n",
    "          'FPR' : [fpr_nn,fpr_rf,fpr_onenn,fpr_threenn,fpr_gnb,fpr_svm,fpr_logreg,fpr_ab,fpr_xgb],\n",
    "          'FNR' : [fnr_nn,fnr_rf,fnr_onenn,fnr_threenn,fnr_gnb,fnr_svm,fnr_logreg,fnr_ab,fnr_xgb]}\n",
    "accuracy_table = pd.DataFrame(data = results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "accuracy_table.sort_values(by = ['Accuracy'], ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For T-SNE visualisations:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "true_labels = pd.DataFrame(y_test +0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_compare = pd.DataFrame()\n",
    "pred_compare['true_label'] = true_labels['bug']\n",
    "pred_compare['neuralnet'] = np.rint(y_predict).astype(int)\n",
    "pred_compare['rf'] = np.rint(y_pred_rf).astype(int)\n",
    "pred_compare['1nn'] = np.rint(y_pred_oneknn).astype(int)\n",
    "pred_compare['3nn'] = np.rint(y_pred_threeknn).astype(int)\n",
    "pred_compare['gnb'] = np.rint(y_pred_gnb).astype(int)\n",
    "pred_compare['svm'] = np.rint(y_pred_svm).astype(int)\n",
    "pred_compare['logreg'] = np.rint(y_pred_lr).astype(int)\n",
    "pred_compare['ab'] = np.rint(y_pred_ab).astype(int)\n",
    "pred_compare['xgb'] = np.rint(y_pred_xgb).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_compare.to_csv('../data/compare_predictions.csv.gz')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting vecstack\n",
      "  Downloading https://files.pythonhosted.org/packages/d0/a1/b9a1e9e9e5a12078da1ab9788c7885e4c745358f7e57d5f94d9db6a4e898/vecstack-0.4.0.tar.gz\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from vecstack) (1.16.4)\n",
      "Requirement already satisfied: scikit-learn>=0.18 in /usr/local/lib/python3.6/dist-packages (from vecstack) (0.21.2)\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from vecstack) (1.3.0)\n",
      "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.18->vecstack) (0.13.0)\n",
      "Building wheels for collected packages: vecstack\n",
      "  Building wheel for vecstack (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/5f/bb/4e/f6488433d53bc0684673d6845e5bf11a25240577c8151c140e\n",
      "Successfully built vecstack\n",
      "Installing collected packages: vecstack\n",
      "Successfully installed vecstack-0.4.0\n",
      "\u001b[33mWARNING: You are using pip version 19.1.1, however version 19.2.2 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install vecstack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "from vecstack import stacking"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model 1 : Stacking SVM and Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "models1 = [clf_svm, logreg]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "task:         [classification]\n",
      "n_classes:    [2]\n",
      "metric:       [accuracy_score]\n",
      "mode:         [oof_pred_bag]\n",
      "n_models:     [2]\n",
      "\n",
      "model  0:     [SVC]\n",
      "    fold  0:  [0.63557257]\n",
      "    fold  1:  [0.65158371]\n",
      "    fold  2:  [0.64891210]\n",
      "    fold  3:  [0.64560487]\n",
      "    ----\n",
      "    MEAN:     [0.64541831] + [0.00606614]\n",
      "    FULL:     [0.64541815]\n",
      "\n",
      "model  1:     [LogisticRegression]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    fold  0:  [0.63748695]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    fold  1:  [0.66063348]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    fold  2:  [0.65221932]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    fold  3:  [0.64664926]\n",
      "    ----\n",
      "    MEAN:     [0.64924725] + [0.00841923]\n",
      "    FULL:     [0.64924724]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "S_train, S_test = stacking(models1,                   \n",
    "                           x_train, y_train, x_test,   \n",
    "                           regression=False, \n",
    "     \n",
    "                           mode='oof_pred_bag', \n",
    "       \n",
    "                           needs_proba=False,\n",
    "         \n",
    "                           save_dir=None, \n",
    "            \n",
    "                           metric=accuracy_score, \n",
    "    \n",
    "                           n_folds=4, \n",
    "                 \n",
    "                           stratified=True,\n",
    "            \n",
    "                           shuffle=True,  \n",
    "            \n",
    "                           random_state=0,    \n",
    "         \n",
    "                           verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "level_two1 = clf_svm.fit(S_train, y_train)\n",
    "y_pred_stack1 = level_two1.predict(S_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final prediction score for model1: [0.64618865]\n"
     ]
    }
   ],
   "source": [
    "print('Final prediction score for model1: [%.8f]' % accuracy_score(y_test, np.rint(y_pred_stack1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAEFCAYAAABjDmX4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZxVdf3H8dcHhmFfBFRk3xTDpURFDRfcEk2FLAtTEzNJ/VmZmUtZkrll/dL6uRSpuUNkLmSWWoq4oIC5BCE6IjjDvivLyPb5/fH9Dp653Htn5jDMnRnez8eDB/d8z/ec8z3fc87nnuWez5i7IyIiNdOk0A0QEWmIFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJotMHTzFqa2V/NbLWZ/Xk75nOmmT1Tm20rFDM7wsxm1/I8x5jZg7U5zyzL+LuZnbMjl5FYVq2tT23tg42VmQ0xs/fMbI2ZjSh0e2qq4MHTzL5uZtNjBy6MB8rhtTDrrwC7A53c/fS0M3H3h9z9C7XQnh3KzNzM+uer4+4vuvuAumpTbXH3E939vtqer5kNNbOy2p5vQq3sg43YtcBt7t7G3R8vdGNqqqDB08wuBW4FbiDsZD2BO4DhtTD7XsC77r6pFubV4JlZUaHbsBPKuw9qm9ALmFnoRqTm7gX5B7QH1gCn56nTnBBcF8R/twLN47ihQBnwA2AJsBA4N477GbAB2BiXcR4wBngwMe/egANFcXgUMAf4GPgAODNR/lJius8D04DV8f/PJ8ZNAn4OvBzn8wzQOce6VbT/8kT7RwAnAe8CK4AfJeoPBqYAq2Ld24DiOG5yXJe1cX2/lpj/FcAi4IGKsjhNv7iMQXG4K7AUGJqjvVcA8+N6zQaOjeVjgAnA/XHcTOCgxHSfif2yKo47NZb3iWVN4vAfgCWJ6R4ALkn067eS2wP4FbAybqsTE9P1if3xMfBP4Pbkdk/Uaw2sB7bEPlsT+6Cq9ekK/CX21QfAd3P0V7Z9cFTcN24BlgPXxbrfBGbF9Xka6JWYz/HAO4T97TbghURfjCH/Pt0euJuwv8wHrgOaVrMfOwJ/JBx3K4HHY/kM4JREvWbAMuCAHP1wPlBC2NcmAl1j+fux79fH/mleqFiUOoYVbMEwDNhUsaFz1LkWeBXYDdgVeAX4eRw3NE5/bdyAJwHrgF1y7Fg5d7R4IH0EDIjj9gD2Se5kiR1qJXB2nO6MONwpcZC/D+wFtIzDN+VYt4r2/zS2/3zCAfkw0BbYJ+5YfWL9A4FD43J7Ew62SxLzc6B/lvn/gvAl1JJE8Ezs2P8FWhEO2l/laOsAoDSx4/cG+iX6tTz2f1PgRuDVxIFVAvwIKAaOIQSkin7+EDgwfp5N+PL6TGLcAYl+TQbPjbHtTYELCQe4xfFTCAGhGDg8btdtgmeij8oyyvKtTxPg9bjNioG+sc0n5Jj/GCrvc6PiNvlO3I4tCVdZJYQvmSLgauCVWL9z7K+vxL78fpy+usHzMeD3hP17N2Aq8O1q9uPfgD8Bu8RlHxXLLwf+lFjmcOA/Odb/GEJgHUTYB/8PmJwYPxc4rlAxaLtjWMEWDGcCi6qo8z5wUmL4BGBuYsdfTyL4Es7gDs2xY+Xc0eLOtQr4MtAyow2j+DR4ng1MzRg/BRgVP08Crk6Muwj4R451q2h/xZlA29ieQxJ1XgdG5Jj+EuCxxHC24LkBaJFRlhksJgL/Ad4mx7c/0D/27XFAs4xxY4B/JoYHAuvj5yMIZ71NEuPHAWPi5weAS4EuhOB5M3AB256VTqJy8CxJzK9VXPcuhNs+m4BWifEPUvPgmWt9DgE+zKh/FfDHHPPP3OdGZZn+78B5ieEmhJOAXsA3iIE7jjPC1USVwZNwG+wTEvsz4cv++Wr04x6Es8JdsqxTV0JAbxeHHwEuz7H+dwM3J4bbEAJ27zg8lwYcPAt5z3M50LmK+z5dgXmJ4XmxbOs8vPL9pHWEDVQj7r6WcKl7AbDQzP5mZntXoz0VbeqWGF5Ug/Ysd/fN8fP6+P/ixPj1FdOb2V5m9qSZLTKzjwj3iTvnmTfAUncvr6LOH4B9gf9z90+yVXD3EkKwHgMsMbPxZpbcDpnr3CJu165AqbtvSYxP9tcLhAB2JOFSexJwVPz3YsZ0SVuX5+7r4sc2cXkrEmUQzphrKtf69AK6mtmqin+Es+rdazDvzPb0An6TmN8KQpDsRuy/iooeIk5116cX4YxxYWLevyecgVbI1Y89CP24MnOm7r6AcOvhy2bWATgReChHGyodL+6+hnDcd8tRv0EpZPCcQvhmzPcThQWEnaBCz1iWxlrCt2uFLsmR7v60ux9P+NZ9hxBUqmpPRZvmp2xTTdxJaNee7t6OcNBaFdN4vpFm1oZwH/luYIyZdcw5I/eH3f1wwvo74XZAVRYAPcwsuZ8l++sFwtnp0Pj5JWAIIXi+UI35Z1oIdDSz5Hbukad+3v7JohT4wN07JP61dfeTajCPzGWWEi6lk/Ns6e6vENZna/vNzKi8Pvn26VLC8dU5Md927r5PNdezYwyO2dwHnAWcDkxx91z7f6XjxcxaA52om+NlhytY8HT31YR7R7eb2Qgza2VmzczsRDO7OVYbB1xtZruaWedYP+1v8N4EjjSznmbWnnC5BYCZ7W5mw+PG/YRwAzvbWc9TwF7x51VFZvY1wmXdkynbVBNtCffv1sSz4gszxi8m3IOrid8A0939W4R7XL/LVsnMBpjZMWbWnHA/sOJBS1VeI5y5XR637VDgFGA8gLu/F+d1FvCCu38U1+PLpAie7j4PmE74Iig2s8Pi8nJZDHSK+0N1TAU+NrMr4m84m5rZvmZ2cE3bmvA74Coz2wfAzNqbWcXPmv4G7GNmp8Uz3+9SOUDm3KfdfSHhgeX/mlk7M2tiZv3M7KiqGhSn/Ttwh5ntErfdkYkqjxPuY36P8GAtl3HAuWb2ubjv3AC85u5zq2pDQ1DQnyq5+/8S7nldTXhYUgpcTNg4EJ4OTifcj/sP8O9YlmZZzxJugL9NuJeYDHhNYjsWEC6bjmLb4IS7LwdOJjzhX064eX6yuy9L06Yaugz4OuF+0x8I65I0BrgvXqJ9taqZmdlwwkO7ivW8FBhkZmdmqd4cuIlw838R4dLvqiz1KnH3DYTgdWKc9g7gG+7+TqLaC4TbF6WJYSNs6zTOBA4jPs0m9FOu2xHvEA7wObHfumarl6i/mbD9P0d4Or0MuIvwVDsVd3+McBY/Pt6OmUHoL+J+dTqh75cDexIumSumzbdPQ7hnWkx4KLiScH9yj2o27WzC/cl3CPe7L0ksdz3hFwd9gEfzrNs/gZ/EugsJv/AYWc3l13sVT9ZEGiUz+xPwjrtfU+i21AYzm0R4SHRXgdvxU2Avdz+rkO0opIK/YSRSm8zs4Hh52sTMhhF+StPg3l6pz+K98fOAsYVuSyEpeEpj04Xw1H4N8FvgQnd/o6AtakTM7HzC7bW/u/vkQrenkHTZLiKSgs48RURSUPCshqqy78SMUDX9mVCDYmaTzOxbBW7DDk1/Z2Yz48+psOCPZrbSzKbaDkjnF5fTM+4/TWt73jtacp+w7UjdaHWYcrA2NcjgWQepxGrEQ0qtOYVuR1pmNsrMXip0OwrN3fdx90lx8HBCUo7u7j7Yaymdn5nNNbPjEsv8MO4/m/NNV995NVM3ZvsC9B2UcnBHq5Pgme0VTKXjSk/9WSd6EfIorC10Q+qC9p8Uqnr5nfA62KOEH7EvJyQvhRB4rya8u7qE8KZBe6+coOA8QnacydnKYt1DCdmSVgFvkUiJRpa0WORIJZal3V8E3iC8lVNKTEaR0b5zYluWAT9OjG8J3BuX+V/gh2QkkMhY1takHHG62wlvh3xMeMumX6LuPsCzhB/jLyamnaN66fe2SS+XLIt1Tya8ebIq9uv++bYlIZtPObA59uWqHOs4iZBhaGrs0yeAjsn2ZdSfS0z6EPvzvtifswgvF+Trz1x9NIbKiTD+HNd9NWEf2ycx7qS47T4mvA54WSzvTPgxecV75C/yaQKSuYTkJ+dl9MnPMtcxW1/G8n7Ac7FsGeG97w5x3ANUTsN2OdtmQupKSNaygpBt6fzEMseQJ11ejv3yu4TMT8uAXybWdRS1mx5vFJVTN26zDQkvZSTT9L2V2Lcq5lOduJLruB1MeKmm4k21X1cV37bnX1WBsykhoN1CCFotgMMTnVxCeCWwTdyRKg7gipW8P07XMkdZt7jhToqddnwc3jXOJ1darKHkOfgSdfaL890/duaIjPb9Ibbjs4S3UCrSod1EOKg6Eg6SGfmWx7bBc3nckEWEg2d8HNeW8KbFD2JftiVmUaJ66fcy08tllh0Qd7hD4rY7hxAQmlexLUeR2PHzBM/5hCQirQlvjTxYzeB5E+FA2wXoTngjJmt/VtFHY6gcPL8Zx1d88byZGLcQOCJ+3oVP85beSHglsln8dwSf/uok2eZKfZJcxyr6sj9hP24et+Nk4NZs/ZKxL1YEz8mEN7FaEN5kWgock1j/rOny8uyXzxP2456EPLHJYFeb6fG29ldNtmGW4FmduJLruJ0CnB0/tyFmWCtU8Dwsbrxtcm4C/wIuSgwPIHyjFCVWsm+WnSRZdkVFxyTKniYc9PnSYg2liuCZZZpbgVsy2tI9MX4qMDJ+ngMMS4wbnW95bBs870qMO4nwhguElGBv5JhHVen3sqWXyyy7kxhwE2WzCa+b5tuWo6he8LwpMTwwLr9ptu1B5UBUKecl8K1c/VlFH40hd3q5DnE7VJylfAh8m5g6LVHvWsJZc/8s80i2uVKfUDl45uzLLPMckVwf8gRPwhf1ZqBtYvyNwL2J9c+aLi/Pfpncjy8C/pVYv9pMj7e1v2q6DakcPKsTV3Idt5MJVwlZE5DX9r+q7nn2AOZ59j8jkC1dXEUewQrZ0mcly3oBp2ek+DqcEDhzpsWqDjM7xMyeN7OlZraakG4uM4VbrvRxlVKBsW0auqrkmm8PQpDMpqr0e9nSy2WW9QJ+kNGfPeJ88m3L6srsk2ZUnRYPtu3PfGnV8vXRVjEpx01m9n58J3xuHFXRni8TvrjmmdkLMUkIhEvXEuAZM5tjZldWo/3Z2pi1L2OSmfFmNj+260Gq10fwaUq9jxNlVaU8bFHF/crMbdY1xziovfR41dqGOVQnruQ6vs4jJCJ/x8ymmdnJKdtQLVUFz1KgZ46Nky1d3CYq56P0LNMly0oJZ57JdFyt3f0m8qfFyjbfTA8T7h31cPf2hEu1qlK4VaiUCoywbrWhlNyZj6pKv1dVX1bM//qM/mzl7uPIvy2r05+wbZ9sJNx3qpQaLf7sZtdE3YWEy/Vs88mUr4+Svk64zDyOkJijd8XiAdx9mrsPJ9wGeZxwrxB3/9jdf+DufYFTgUvN7NhqLC+zjbn68gZCf+7nIXXgWVTe7/L19QLCPt82Uba9KQ8zt1m+fWp70uNlzifXNqxqX6tOXMk+Y/f33P0Mwjb/BfCIhUxpO0RVwXMqodNuMrPWZtbCzIbEceOA75tZHwt5IW8gpOevyZnNg8ApZnZCPJNoEX+G1N3zp8WqTiqxtoRv8XIzG0w42KprAiFN2C5m1p1wX6g2PAnsYWaXmFlzM2trZofEcbWRfu8PwAXxrNviNvtiPBjzbcvFQHczK65i/meZ2UAL+TKvBR7x8BObdwlnQF80s2aEe2XNE9Ml+7MbIXNWLvn6KKkt4X7XckLgvqFihIV0dGeaWXt330h4gLAljjvZzPrHg3814TK5Oun1kvL1ZVvCw5DVcV1/mDFtztSBHjJLvQLcGOe5P+Fsant+2/rD2O89CCnkMrNxJW1PerykfNtwMdDbKud4TUodV8zsLDPb1UMS7VWxuKbbttryBs94YJxCuAn+IeEex9fi6HsITw8nE9JzlVPDIBN3luGEJ3EVKel+mGhX1rRYXr1UYhcB15rZx4RANKEGTfsZ4XLhA0JOxAdqsl65xMux4wl9ugh4Dzg6jt7u9HvuPp3wN2luIzwtLSHci6pqWz5HeHK7yMzypdd7gHBPdxHhQcB347xXE/r7LsJZ0to4/wrXxuEPCH+U7RFyp4nL10dJ9xO20XzCU/VXM8afDcyNl84XEFLVQUjr9k9CgJsC3OHuz+dZ52xtzNeXPyPkulxNCDiZKdtuJHxJrjKzy7LM/gzCWfQCwt8gusZDare0niCkq3sztufuXBV9O9LjZcwn3zas+Pv1y80sW9rB7Ykrw4CZZraGkKt2pIf0eRUvshxRzflUi95tlzpnZhcSduyjCt2WxszMnPCXB0oK3ZbGqEG+YSQNi5ntYWZDLKSJG0D4CctjhW6XyPbQWwVSF4oJf3ys4q9ijif8llGkwdJlu4hICrpsFxFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwbADMbJiZzTazEjO7stDtkeozs3vMbImZzSh0W6R2KXjWc2bWFLgdOBEYCJxhZgML2yqpgXuBYYVuhNQ+Bc/6bzBQ4u5z3H0DMB4YXuA2STW5+2RgRaHbIbVPwbP+6waUJobLYpmIFJCCp4hICgqe9d98oEdiuHssE5ECUvCs/6YBe5pZHzMrBkYCEwvcJpGdnoJnPefum4CLgaeBWcAEd59Z2FZJdZnZOGAKMMDMyszsvEK3SWqHuXuh2yAi0uDozFNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFzwbCzEYXug2SnrZf46Pg2XDo4GvYtP0aGQVPEZEUGs2P5Nt89d7GsSI5bJwziWZ9hxa6GTvEsBP3L3QTdrg5kx6l79DTCt2MHeaRcwdZmumqOm7XTBiVar51QWeeDURjDZw7i8YcOHdWCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIpIg2Vm95jZEjObkVH+HTN7x8xmmtnNifKrzKzEzGab2QmJ8mGxrMTMrqzOsotqbzVEROrcvcBtwP0VBWZ2NDAc+Ky7f2Jmu8XygcBIYB+gK/BPM9srTnY7cDxQBkwzs4nu/t98C1bwFJEGy90nm1nvjOILgZvc/ZNYZ0ksHw6Mj+UfmFkJMDiOK3H3OQBmNj7WzRs8ddkuIvWWmY02s+mJf6OrMdlewBFm9pqZvWBmB8fybkBpol5ZLMtVnpfOPEWkYHrvlT9GzXAfC4yt4WyLgI7AocDBwAQz65uqgVUsRESkMSkDHnV3B6aa2RagMzAf6JGo1z2Wkac8J122i0hj8zhwNEB8IFQMLAMmAiPNrLmZ9QH2BKYC04A9zayPmRUTHipNrGohOvMUkQbLzMYBQ4HOZlYGXAPcA9wTf760ATgnnoXONLMJhAdBm4D/cffNcT4XA08DTYF73H1mVctW8BSRBsvdz8gx6qwc9a8Hrs9S/hTwVE2Wrct2EZEUFDxFRFJQ8BQRSUHBU0QkBT0wKqA7LhzCiYO6s3R1OYMvewKA/Xp15DfnH0aL4qZs2ryF79/1Kq+/vwyAX547mC8c0J31n2zi23e8xFsfrGC/Xh259fxDadeyGZu3OL989G3+MmVu4VZqJ3HRkJ4c2KM9q8s3cenjswA4rHcHvvq5PejWoQVX/XU27y9ft7X+l/bbnWP26sQWh3teLeWtBR/TqXUzvnNEb9q3LAKHZ99dxlP/XVqoVZIa0plnAT00qYQRNzxbqey6sw7kxkfe5POXT+S6CW9w3VkHAfCFA7rRr0s7PvvdR/nO2Cnc+q3DAFi/YROjb3uRg3/wBCNueJZfjBpM+1bFdb4uO5vnS1Zw3bMllco+XFnOL5+bw6xFayqVd2/fgiF9d+H7j83i+mdKOP+wnjQx2LzFuW9aGd9/bBZXPTmbYXvvSvf2LepyNWQ71JszTzPbm/AyfsX7WvOBie4+q3Ct2rFenrWYnru2qVTmDu1aNgOgfatiFq4MZy8nH9STcZPfB2Dae0tp37qY3Tu0pGThR1unXbRyPUtXl9O5XXNWr9tQR2uxc5q1eA27tqn8JTV/dXnWugf3bM/Lc1ayaYuzZM0GFn38Cf07t+bdpWtZtX4TAOWbtjB/dTkdWzejLMd8pH6pF2eeZnYFMB4wwi/+p8bP46qbW6+xuOK+qVx39kG8c8fpXH/2QVzz8OsA7NGxFWXL1m6tt2D5Wrp2bFVp2gP7daa4qAlzFn9cp22W/Dq2bsaytRu3Di9fu4GOrZpVqrNrm2J6d2zFe0vXZk4u9VR9OfM8D9jH3TcmC83s18BM4KZsE8UMK6MBig88h2Z9h+7YVtaBb31hAFfeN40nXpvHaYf15o4LhnDKdc9UOd3uHVryh+8cwbdvfwn3Omio1JoWRU247Oi+3Du1jPUbtxS6OVJN9eLME9hCSE6aaY84Lit3H+vuB7n7QY0hcAJ8/aj+PPHaPAAenTKXA/t3BmDhinV079x6a72unVqzYEW4pG/bshl/ufI4rh33b6a9pwcO9c2KtRvp3PrTM81OrYtZsS6cJzQ1uOyYvrw4ZwWvzVtVqCZKCvUleF4C/MvM/m5mY+O/fwD/Ar5X4LbVqUUr1nHEwC4ADN13D95fFO5p/m16KWcc2Q+Ag/fclY/WbWDxqvU0a9qEcZcdzcOT3+fxGHSlfplWupohfXehqImxW5ti9mjXnJJ4C+aiw3tRtqqcJ2cuqWIuUt/Ui8t2d/9HzH4ymMoPjKZVvLjfGP3xe0dyxMAudGrbgtl3ns71E97k4t+/ws3nDqaoSRPKN27mO7+fAsDTb5RxwqBuvP3b01i/YTMX3PESAKd9vjdDPtOFjm1bcNbQ/gB8+/aX+M+8FQVbr53BJUf1Zp8ubWnboojff3Vf/vTGQtZ8sonzDu1BuxZFXHV8P+auWM91z5RQtqqcVz5Yxa1fGshmd+6aUsoWh713a81R/Tsxb8V6fnnq3gA8/O8FvFH2URVLl/rAvJHcIGvz1Xsbx4rshIaduH+hmyDb6ZFzB1ma6fa9+tm8x+2M645PNd+6UF8u20VEGhQFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFOrFu+0isnPau1+nQjchNZ15ioikoOApIg2Wmd1jZkvMbEai7Jdm9o6ZvW1mj5lZh8S4q8ysxMxmm9kJifJhsaykun+9QsFTRBqye4FhGWXPAvu6+/7Au8BVAGY2EBgJ7BOnucPMmppZU+B24ERgIHBGrJuXgqeINFjuPhlYkVH2jLtvioOvAt3j5+HAeHf/xN0/AEoIOYQHAyXuPsfdNxD+ntrwqpat4Cki9ZaZjTaz6Yl/o2s4i28Cf4+fuwGliXFlsSxXeV562i4i9Za7jwXGppnWzH4MbAIeqtVGRQqeItLomNko4GTgWP/0z2XMB3okqnWPZeQpz0mX7SLSqJjZMOBy4FR3X5cYNREYaWbNzawPsCcwFZgG7GlmfcysmPBQaWJVy9GZp4g0WGY2DhgKdDazMuAawtP15sCzZgbwqrtf4O4zzWwC8F/C5fz/VPyBSTO7GHgaaArc4+4zq1q2gqeINFjufkaW4rvz1L8euD5L+VPAUzVZti7bRURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJIVGk8/zpxceUegmSErX3vlioZsg2+vcQYVuQZ1rNMFTRBqewb3bF7oJqemyXUQkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QaLDP7vpnNNLMZZjbOzFqYWR8ze83MSszsT2ZWHOs2j8MlcXzv7Vm2gqeINEhm1g34LnCQu+8LNAVGAr8AbnH3/sBK4Lw4yXnAylh+S6yXmoKniDRkRUBLMysCWgELgWOAR+L4+4AR8fPwOEwcf6yZWdoFK3iKSIPk7vOBXwEfEoLmauB1YJW7b4rVyoBu8XM3oDROuynW75R2+QqeIlJvmdloM5ue+Dc6MW4XwtlkH6Ar0BoYVldtU1YlEam33H0sMDbH6OOAD9x9KYCZPQoMATqYWVE8u+wOzI/15wM9gLJ4md8eWJ62bTrzFJGG6kPgUDNrFe9dHgv8F3ge+Eqscw7wRPw8MQ4Txz/n7p524QqeItIguftrhAc//wb+Q4hnY4ErgEvNrIRwT/PuOMndQKdYfilw5fYsX5ftItJgufs1wDUZxXOAwVnqlgOn19aydeYpIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKC3jASkYI5oEv7QjchNZ15ioikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKehH8vXEjH89zuyXnwZ3Bhw+jH2PHcHrE+9n3luvYtaEFm3bc+Q5l9K6QydWLSpl8n23sLy0hINOPYf9vvDlQjd/p3PHhUM4cVB3lq4uZ/Bl4e+L7dtrF35z/mG0adGMeUvXcN5vJ/Px+o10bNOcBy8dyqD+nXloUgk/uOe1rfO5ZuQBnHFkfzq0KabLNx4q1OpICjrzrAdWzJ/L7JefZviVt/Clq2+n9D9T+WjJAvY7/iuc9pM7+NLVt9Fzv8G8+beHAWjeqi2Hfe0C9jtOQbNQHppUwogbnq1Udvu3h3DNQ69zyGVP8Nep87jk1H0BKN+4mZ//6Q1+/MD0bebz1OtlHPWjJ+ukzVK7FDzrgdWLStmt9wCKilvQpGlTuuy5L3PfeJnilq221tm0oRzMAGjZrgO79t6LJk2bFqrJO72XZy1m5ZoNlcr6d23HS7MWA/Dc2wsYfkgvAIhTHwcAAATbSURBVNZ9sokps5dQvmHzNvOZ9t5SFq9av+MbLLWu3gdPMzu30G3Y0Xbp2otFJTMoX/MRmzaUUzpjOmtXLgNg+uP3Mf6qb1AydRKDTjm7wC2VfGaVruLkg3sC8KVDe9OtU+sCt0h2pHofPIGf5RphZqPNbLqZTX/tyfF12aZa1WGPnux/wun847dX84/f/oROPfpiTcKmOWjEOYy88X76Dx7KrEl/LXBLJZ+L7nyZ878wgBdvOpm2LZuxYdO2Z5rSeNSLB0Zm9nauUcDuuaZz97GEP3LPzc+/7zugaXVmwJATGDDkBACmP34vrTp0rjS+3+Cjefq2axh0ylmFaJ5Uw7sLVjP8+nAftP8e7ThhUPcCt0h2pHoRPAkB8gRgZUa5Aa/UfXPq3vqPVtGyXQfWrFjC3Dde4ZQrfs3qxfNpv3s3AOa99SoddtfBWJ/t2q4FSz8qxwwuP21/7n52dqGbJDtQfQmeTwJt3P3NzBFmNqnum1P3/jX2ej5Z8xFNmhbx+TMuonmrNrz0wK2sWjwfM6NNx90Y8vWLAVi3egVP3Pg9Npavw6wJM557nC9f8/tKD5hkx/rj947kiIFd6NS2BbPvPJ3rJ7xJmxZFnH/C3gBMnPohDzxfsrX+zNu+QttWzSguasLJB/dk+HXP8M781fz8zAP56uF9aVVcxOw7T+e+597jhj9vcxhIHmbWFJgOzHf3k82sDzAe6AS8Dpzt7hvMrDlwP3AgsBz4mrvPTb1c9wZ9tbtVQ79s35lde+eLhW6CbKc1E0ZZmumenbUs73F7/Gc6VzlfM7sUOAhoF4PnBOBRdx9vZr8D3nL3O83sImB/d7/AzEYCX3L3r6VpNzSMB0YiIlmZWXfgi8BdcdiAY4BHYpX7gBHx8/A4TBx/bKyfioKniDRktwKXA1vicCdglbtvisNlQLf4uRtQChDHr471U1HwFJF6K/lzxPhvdGLcycASd3+9EG2rLw+MRES2kfw5YhZDgFPN7CSgBdAO+A3QwcyK4tlld2B+rD8f6AGUmVkR0J7w4CgVnXmKSIPk7le5e3d37w2MBJ5z9zOB54GvxGrnAE/EzxPjMHH8c74dT8wVPEWksbkCuNTMSgj3NO+O5XcDnWL5pcCV27MQXbaLSIPn7pOASfHzHGBwljrlwOm1tUydeYqIpKDgKSKSgi7bRaRg+jTgtH068xQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEZEUFDxFRFJQ8BQRSUHBU0QkBQVPEWmwzGyYmc02sxIzu7Iul63gKSINkpk1BW4HTgQGAmeY2cC6Wr6Cp4g0VIOBEnef4+4bgPHA8LpauIKniDRU3YDSxHBZLKsTRXW1oB3t8qP7WaHbIOlcfnS/QjdBCqT/bi3zHrdmNhoYnSga6+5jd2yrqqfRBE8RaXxioMwVLOcDPRLD3WNZndBlu4g0VNOAPc2sj5kVAyOBiXW1cJ15ikiD5O6bzOxi4GmgKXCPu8+sq+Wbu9fVskREGg1dtouIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIin8P3wbzzjkxy31AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.DataFrame(\n",
    "    data=metrics.confusion_matrix(y_test, np.rint(y_pred_stack1)), \n",
    "    \n",
    ")\n",
    "\n",
    "confusion_figure, confusion_axes = matplotlib.pyplot.subplots()\n",
    "confusion_figure.set_size_inches(5, 3)\n",
    "confusion_axes.set_title(\n",
    "    'Confusion matrix showing the frequency of \\n'\n",
    "    'correct and incorrect bug classification predictions.'\n",
    "    '\\n\\n'  # hack to avoid overlap with x-axis labels below\n",
    ")\n",
    "confusion_axes.xaxis.tick_top()  # move x-axis labels to top of matrix\n",
    "_ = sns.heatmap(\n",
    "    confusion_matrix,\n",
    "    annot=True,\n",
    "    fmt=\"d\",\n",
    "    cmap=sns.color_palette(\"Blues\"),\n",
    "    vmin=0,\n",
    "    ax=confusion_axes,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "tn, fp, fn, tp = metrics.confusion_matrix(y_test, np.rint(y_pred_stack1)).flatten().tolist()\n",
    "\n",
    "fpr_stack1 = fp/(fp+tp)\n",
    "fnr_stack1 = fn/(fn+tn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3657484234981746"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fpr_stack1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.340651298938895"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fnr_stack1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import auc, roc_curve, precision_recall_curve, matthews_corrcoef, f1_score, roc_auc_score\n",
    "fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_test, np.rint(y_pred_stack1))\n",
    "precision, recall, thresholds = precision_recall_curve(y_test, np.rint(y_pred_stack1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "auc_keras = auc(fpr_keras, tpr_keras)\n",
    "roc_auc = roc_auc_score(y_test, np.rint(y_pred_stack1))\n",
    "pr_auc = auc(recall, precision)\n",
    "f1 = f1_score(y_test, np.rint(y_pred_stack1))\n",
    "mcc = matthews_corrcoef(y_test, np.rint(y_pred_stack1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PR AUC: 0.734\n",
      "ROC AUC: 0.646\n",
      "MCC: 0.293\n",
      "F1: 0.653\n"
     ]
    }
   ],
   "source": [
    "print('PR AUC: %.3f' % pr_auc)\n",
    "print('ROC AUC: %.3f' % roc_auc)\n",
    "print('MCC: %.3f' % mcc)\n",
    "print('F1: %.3f' % f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model 2 : Adding in 1-Nearest Neighbour"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "models2 = [clf_svm, logreg, oneknn]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "task:         [classification]\n",
      "n_classes:    [2]\n",
      "metric:       [accuracy_score]\n",
      "mode:         [oof_pred_bag]\n",
      "n_models:     [3]\n",
      "\n",
      "model  0:     [SVC]\n",
      "    fold  0:  [0.63557257]\n",
      "    fold  1:  [0.65158371]\n",
      "    fold  2:  [0.64891210]\n",
      "    fold  3:  [0.64560487]\n",
      "    ----\n",
      "    MEAN:     [0.64541831] + [0.00606614]\n",
      "    FULL:     [0.64541815]\n",
      "\n",
      "model  1:     [LogisticRegression]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    fold  0:  [0.63731291]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    fold  1:  [0.66063348]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    fold  2:  [0.65204526]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    fold  3:  [0.64595300]\n",
      "    ----\n",
      "    MEAN:     [0.64898616] + [0.00852178]\n",
      "    FULL:     [0.64898616]\n",
      "\n",
      "model  2:     [KNeighborsClassifier]\n",
      "    fold  0:  [0.58997564]\n",
      "    fold  1:  [0.60389836]\n",
      "    fold  2:  [0.59216710]\n",
      "    fold  3:  [0.60087032]\n",
      "    ----\n",
      "    MEAN:     [0.59672786] + [0.00580881]\n",
      "    FULL:     [0.59672787]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "S_train, S_test = stacking(models2,                   \n",
    "                           x_train, y_train, x_test, #v disc testing,   \n",
    "                           regression=False, \n",
    "     \n",
    "                           mode='oof_pred_bag', \n",
    "       \n",
    "                           needs_proba=False,\n",
    "         \n",
    "                           save_dir=None, \n",
    "            \n",
    "                           metric=accuracy_score, \n",
    "    \n",
    "                           n_folds=4, \n",
    "                 \n",
    "                           stratified=True,\n",
    "            \n",
    "                           shuffle=True,  \n",
    "            \n",
    "                           random_state=0,    \n",
    "         \n",
    "                           verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "level_two2 = clf_svm.fit(S_train, y_train)\n",
    "y_pred_stack2 = level_two2.predict(S_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final prediction score for model2: [0.64914723]\n"
     ]
    }
   ],
   "source": [
    "print('Final prediction score for model2: [%.8f]' % accuracy_score(y_test, np.rint(y_pred_stack2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAEFCAYAAABjDmX4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd5xV1bn/8c/DDL03RTqKDdSoUTRBFE2MYAnGBKNXvWKw38QYYyy5/iIxUdHcxCTXkmCJLQFLLKQYo1HEjiVcFQEdAYWhijDSy/D8/lhrcM/hnDMzm2HOnPH7fr0Gzl67rb323s/Z9Tnm7oiISN00K3QFRESKkYKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKTTZ4mllrM/uLmVWY2UPbMZ3TzOyf9Vm3QjGzYWY2u56nOc7M7q/PaWaZxxNmduaOnEdiXvW2PPW1DTZVZjbUzN43s9VmdmKh61NXBQ+eZvYfZvZ6bMBFcUc5rB4m/S1gZ6Cru49OOxF3/6O7f60e6rNDmZmb2cB8w7j78+6+Z0PVqb64+0h3v6e+p2tmw81sQX1PN6FetsEm7BrgZndv5+6PFboydVXQ4GlmlwC/Bq4jbGR9gVuBUfUw+X7Ae+6+uR6mVfTMrLTQdfgcyrsNap3QD5hR6Eqk5u4F+QM6AquB0XmGaUkIrgvj36+BlrHfcGAB8ENgKbAIOCv2+ymwEdgU5zEWGAfcn5h2f8CB0tg9BpgDrALmAqclyl9IjPdl4DWgIv7/5US/KcDPgBfjdP4JdMuxbFX1vyxR/xOBY4H3gE+AHyeGHwK8DKyMw94MtIj9psZlWROX99uJ6V8OLAbuqyqL4+wW53Fg7O4JLAOG56jv5UB5XK7ZwFdi+TjgQeDe2G8GcFBivL1ju6yM/b4eywfEsmax+3ZgaWK8+4CLE+16dnJ9AP8DrIjramRivAGxPVYBTwO3JNd7Yri2wDpgS2yz1bENalqensCfY1vNBS7K0V7ZtsExcdu4CVgO/DwO+x1gZlyeJ4F+iekcDcwibG83A88l2mIc+bfpjsCdhO2lHPg5UFLLduwC/IGw360AHovl7wAnJIZrDnwMHJCjHc4Bygjb2mSgZyz/ILb9utg+LQsVi1LHsILNGEYAm6tWdI5hrgFeAXYCugMvAT+L/YbH8a+JK/BYYC3QOceGlXNDizvSp8Cesd8uwODkRpbYoFYAZ8TxTo3dXRM7+QfAHkDr2D0+x7JV1f8nsf7nEHbIPwHtgcFxwxoQh/8icGicb3/CznZxYnoODMwy/RsIX0KtSQTPxIb9LtCGsNP+T4667gnMT2z4/YHdEu26PrZ/CXA98EpixyoDfgy0AI4iBKSqdv4I+GL8PJvw5bV3ot8BiXZNBs9Nse4lwAWEHdxi/5cJAaEFcFhcr9sEz0QbLcgoy7c8zYA34jprAewa63xMjumPo/o2Nyauk+/F9diacJZVRviSKQWuAl6Kw3eL7fWt2JY/iOPXNng+CvyesH3vBEwDzqtlO/4NeADoHOd9RCy/DHggMc9RwNs5lv8oQmA9kLAN/i8wNdF/HvDVQsWg7Y5hBZsxnAYsrmGYD4BjE93HAPMSG/46EsGXcAR3aI4NK+eGFjeulcA3gdYZdRjDZ8HzDGBaRv+XgTHx8xTgqkS/C4F/5Fi2qvpXHQm0j/U5JDHMG8CJOca/GHg00Z0teG4EWmWUZQaLycDbwFvk+PYHBsa2/SrQPKPfOODpRPcgYF38PIxw1Nss0X8iMC5+vg+4BOhBCJ43Auez7VHpFKoHz7LE9NrEZe9BuOyzGWiT6H8/dQ+euZbnEOCjjOGvBP6QY/qZ29yYLOM/AYxNdDcjHAT0A/6TGLhjPyOcTdQYPAmXwTaQ2J4JX/bP1qIddyEcFXbOskw9CQG9Q+x+GLgsx/LfCdyY6G5HCNj9Y/c8ijh4FvKa53KgWw3XfXoCHya6P4xlW6fh1a8nrSWsoDpx9zWEU93zgUVm9jcz26sW9amqU69E9+I61Ge5u1fGz+vi/0sS/ddVjW9me5jZX81ssZl9SrhO3C3PtAGWufv6Goa5HdgH+F9335BtAHcvIwTrccBSM5tkZsn1kLnMreJ67QnMd/ctif7J9nqOEMAOJ5xqTwGOiH/PZ4yXtHV+7r42fmwX5/dJogzCEXNd5VqefkBPM1tZ9Uc4qt65DtPOrE8/4DeJ6X1CCJK9iO1XNaCHiFPb5elHOGJclJj27wlHoFVytWMfQjuuyJyouy8kXHr4ppl1AkYCf8xRh2r7i7uvJuz3vXIMX1QKGTxfJnwz5ntEYSFhI6jSN5alsYbw7VqlR7Knuz/p7kcTvnVnEYJKTfWpqlN5yjrVxW2Eeu3u7h0IO63VMI7n62lm7QjXke8ExplZl5wTcv+Tux9GWH4nXA6oyUKgj5klt7Nkez1HODodHj+/AAwlBM/najH9TIuALmaWXM998gyft32ymA/MdfdOib/27n5sHaaROc/5hFPp5DRbu/tLhOXZWn8zM6ovT75tej5h/+qWmG4Hdx9cy+XsEoNjNvcApwOjgZfdPdf2X21/MbO2QFcaZn/Z4QoWPN29gnDt6BYzO9HM2phZczMbaWY3xsEmAleZWXcz6xaHT/sM3nTgcDPra2YdCadbAJjZzmY2Kq7cDYQL2NmOev4O7BEfryo1s28TTuv+mrJOddGecP1udTwqviCj/xLCNbi6+A3wurufTbjG9btsA5nZnmZ2lJm1JFwPrLrRUpNXCUdul8V1Oxw4AZgE4O7vx2mdDjzn7p/G5fgmKYKnu38IvE74ImhhZl+K88tlCdA1bg+1MQ1YZWaXx2c4S8xsHzM7uK51TfgdcKWZDQYws45mVvVY09+AwWZ2UjzyvYjqATLnNu3uiwg3LH9pZh3MrJmZ7WZmR9RUoTjuE8CtZtY5rrvDE4M8RriO+X3CjbVcJgJnmdn+cdu5DnjV3efVVIdiUNBHldz9l4RrXlcRbpbMB75LWDkQ7g6+Trge9zbwZixLM6+nCBfA3yJcS0wGvGaxHgsJp01HsG1wwt2XA8cT7vAvJ1w8P97dP05Tpzq6FPgPwvWm2wnLkjQOuCeeop1c08TMbBThpl3Vcl4CHGhmp2UZvCUwnnDxfzHh1O/KLMNV4+4bCcFrZBz3VuA/3X1WYrDnCJcv5ie6jbCu0zgN+BLxbjahnXJdjphF2MHnxHbrmW24xPCVhPW/P+Hu9MfAHYS72qm4+6OEo/hJ8XLMO4T2Im5XowltvxzYnXDKXDVuvm0awjXTFoSbgisI1yd3qWXVziBcn5xFuN59cWK+6whPHAwAHsmzbE8D/y8Ou4jwhMcptZx/o1d1Z02kSTKzB4BZ7n51oetSH8xsCuEm0R0FrsdPgD3c/fRC1qOQCv6GkUh9MrOD4+lpMzMbQXiUpujeXmnM4rXxscCEQtelkBQ8panpQbhrvxr4LXCBu/+7oDVqQszsHMLltSfcfWqh61NIOm0XEUlBR54iIikoeNZCTdl3Ykaouj4mVFTMbIqZnV3gOuzQ9HdmNiM+ToUFfzCzFWY2zXZAOr84n75x+ymp72nvaMltwrYjdaM1YMrB+lSUwbMBUonViYeUWnMKXY+0zGyMmb1Q6HoUmrsPdvcpsfMwQlKO3u4+xOspnZ+ZzTOzrybm+VHcfirzjdfYeS1TN2b7AvQdlHJwR2uQ4JntFUyl40pP7dkg+hHyKKwpdEUagrafFGp6+Z3wOtgjhIfYlxOSl0IIvFcR3l1dSnjToKNXT1AwlpAdZ2q2sjjsoYRsSSuB/yOREo0sabHIkUosS72PA/5NeCtnPjEZRUb9zox1+Rj470T/1sDdcZ7vAj8iI4FExry2JuWI491CeDtkFeEtm90Sww4GniI8jL+EmHaO2qXf2ya9XLIsDns84c2TlbFd98u3LgnZfNYDlbEtV+ZYximEDEPTYps+DnRJ1i9j+HnEpA+xPe+J7TmT8HJBvvbM1UbjqJ4I46G47BWEbWxwot+xcd2tIrwOeGks70Z4mLzqPfLn+SwByTxC8pOxGW3y08xlzNaWsXw34JlY9jHhve9Osd99VE/DdhnbZkLqSUjW8gkh29I5iXmOI0+6vBzb5UWEzE8fA79ILOsY6jc93hiqp27cZh0SXspIpun7v8S2VTWd2sSVXPvtEMJLNVVvqv2qpvi2PX81Bc4SQkC7iRC0WgGHJRq5jPBKYLu4IVXtwFULeW8cr3WOsl5xxR0bG+3o2N09TidXWqzh5Nn5EsPsG6e7X2zMEzPqd3usxxcIb6FUpUMbT9ipuhB2knfyzY9tg+fyuCJLCTvPpNivPeFNix/GtmxPzKJE7dLvZaaXyyw7IG5wh8R1dyYhILSsYV2OIbHh5wme5YQkIm0Jb43cX8vgOZ6wo3UGehPeiMnanjW00TiqB8/vxP5VXzzTE/0WAcPi5858lrf0esIrkc3j3zA+e+okWedqbZJcxhraciBhO24Z1+NU4NfZ2iVjW6wKnlMJb2K1IrzJtAw4KrH8WdPl5dkunyVsx30JeWKTwa4+0+Ntba+6rMMswbM2cSXXfvsycEb83I6YYa1QwfNLceVtk3MT+BdwYaJ7T8I3SmliIXfNspEkyy6vaphE2ZOEnT5fWqzh1BA8s4zza+CmjLr0TvSfBpwSP88BRiT6nZtvfmwbPO9I9DuW8IYLhJRg/84xjZrS72VLL5dZdhsx4CbKZhNeN823LsdQu+A5PtE9KM6/JNv6oHogqpbzEjg7V3vW0EbjyJ1erlNcD1VHKR8B5xFTpyWGu4Zw1DwwyzSSda7WJlQPnjnbMss0T0wuD3mCJ+GLuhJon+h/PXB3YvmzpsvLs10mt+MLgX8llq8+0+Ntba+6rkOqB8/axJVc++1UwllC1gTk9f1X0zXPPsCHnv1nBLKli6vKI1glW/qsZFk/YHRGiq/DCIEzZ1qs2jCzQ8zsWTNbZmYVhHRzmSnccqWPq5YKjG3T0NUk13T7EIJkNjWl38uWXi6zrB/ww4z27BOnk29d1lZmmzSn5rR4sG175kurlq+NtopJOcab2QfxnfB5sVdVfb5J+OL60Myei0lCIJy6lgH/NLM5ZnZFLeqfrY5Z2zImmZlkZuWxXvdTuzaCz1LqrUqU1ZTysFUN1ysz11nPHP2g/tLj1Wod5lCbuJJr/xpLSEQ+y8xeM7PjU9ahVmoKnvOBvjlWTrZ0cZupno/Ss4yXLJtPOPJMpuNq6+7jyZ8WK9t0M/2JcO2oj7t3JJyq1ZTCrUq1VGCEZasP88md+aim9Hs1tWXV9K/NaM827j6R/OuyNu0J27bJJsJ1p2qp0eJjN90Twy4inK5nm06mfG2U9B+E08yvEhJz9K+aPYC7v+buowiXQR4jXCvE3Ve5+w/dfVfg68AlZvaVWswvs4652vI6Qnvu6yF14OlU3+7ytfVCwjbfPlG2vSkPM9dZvm1qe9LjZU4n1zqsaVurTVzJPmH39939VMI6vwF42EKmtB2ipuA5jdBo482srZm1MrOhsd9E4AdmNsBCXsjrCOn563Jkcz9wgpkdE48kWsXHkHp7/rRYtUkl1p7wLb7ezIYQdrbaepCQJqyzmfUmXBeqD38FdjGzi82spZm1N7NDYr/6SL93O3B+POq2uM6OiztjvnW5BOhtZi1qmP7pZjbIQr7Ma4CHPTxi8x7hCOg4M2tOuFbWMjFesj17ETJn5ZKvjZLaE653LScE7uuqelhIR3eamXV0902EGwhbYr/jzWxg3PkrCKfJtUmvl5SvLdsTboZUxGX9Uca4OVMHesgs9RJwfZzmfoSjqe15tvVHsd37EFLIZWbjStqe9HhJ+dbhEqC/Vc/xmpQ6rpjZ6WbW3UMS7ZWxuK7rttbyBs+4Y5xAuAj+EeEax7dj77sIdw+nEtJzraeOQSZuLKMId+KqUtL9KFGvrGmxvHapxC4ErjGzVYRA9GAdqvZTwunCXEJOxPvqsly5xNOxowltuhh4Hzgy9t7u9Hvu/jrhN2luJtwtLSNci6ppXT5DuHO72Mzypde7j3BNdzHhRsBFcdoVhPa+g3CUtCZOv8o1sXsu4UfZHiZ3mrh8bZR0L2EdlRPuqr+S0f8MYF48dT6fkKoOQlq3pwkB7mXgVnd/Ns8yZ6tjvrb8KSHXZQUh4GSmbLue8CW50swuzTL5UwlH0QsJv0F0tYfUbmk9TkhXNz3W585cA/p2pMfLmE6+dVj1+/XLzSxb2sHtiSsjgBlmtpqQq/YUD+nzql5kGVbL6dSK3m2XBmdmFxA27CMKXZemzMyc8MsDZYWuS1NUlG8YSXExs13MbKiFNHF7Eh5hebTQ9RLZHnqrQBpCC8KPj1X9KuYkwrOMIkVLp+0iIinotF1EJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwbMImNkIM5ttZmVmdkWh6yO1Z2Z3mdlSM3un0HWR+qXg2ciZWQlwCzASGAScamaDClsrqYO7gRGFroTUPwXPxm8IUObuc9x9IzAJGFXgOkktuftU4JNC10Pqn4Jn49cLmJ/oXhDLRKSAFDxFRFJQ8Gz8yoE+ie7esUxECkjBs/F7DdjdzAaYWQvgFGBygesk8rmn4NnIuftm4LvAk8BM4EF3n1HYWkltmdlE4GVgTzNbYGZjC10nqR/m7oWug4hI0dGRp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeRcLMzi10HSQ9rb+mR8GzeGjnK25af02MgqeISApN5iH5diff3TQWJIdNc6bQfNfhha7GDjFi5H6FrsION2fKI+w6/KRCV2OHefisAy3NeDXtt6sfHJNqug1BR55FoqkGzs+Lphw4P68UPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RKVpmdpeZLTWzdzLKv2dms8xshpndmCi/0szKzGy2mR2TKB8Ry8rM7IrazLu0/hZDRKTB3Q3cDNxbVWBmRwKjgC+4+wYz2ymWDwJOAQYDPYGnzWyPONotwNHAAuA1M5vs7u/mm7GCp4gUTP89em3X+O4+1cz6ZxRfAIx39w1xmKWxfBQwKZbPNbMyYEjsV+bucwDMbFIcNm/w1Gm7iDQ1ewDDzOxVM3vOzA6O5b2A+YnhFsSyXOV5KXiKSKNlZuea2euJv3NrMVop0AU4FPgR8KCZWX3XTaftItJoufsEYEIdR1sAPOLuDkwzsy1AN6Ac6JMYrncsI095TjryFJGm5jHgSIB4Q6gF8DEwGTjFzFqa2QBgd2Aa8Bqwu5kNMLMWhJtKk2uaiY48RaRomdlEYDjQzcwWAFcDdwF3xceXNgJnxqPQGWb2IOFG0Gbgv9y9Mk7nu8CTQAlwl7vPqGneCp4iUrTc/dQcvU7PMfy1wLVZyv8O/L0u89Zpu4hICgqeIiIp6LS9gG69YCgjD+zNsor1DLn0cQDuufgIdu/ZEYCObVpQsXYjX75sMqUlxi3nD2X/AV0pbWb8aeoH/PKxt7cOd8v5X2ZQn864Oxfc9iLT3l9WsOX6PLhwaF++2KcjFes3c8ljMwH4Uv9OnLz/LvTq1Ior/zKbD5avBaBdyxIuPXJXduvWhilly7nzlQVbpzN0QGdO+kIPcPhk7UZ+O3UeqzZUFmSZpG4UPAvoj1PK+P0/ZnL7fw3bWnbmr5/b+vm6Mw7i07WbAPjGof1pWVrCIZc+TusWJbz+q2/w0Itz+WjZam48awhPTS/n9F9NoXlJM9q01Grd0Z4t+4QnZi3je8P6by37aMV6fvHMHM77ct9qw26qdCa9uZC+nVvTp3OrreXNDL5zSG8ufvRdVm2o5PSDejFy7514cPqihloM2Q6NZi8zs70Ir0RVPdlfDkx295mFq9WO9eLMJfTt3i5n/5O+NIDjrvnH1u42rUopaWa0blHKxs2VrFq7kQ6tmzN0750575YXANhUuYWKtRt3eN0/72YuWU33di2qlZVXrM867IbNW5i1dA09OrSsVm7xn5alJazaUEmb5s1Y/Gn2aUjj0yiCp5ldDpwKTCI8dwXhQdWJZjbJ3ccXrHIFMnTvnVlasY4PFq8C4NFX5nHcQX35YMK3ad2ihCvueY0Vazayb78ufPzpen534WHs268z/56znMvunsbaDZsLvARSk0qH21+az69O3JsNm7ew6NP13PHK/JpHlEahsdwwGgsc7O7j3f3++Dee8NL+2FwjJV/d2jRnSkPVtUGMHjqAh16cu7X7oIHdqdyyhYHnPcA+3/0z3zthMP13akdpibH/gK7c8c9ZDL38L6zdsJkfnrhvAWsutVVicMxe3fjR5Jmc88DbfLhiHd/Yr0ehqyW11FiC5xZCiqhMu8R+Wbn7BHc/yN0Par7r8B1UtYZX0sz4+pB+/Pmlz4LnyYcN4Knp5WyudJZ9up5XZi/lwN26Ub58LeXL1/J62ccAPPbKPL4woEuhqi510L9rGwCWrAqXWV6au5I9d2pbyCpJHTSW4Hkx8C8ze8LMJsS/fwD/Ar5f4Lo1uCP37cl7CytY+MnarWXzP17DEfvsAkCblqUM2b07s8srWFqxjvLla9h9lw4ADN+3J7MWVBSk3lI3n6zZRO9OrekQb/B9oWd7FqzUNc9i0Siuebr7P+I7qEOofsPotarXp5qiP3z/cIYN6kHX9q2Yfdtorn1wOvc++z7fyjhlB5jwj1n87sLDeO2XozAz7nv2fWZ8tAKAH971KndedDgtSpsxd+lqLrj1hUIszufKxUf0Z3CP9rRvVcrvT96HB/69iNUbNjP20D50aFXKlUfvxrxP1vHzf5YBcOu3BtO6RQmlzYwhfTvxsyfLWFCxnoemL+KaY/egcouzbPVGbn5hXmEXTGrNwiufxa/dyXc3jQX5HBoxcr9CV0G208NnHZgq5ds+Vz2Vd7995+dH13squfrSWE7bRUSKioKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKCp4iIikoeIqIpKDgKSKSgoKniEgKjeLddhH5fNprt66FrkJqOvIUEUlBwVNEJAUFTxGRFBQ8RaRomdldZrbUzN5JlP3CzGaZ2Vtm9qiZdUr0u9LMysxstpkdkygfEcvKzOyK2sxbwVNEitndwIiMsqeAfdx9P+A94EoAMxsEnAIMjuPcamYlZlYC3AKMBAYBp8Zh81LwFJGi5e5TgU8yyv7p7lU/H/sK4Zd4Ify0+SR33+Duc4Eywq9XDAHK3H2Ou28k/IrvqJrmreApIo1W8hdy49+5dZzEd4An4udeQPK3nRfEslzleek5TxFptNx9AjAhzbhm9t/AZuCP9VqpSMFTRJocMxsDHA98xT/7obZyoE9isN6xjDzlOem0XUSaFDMbAVwGfN3d1yZ6TQZOMbOWZjYA2B2YBrwG7G5mA8ysBeGm0uSa5qMjTxEpWmY2ERgOdDOzBcDVhLvrLYGnzAzgFXc/391nmNmDwLuE0/n/qvppczP7LvAkUALc5e4zapq3gqeIFC13PzVL8Z15hr8WuDZL+d+Bv9dl3jptFxFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJoclkVfrJBcMKXQVJ6Zrbni90FWR7nXVgoWvQ4JpM8BSR4jOkf8dCVyE1nbaLiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkikoKCp4hICgqeIiIpKHiKiKSg4CkiRcvMfmBmM8zsHTObaGatzGyAmb1qZmVm9oCZtYjDtozdZbF//+2Zt4KniBQlM+sFXAQc5O77ACXAKcANwE3uPhBYAYyNo4wFVsTym+JwqSl4ikgxKwVam1kp0AZYBBwFPBz73wOcGD+Pit3E/l8xM0s7YwVPESlK7l4O/A/wESFoVgBvACvdfXMcbAHQK37uBcyP426Ow3dNO38FTxFptMzsXDN7PfF3bqJfZ8LR5ACgJ9AWGNFQdVNKOhFptNx9AjAhR++vAnPdfRmAmT0CDAU6mVlpPLrsDZTH4cuBPsCCeJrfEVietm468hSRYvURcKiZtYnXLr8CvAs8C3wrDnMm8Hj8PDl2E/s/4+6eduYKniJSlNz9VcKNnzeBtwnxbAJwOXCJmZURrmneGUe5E+gayy8Brtie+eu0XUSKlrtfDVydUTwHGJJl2PXA6Pqat448RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFPSQvIgUzAE9Oha6CqnpyFNEJAUFTxGRFBQ8RURSUPAUEUlBwVNEJAUFTxGRFBQ8RURSUPAUEUlBD8k3Eu88/SizX3wSzOjSsz/DzvwBSz94l2l/vpPKys106zuQYWdcTLOSEjauW8OUu37Bmk+WsWVLJfsefRJ7fPlrhV6Ez5VbLxjKyAN7s6xiPUMuDT+Rs0+/zvzmnC/RrlVzPly2mrG/ncqqdZu2jtO7a1tev+lErntoOr/9y4yt5c3MeH788Sz8ZC2jb/hXgy+LpKMjz0ZgzYqPmfHsZEZd+Ru++ZPb8C2VzJk2han3/Iojz76cb/7kNtp12Yn3X3kagHen/JVOu/TlG//vFo695AZeffgOKjdvqmEuUp/+OKWME697qlrZLecN5eo/vsEhlz7OX6Z9yMVf36da//FnHsxT/y4n04XH7s3s8oodWl+pfwqejYRvqaRy00a2VFayedMGSlu2pFlJKR137g1Ar70PYN6bLwJgBpvWr8Pd2bxhHS3btqdZs5JCVv9z58WZS1ixemO1soE9O/DCzCUAPPPWQkYd0m9rv+MP7su8pauZuWBltXF6djH1cRwAAAWFSURBVGnDiAN7c8+/3tvxlZZ61eiDp5mdVeg67GhtO3djn6+exKQfn8nEy0+jRau2DPji4WzZUsmyD8NONffNF1izYhkAew8/gYrF85l4+ek88rMLOfTk87BmjX5VNnkz56/k+IP7AvCNQ/vTq2tbANq2LOUHo/bh+oembzPOjWOGcNX9b7Al9Q/gSqEUwx7301w9zOxcM3vdzF5/9a+TGrJO9WrDmlV89NYrnPzzP3DqDfezaeN6Ppj2LEeefQWvPnQ7j19/Mc1btcbi0WX5jDfp0ntXTr3hfr7x3zfz8qTb2LhubYGXQi687UXO+dqePD/+eNq3bs7GzZUA/Pjk/bnlb++yZsPmasOPiNdMp89dXojqynZqFDeMzOytXL2AnXON5+4TCL/TzI3PflC0390LZ02nfdcetG4fMsz0P2AoSz6YycBDjuL4S38BwIJ33+TTJeF62XsvP8UXjhmNmdFhp56077YzFYvn033AngVbBoH3FlYw6tpwHXTgLh045sBwyeXggd058ZD+/Oy0g+jYtgVb3NmwsZJdurTh2IP68LUDetOqRQntWzfnju8N4+z/fb6QiyG11CiCJyFAHgOsyCg34KWGr07DatulO0vnzmLzxvWUNG/JwlnT6dZvd9Z9upLWHTpRuWkTbz35EPuP/DYA7bp0Z+Gs6fTYfR/WfbqCisXltO/eo8BLId07tGLZp+sxg8tO2o87n5oNwNeufmLrMD8evT+r12/i90/OAmDcxDcBGDaoBxedMFiBs4g0luD5V6Cdu29zUcjMpjR8dRrWTgP2YsCBh/HYtRdhJSV07bMrex02kjcm38tHb08D38Jehx9Hz732B2D/Y09l6j2/4pFrLsCBg086i1btijcvYjH6w/cPZ9igHnRt34rZt43m2gen065VKeccsxcAk6d9xH3PlhW4lrIjmXvRnu1WU8yn7Z9319ymo61it/rBMZZmvKdmfpx3vz167241TtfMSoDXgXJ3P97MBgCTgK7AG8AZ7r7RzFoC9wJfBJYD33b3eWnqDcVxw0hEJJ/vAzMT3TcAN7n7QMKlwLGxfCywIpbfFIdLTcFTRIqWmfUGjgPuiN0GHAU8HAe5Bzgxfh4Vu4n9vxKHT0XBU0QareTjiPHv3IxBfg1cBmyJ3V2Ble5e9VzYAqBX/NwLmA8Q+1fE4VNpLDeMRES2kXwcMZOZHQ8sdfc3zGx4g1YMBU8RKV5Dga+b2bFAK6AD8Bugk5mVxqPL3kBVQoFyoA+wwMxKgY6EG0ep6LRdRIqSu1/p7r3dvT9wCvCMu58GPAt8Kw52JvB4/Dw5dhP7P+Pb8biRgqeINDWXA5eYWRnhmuadsfxOoGssvwS4YntmotN2ESl67j4FmBI/zwGGZBlmPTC6vuapI08RkRR05CkiBTMgpu0rRjryFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RkRQUPEVEUlDwFBFJQcFTRCQFBU8RKVpmNsLMZptZmZld0ZDzVvAUkaJkZiXALcBIYBBwqpkNaqj5K3iKSLEaApS5+xx33whMAkY11MwVPEWkWPUC5ie6F8SyBlHaUDPa0S47cjcrdB0kncuO3K3QVZACGbhT67z7rZmdC5ybKJrg7hN2bK1qp8kETxFpemKgzBUsy4E+ie7esaxB6LRdRIrVa8DuZjbAzFoApwCTG2rmOvIUkaLk7pvN7LvAk0AJcJe7z2io+Zu7N9S8RESaDJ22i4ikoOApIpKCgqeISAoKniIiKSh4ioikoOApIpKCgqeISAoKniIiKfx/KkEji590x3AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.DataFrame(\n",
    "    data=metrics.confusion_matrix(y_test, np.rint(y_pred_stack2)), \n",
    "    \n",
    ")\n",
    "\n",
    "confusion_figure, confusion_axes = matplotlib.pyplot.subplots()\n",
    "confusion_figure.set_size_inches(5, 3)\n",
    "confusion_axes.set_title(\n",
    "    'Confusion matrix showing the frequency of \\n'\n",
    "    'correct and incorrect bug classification predictions.'\n",
    "    '\\n\\n'  # hack to avoid overlap with x-axis labels below\n",
    ")\n",
    "confusion_axes.xaxis.tick_top()  # move x-axis labels to top of matrix\n",
    "_ = sns.heatmap(\n",
    "    confusion_matrix,\n",
    "    annot=True,\n",
    "    fmt=\"d\",\n",
    "    cmap=sns.color_palette(\"Blues\"),\n",
    "    vmin=0,\n",
    "    ax=confusion_axes,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "tn, fp, fn, tp = metrics.confusion_matrix(y_test, np.rint(y_pred_stack2)).flatten().tolist()\n",
    "\n",
    "fpr_stack2 = fp/(fp+tp)\n",
    "fnr_stack2 = fn/(fn+tn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3651208360548661"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fpr_stack2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.334575260804769"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fnr_stack2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_test, np.rint(y_pred_stack2))\n",
    "precision, recall, thresholds = precision_recall_curve(y_test, np.rint(y_pred_stack2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "auc_keras = auc(fpr_keras, tpr_keras)\n",
    "roc_auc = roc_auc_score(y_test, np.rint(y_pred_stack2))\n",
    "pr_auc = auc(recall, precision)\n",
    "f1 = f1_score(y_test, np.rint(y_pred_stack2))\n",
    "mcc = matthews_corrcoef(y_test, np.rint(y_pred_stack2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PR AUC: 0.738\n",
      "ROC AUC: 0.650\n",
      "MCC: 0.300\n",
      "F1: 0.659\n"
     ]
    }
   ],
   "source": [
    "print('PR AUC: %.3f' % pr_auc)\n",
    "print('ROC AUC: %.3f' % roc_auc)\n",
    "print('MCC: %.3f' % mcc)\n",
    "print('F1: %.3f' % f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stack Layer Above (Model 2) and Convolutional Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import RepeatVector, Permute, Lambda, ActivityRegularization, AveragePooling1D\n",
    "from keras.optimizers import Nadam, Adagrad\n",
    "from keras.layers import Dense, Dropout, Flatten, Reshape\n",
    "from keras.layers import Conv1D, MaxPooling1D, Conv2D\n",
    "from keras.models import Sequential\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.layers.advanced_activations import LeakyReLU\n",
    "from keras.optimizers import RMSprop, Adadelta, Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "repeat_vector_1 (RepeatVecto (None, 3, 128)            0         \n",
      "_________________________________________________________________\n",
      "conv1d_1 (Conv1D)            (None, 1, 64)             24640     \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 1, 64)             0         \n",
      "_________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1 (None, 1, 64)             0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "lambda_1 (Lambda)            (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 128)               8320      \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 33,601\n",
      "Trainable params: 33,601\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "conv_model_stack = Sequential()\n",
    "\n",
    "conv_model_stack.add(Dense(128, input_dim=3, activation='sigmoid'))\n",
    "conv_model_stack.add(RepeatVector(3))\n",
    "\n",
    "conv_model_stack.add(Conv1D(filters = 64, kernel_size = 3, strides = 1, activation ='sigmoid'))\n",
    "conv_model_stack.add(Dropout(0.5))\n",
    "conv_model_stack.add(MaxPooling1D(pool_size = 2, padding = 'same'))\n",
    "\n",
    "conv_model_stack.add(Flatten())\n",
    "\n",
    "conv_model_stack.add(Lambda(lambda x: x**2))\n",
    "conv_model_stack.add(Dense(128, activation='sigmoid'))\n",
    "conv_model_stack.add(Dropout(0.05))\n",
    "\n",
    "conv_model_stack.add(Dense(units=1, activation='sigmoid'))\n",
    "\n",
    "conv_model_stack.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "conv_model_stack.compile(optimizer=Adam(), loss='binary_crossentropy', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Epoch 1/50\n",
      "22982/22982 [==============================] - 1s 52us/step - loss: 0.6698 - acc: 0.5875\n",
      "Epoch 2/50\n",
      "22982/22982 [==============================] - 1s 45us/step - loss: 0.6504 - acc: 0.6404\n",
      "Epoch 3/50\n",
      "22982/22982 [==============================] - 1s 45us/step - loss: 0.6481 - acc: 0.6428\n",
      "Epoch 4/50\n",
      "22982/22982 [==============================] - 1s 46us/step - loss: 0.6472 - acc: 0.6443\n",
      "Epoch 5/50\n",
      "22982/22982 [==============================] - 1s 45us/step - loss: 0.6465 - acc: 0.6469\n",
      "Epoch 6/50\n",
      "22982/22982 [==============================] - 1s 45us/step - loss: 0.6459 - acc: 0.6479\n",
      "Epoch 7/50\n",
      "22982/22982 [==============================] - 1s 46us/step - loss: 0.6459 - acc: 0.6485\n",
      "Epoch 8/50\n",
      "22982/22982 [==============================] - 1s 47us/step - loss: 0.6439 - acc: 0.6494\n",
      "Epoch 9/50\n",
      "22982/22982 [==============================] - 1s 57us/step - loss: 0.6442 - acc: 0.6494\n",
      "Epoch 10/50\n",
      "22982/22982 [==============================] - 1s 55us/step - loss: 0.6438 - acc: 0.6495\n",
      "Epoch 11/50\n",
      "22982/22982 [==============================] - 1s 51us/step - loss: 0.6437 - acc: 0.6502\n",
      "Epoch 12/50\n",
      "22982/22982 [==============================] - 1s 47us/step - loss: 0.6443 - acc: 0.6492\n",
      "Epoch 13/50\n",
      "22982/22982 [==============================] - 1s 45us/step - loss: 0.6433 - acc: 0.6492\n",
      "Epoch 14/50\n",
      "22982/22982 [==============================] - 1s 44us/step - loss: 0.6437 - acc: 0.6495\n",
      "Epoch 15/50\n",
      "22982/22982 [==============================] - 1s 45us/step - loss: 0.6434 - acc: 0.6486\n",
      "Epoch 16/50\n",
      "22982/22982 [==============================] - 1s 46us/step - loss: 0.6421 - acc: 0.6493\n",
      "Epoch 17/50\n",
      "22982/22982 [==============================] - 1s 43us/step - loss: 0.6428 - acc: 0.6493\n",
      "Epoch 18/50\n",
      "22982/22982 [==============================] - 1s 44us/step - loss: 0.6428 - acc: 0.6493\n",
      "Epoch 19/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 0.6424 - acc: 0.6487\n",
      "Epoch 20/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 0.6424 - acc: 0.6483\n",
      "Epoch 21/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 0.6423 - acc: 0.6490\n",
      "Epoch 22/50\n",
      "22982/22982 [==============================] - 1s 40us/step - loss: 0.6423 - acc: 0.6495\n",
      "Epoch 23/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 0.6418 - acc: 0.6501\n",
      "Epoch 24/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 0.6417 - acc: 0.6499\n",
      "Epoch 25/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 0.6412 - acc: 0.6500\n",
      "Epoch 26/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 0.6413 - acc: 0.6496\n",
      "Epoch 27/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 0.6413 - acc: 0.6509\n",
      "Epoch 28/50\n",
      "22982/22982 [==============================] - 1s 44us/step - loss: 0.6418 - acc: 0.6492\n",
      "Epoch 29/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 0.6413 - acc: 0.6500\n",
      "Epoch 30/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 0.6413 - acc: 0.6508\n",
      "Epoch 31/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 0.6414 - acc: 0.6502\n",
      "Epoch 32/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 0.6410 - acc: 0.6500\n",
      "Epoch 33/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 0.6415 - acc: 0.6497\n",
      "Epoch 34/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 0.6415 - acc: 0.6502\n",
      "Epoch 35/50\n",
      "22982/22982 [==============================] - 1s 52us/step - loss: 0.6411 - acc: 0.6494\n",
      "Epoch 36/50\n",
      "22982/22982 [==============================] - 1s 51us/step - loss: 0.6409 - acc: 0.6500\n",
      "Epoch 37/50\n",
      "22982/22982 [==============================] - 1s 47us/step - loss: 0.6416 - acc: 0.6500\n",
      "Epoch 38/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 0.6405 - acc: 0.6486\n",
      "Epoch 39/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 0.6411 - acc: 0.6500\n",
      "Epoch 40/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 0.6413 - acc: 0.6498\n",
      "Epoch 41/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 0.6408 - acc: 0.6509\n",
      "Epoch 42/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 0.6411 - acc: 0.6499\n",
      "Epoch 43/50\n",
      "22982/22982 [==============================] - 1s 40us/step - loss: 0.6411 - acc: 0.6500\n",
      "Epoch 44/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 0.6413 - acc: 0.6496\n",
      "Epoch 45/50\n",
      "22982/22982 [==============================] - 1s 40us/step - loss: 0.6408 - acc: 0.6506\n",
      "Epoch 46/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 0.6408 - acc: 0.6503\n",
      "Epoch 47/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 0.6407 - acc: 0.6512\n",
      "Epoch 48/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 0.6407 - acc: 0.6504\n",
      "Epoch 49/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 0.6408 - acc: 0.6493\n",
      "Epoch 50/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 0.6408 - acc: 0.6496\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f3ad1e8da58>"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conv_model_stack.fit(S_train, y_train, epochs=50, batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_stack_conv = conv_model_stack.predict(S_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5746/5746 [==============================] - 0s 34us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.6386029740015555, 0.6526279150506075]"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conv_model_stack.evaluate(S_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAEFCAYAAABjDmX4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZhU1Z3/8fe3u2n2TVBZxVbccElcwCiiGJeA0eBMonGNOBqixiQucZv4G4njQpwkmploRtyjiYQ4LiRxiUZxSVTQaEQEYwsItOxLyw4N398f5zTeLqqquy9NVxd8Xs/TD1Xnbueee++n7r1V92DujoiINE5JoSsgIlKMFJ4iIikoPEVEUlB4ioikoPAUEUlB4SkiksJ2G55m1tbM/mBm1Wb2+62Yz9lm9uemrFuhmNkQM/uwiec52sweacp5ZlnGM2Z23rZcRmJZTbY+TbUPbq/MbLCZfWRmK83s1ELXp7EKHp5mdpaZvRUbcF48UI5qgll/A9gV6Obup6Wdibv/xt1PbIL6bFNm5mbWP9847v6qu+/TXHVqKu4+3N0faur5mtlQM5vb1PNNaJJ9cDt2I/BLd+/g7k8WujKNVdDwNLMrgDuAWwg72W7AXcCIJph9P+Cf7l7TBPMqemZWVug67IDy7oPaJvQDpha6Eqm5e0H+gM7ASuC0POO0JoTrp/HvDqB1HDYUmAtcCSwE5gHnx2E/BtYDG+IyLgBGA48k5r074EBZfD8SmAGsAGYCZyfKX0tMdyQwGaiO/x6ZGDYR+E/gr3E+fwa651i32vpfnaj/qcBJwD+BpcC/J8YfBLwOLI/j/hIoj8NeieuyKq7vNxPzvwaYDzxcWxan2TMu45D4vhewCBiao77XAFVxvT4Ejovlo4HxwK/jsKnAYYnp9ovtsjwO+1osr4hlJfH9PcDCxHQPA5cl2vXC5PYAfgosi9tqeGK6itgeK4AXgDuT2z0xXntgDbApttnK2Ab1rU8v4P9iW80Evp+jvbLtgyPjvnE7sAS4KY77b8C0uD7PAf0S8zkBmE7Y334JvJxoi9Hk36c7A/cR9pcq4CagtIHtuBPwAOG4WwY8GcvfB05JjNcKWAwcnKMdvg1UEva1CUCvWP5xbPs1sX1aFyqLUmdYwRYMw4Ca2g2dY5wbgTeAXYCdgb8B/xmHDY3T3xg34EnAaqBrjh0r544WD6TPgH3isJ7A/smdLLFDLQPOjdOdGd93SxzkHwN7A23j+zE51q22/v8R6/9twgH5W6AjsH/csSri+IcCX4rL3Z1wsF2WmJ8D/bPM/yeED6G2JMIzsWN/ALQjHLQ/zVHXfYA5iR1/d2DPRLuuje1fCtwKvJE4sCqBfwfKgS8TAqm2nWcDh8bXHxI+vPZLDDs40a7J8NwQ614KXEw4wC0Of50QCOXAUXG7bhGeiTaam1GWb31KgLfjNisH9oh1/kqO+Y+m7j43Mm6T78Xt2JZwlVVJ+JApA64H/hbH7x7b6xuxLS+P0zc0PJ8A7ibs37sAk4DvNLAd/wT8Dugal31MLL8a+F1imSOAKTnW/8uEYD2EsA/+D/BKYvgs4PhCZdBWZ1jBFgxnA/PrGedj4KTE+68AsxI7/hoS4Us4g/tSjh0r544Wd67lwNeBthl1GMnn4XkuMClj+OvAyPh6InB9YtglwLM51q22/rVnAh1jfQ5PjPM2cGqO6S8Dnki8zxae64E2GWWZYTEBmAK8R45Pf6B/bNvjgVYZw0YDLyTeDwDWxNdDCGe9JYnhjwKj4+uHgSuAHoTwvA24iC3PSidSNzwrE/NrF9e9B+G2Tw3QLjH8ERofnrnW53Bgdsb41wEP5Jh/5j43Msv0zwAXJN6XEE4C+gHfIgZ3HGaEq4l6w5NwG2wdif2Z8GH/UgPasSfhrLBrlnXqRQj0TvH9Y8DVOdb/PuC2xPsOhMDePb6fRRGHZyHveS4Butdz36cX8Eni/SexbPM8vO79pNWEDdQo7r6KcKl7ETDPzP5kZvs2oD61deqdeD+/EfVZ4u4b4+s18d8FieFraqc3s73N7I9mNt/MPiPcJ+6eZ94Ai9x9bT3j3AMcAPyPu6/LNoK7VxLCejSw0MzGmVlyO2Suc5u4XXsBc9x9U2J4sr1eJgTY0YRL7YnAMfHv1YzpkjYvz91Xx5cd4vKWJsognDE3Vq716Qf0MrPltX+Es+pdGzHvzPr0A36RmN9SQkj2JrZf7YgeEqeh69OPcMY4LzHvuwlnoLVytWNfQjsuy5ypu39KuPXwdTPrAgwHfpOjDnWOF3dfSTjue+cYv6gUMjxfJ3wy5vuJwqeEnaDWbrEsjVWET9daPZID3f05dz+B8Kk7nRAq9dWntk5VKevUGL8i1Gsvd+9EOGitnmk830Az60C4j3wfMNrMdso5I/ffuvtRhPV3wu2A+nwK9DWz5H6WbK+XCWenQ+Pr14DBhPB8uQHzzzQP2MnMktu5b57x87ZPFnOAme7eJfHX0d1PasQ8Mpc5h3ApnZxnW3f/G2F9NtffzIy665Nvn55DOL66J+bbyd33b+B67hTDMZuHgHOA04DX3T3X/l/neDGz9kA3mud42eYKFp7uXk24d3SnmZ1qZu3MrJWZDTez2+JojwLXm9nOZtY9jp/2N3jvAkeb2W5m1plwuQWAme1qZiPixl1HuIGd7aznaWDv+POqMjP7JuGy7o8p69QYHQn371bGs+KLM4YvINyDa4xfAG+5+4WEe1z/m20kM9vHzL5sZq0J9wNrv2ipz5uEM7er47YdCpwCjANw94/ivM4BXnb3z+J6fJ0U4enunwBvET4Iys3siLi8XBYA3eL+0BCTgBVmdk38DWepmR1gZgMbW9eE/wWuM7P9Acyss5nV/qzpT8D+Zvav8cz3+9QNyJz7tLvPI3xh+TMz62RmJWa2p5kdU1+F4rTPAHeZWde47Y5OjPIk4T7mDwhfrOXyKHC+mX0x7ju3AG+6+6z66lAMCvpTJXf/GeGe1/WEL0vmAJcSNg6EbwffItyPmwL8PZalWdbzhBvg7xHuJSYDryTW41PCZdMxbBlOuPsS4GTCN/xLCDfPT3b3xWnq1Eg/BM4i3G+6h7AuSaOBh+Il2un1zczMRhC+tKtdzyuAQ8zs7CyjtwbGEG7+zydc+l2XZbw63H09IbyGx2nvAr7l7tMTo71MuH0xJ/HeCNs6jbOBI4jfZhPaKdftiOmEA3xGbLde2cZLjL+RsP2/SPh2ejFwL+Fb7VTc/QnCWfy4eDvmfUJ7Efer0whtvwTYi3DJXDttvn0awj3TcsKXgssI9yd7NrBq5xLuT04n3O++LLHcNYRfHFQAj+dZtxeA/xfHnUf4hccZDVx+i1f7zZrIdsnMfgdMd/cbCl2XpmBmEwlfEt1b4Hr8B7C3u59TyHoUUsGfMBJpSmY2MF6elpjZMMJPaYru6ZWWLN4bvwAYW+i6FJLCU7Y3PQjf2q8E/hu42N3fKWiNtiNm9m3C7bVn3P2VQtenkHTZLiKSgs48RURSUHg2QH2978QeoRr7M6GiYmYTzezCAtdhm3Z/Z2ZT48+psOABM1tmZpNsG3TnF5ezW9x/Spt63ttacp+wrei60Zqxy8GmVJTh2QxdiTWKhy61ZhS6HmmZ2Ugze63Q9Sg0d9/f3SfGt0cROuXo4+6DvIm68zOzWWZ2fGKZs+P+szHfdC2dN7DrxmwfgL6Nuhzc1polPLM9gqnuuNJTezaLfoR+FFYVuiLNQftPCvU9/E54HOxxwo/YlxA6L4UQvNcTnl1dSHjSoLPX7aDgAkLvOK9kK4vjfonQW9Jy4B8kukQjS7dY5OhKLEu9vwq8Q3gqZw6xM4qM+p0X67IY+FFieFvgwbjMD4CryOhAImNZmzvliNPdSXg6ZAXhKZs9E+PuDzxP+DH+AmK3czSs+70tupdLlsVxTyY8ebI8tutB+bYloTeftcDG2JbLc6zjREIPQ5Nimz4F7JSsX8b4s4idPsT2fCi25zTCwwX52jNXG42mbkcYv4/rXk3Yx/ZPDDspbrsVhMcBfxjLuxN+TF77HPmrfN4BySxC5ycXZLTJjzPXMVtbxvI9gRdj2WLCc99d4rCHqdsN29Vs2RNSL0JnLUsJvS19O7HM0eTpLi/Hfvl9Qs9Pi4H/SqzrSJq2e7yR1O26cYttSHgoI9lN3z8S+1btfBqSK7mO20GEh2pqn1T7eX35tjV/9QVnKSHQbieEVhvgqEQjVxIeCewQd6TaA7h2JX8dp2ubo6x33HAnxUY7Ib7fOc4nV7dYQ8lz8CXGOTDO96DYmKdm1O+eWI8vEJ5Cqe0ObQzhoNqJcJC8n295bBmeS+KGLCMcPOPisI6EJy2ujG3ZkdiLEg3rfi+ze7nMsoPjDnd43HbnEQKhdT3bciSJHT9PeFYROhFpT3hq5JEGhucYwoHWFehDeCIma3vW00ajqRue/xaH137wvJsYNg8YEl935fN+S28lPBLZKv4N4fNfnSTrXKdNkutYT1v2J+zHreN2fAW4I1u7ZOyLteH5CuFJrDaEJ5kWAV9OrH/W7vLy7JcvEfbj3Qj9xCbDrim7x9vcXo3ZhlnCsyG5kuu4fR04N77uQOxhrVDheUTceFv0uQn8Bbgk8X4fwidKWWIl98iykyTLrqltmETZc4SDPl+3WEOpJzyzTHMHcHtGXfokhk8CzoivZwDDEsNG5VseW4bnvYlhJxGecIHQJdg7OeZRX/d72bqXyyz7FTFwE2UfEh43zbctR9Kw8ByTeD8gLr802/agbhDV6fMSuDBXe9bTRqPJ3b1cl7gdas9SZgPfIXadlhjvRsJZc/8s80jWuU6bUDc8c7Zllnmemlwf8oQn4YN6I9AxMfxW4MHE+mftLi/Pfpncjy8B/pJYv6bsHm9zezV2G1I3PBuSK7mO21cIVwlZOyBv6r/67nn2BT7x7P+NQLbu4mr7EayVrfusZFk/4LSMLr6OIgRnzm6xGsLMDjezl8xskZlVE7qby+zCLVf3cXW6AmPLbujqk2u+fQkhmU193e9l614us6wfcGVGe/aN88m3LRsqs01aUX+3eLBle+brVi1fG20WO+UYY2Yfx2fCZ8VBtfX5OuGD6xMzezl2EgLh0rUS+LOZzTCzaxtQ/2x1zNqWsZOZcWZWFev1CA1rI/i8S70VibL6ujxsU8/9ysxt1ivHMGi67vEatA1zaEiu5Dq+LiB0RD7dzCab2ckp69Ag9YXnHGC3HBsnW3dxNdTtj9KzTJcsm0M480x2x9Xe3ceQv1usbPPN9FvCvaO+7t6ZcKlWXxdutep0BUZYt6Ywh9w9H9XX/V59bVk7/5sz2rOduz9K/m3ZkPaELdtkA+G+U52u0eLPbnZOjDuPcLmebT6Z8rVR0lmEy8zjCR1z7F67eAB3n+zuIwi3QZ4k3CvE3Ve4+5XuvgfwNeAKMzuuAcvLrGOutryF0J4Heug68Bzq7nf52vpTwj7fMVG2tV0eZm6zfPvU1nSPlzmfXNuwvn2tIbmSfcbuH7n7mYRt/hPgMQs9pW0T9YXnJEKjjTGz9mbWxswGx2GPApebWYWFfiFvIXTP35gzm0eAU8zsK/FMok38GVIfz98tVkO6EutI+BRfa2aDCAdbQ40ndBPW1cz6EO4LNYU/Aj3N7DIza21mHc3s8DisKbrfuwe4KJ51W9xmX40HY75tuQDoY2bl9cz/HDMbYKG/zBuBxzz8xOafhDOgr5pZK8K9staJ6ZLt2ZvQc1Yu+dooqSPhftcSQnDfUjvAQnd0Z5tZZ3ffQPgCYVMcdrKZ9Y8HfzXhMrkh3esl5WvLjoQvQ6rjul6VMW3OrgM99Cz1N+DWOM+DCGdTW/Pb1qtiu/cldCGX2RtX0tZ0j5eUbxsuAHa3un28JqXOFTM7x8x29tCJ9vJY3Nht22B5wzMeGKcQboLPJtzj+GYcfD/h28NXCN1zraWRIRN3lhGEb+Jqu6S7KlGvrN1iecO6ErsEuNHMVhCCaHwjqvZjwuXCTEKfiA83Zr1yiZdjJxDadD7wEXBsHLzV3e+5+1uE/5Pml4RvSysJ96Lq25YvEr65nW9m+brXe5hwT3c+4YuA78d5VxPa+17CWdKqOP9aN8b3Mwn/Kdtj5O4mLl8bJf2asI2qCN+qv5Ex/FxgVrx0vojQVR2Ebt1eIATc68Bd7v5SnnXOVsd8bfljQl+X1YTAyeyy7VbCh+RyM/thltmfSTiL/pTwfxDd4KFrt7SeInRX926sz325RvSt6B4vYz75tmHt/1+/xMyydTu4NbkyDJhqZisJfdWe4aH7vNoHWYY0cD4NomfbpdmZ2cWEHfuYQtdle2ZmTvifByoLXZftUVE+YSTFxcx6mtlgC93E7UP4CcsTha6XyNbQUwXSHMoJ//lY7f+KOY7wW0aRoqXLdhGRFHTZLiKSgsJTRCQFhaeISAoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIikoPEVEUlB4ioikoPAUEUlB4SkikoLCU0QkBYWniEgKCk8RkRQUniIiKSg8RURSUHiKiKSg8BQRSUHhKSKSgsJTRCQFhaeISAoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQeFZBMxsmJl9aGaVZnZtoesjDWdm95vZQjN7v9B1kaal8GzhzKwUuBMYDgwAzjSzAYWtlTTCg8CwQldCmp7Cs+UbBFS6+wx3Xw+MA0YUuE7SQO7+CrC00PWQpqfwbPl6A3MS7+fGMhEpIIWniEgKCs+Wrwrom3jfJ5aJSAEpPFu+ycBeZlZhZuXAGcCEAtdJZIen8Gzh3L0GuBR4DpgGjHf3qYWtlTSUmT0KvA7sY2ZzzeyCQtdJmoa5e6HrICJSdHTmKSKSgsJTRCQFhaeISAoKTxGRFBSeIiIpKDyLhJmNKnQdJD1tv+2PwrN46OArbtp+2xmFp4hICtvNj+Q7nP7g9rEiOWyYMZFWewwtdDW2iWHDDyp0Fba5GRMfZ4+h/1roamwzj51/iKWZrr7jduX4kanm2xx05lkkttfg3FFsz8G5o1J4ioikoPAUEUlB4SkikoLCU0QkBYWniEgKCk8RkRQUniIiKSg8RURSUHiKiKSg8BQRSUHhKSKSgsJTRCQFhaeISAoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpKCwlNEipaZ3W9mC83s/Yzy75nZdDObama3JcqvM7NKM/vQzL6SKB8WyyrN7NqGLLus6VZDRKTZPQj8Evh1bYGZHQuMAL7g7uvMbJdYPgA4A9gf6AW8YGZ7x8nuBE4A5gKTzWyCu3+Qb8EKTxEpWu7+ipntnlF8MTDG3dfFcRbG8hHAuFg+08wqgUFxWKW7zwAws3Fx3Lzhqct2EWmxzGyUmb2V+BvVgMn2BoaY2Ztm9rKZDYzlvYE5ifHmxrJc5XnpzFNEWix3HwuMbeRkZcBOwJeAgcB4M9ujqeum8BSR7c1c4HF3d2CSmW0CugNVQN/EeH1iGXnKc1J4ikjB7L53vVfHaTwJHAu8FL8QKgcWAxOA35rZzwlfGO0FTAIM2MvMKgiheQZwVn0LUXiKSNEys0eBoUB3M5sL3ADcD9wff760HjgvnoVONbPxhC+CaoDvuvvGOJ9LgeeAUuB+d59a37IVniJStNz9zByDzskx/s3AzVnKnwaebsyy9W27iEgKCk8RkRQUniIiKSg8RURSUHiKiKSg8BQRSUE/VSqguy4ezPBD+rCoei2DfvgUAAf224lffPsI2pSXUrNxE5ff+wZvf7yYTm1bce/3j6Zvt/aUlRq/+MNUHplYCUCfbu2586Ij6dOtPQ78660vMHvRygKu2fbvksG7cWjfzlSvreGKJ6cB0KG8lMuHVrBLx3IWrljPzyfOZNX6jfTq3JrvHtWPPbq149G/f8qE9xduns/JA3bhuL274cDsZWu487VP2LDRC7RW0hg68yyg30ys5NRbnq9TdtM5h3LrY+9y5NUTuGn8O9x0zmEAjBq2L9PnLueIqycwfPSz3PKtgbQqDZvvnkuHcMeE9zn0iic55ro/sqh6TbOvy47mpcql3PR8ZZ2yUw/qwZR5K/je/33AlHkr+JeDdgVg5bqN3P/m3DqhCbBTu1YMH7Az1/xhOlc8OY0SMwZXdG22dZCt02LC08z2NbNrzOy/4981ZrZfoeu1Lf112gKWrVxfp8wdOrVtBUDnduXMW7Z6c3nHNqG8fZtWLFu5jppNm9i3d2dKS42XpswDYNW6Gtas39iMa7FjmrZgJSvX1W3ngbt1ZmLlEgAmVi5h4G5dAPhsbQ0fL17Nxk1bnlGWlhjlpSWUGLQuK2HZ6g3bvvLSJFrEZbuZXQOcCYwjPGsK4eH8R81snLuPKVjlmtk1D03iyR+dwM3nDqSkBI67Pjz0cPez0xh/9XFU3n06Hdq24rzbX8Yd+vfqTPWq9fz2ymPpt0sHXpoyj//4zdtscl36NbcubcpYvqYGgOVraujSJv/htXT1Bia8v4BfnX4A6zdu4r2qFfzj0xXNUVVpAi3lzPMCYKC7j3H3R+LfGEJHpRfkmijZ19+GGRObq67b1IUn7sO1D01m30t+z7UPTeauiwYDcPwXevPeJ0vp/53xHHnVBH52weF0bNuKshLjyP125d8fnszR1/2Ril07cM7Q/gVeCwGo7+OrfXkpA3frwnd/P5VR46bQuqyEIXvs1Cx1k63XUsJzE6GXk0w947Cs3H2sux/m7oe12mPoNqpa8zrrmP489eYnADz++iwO7d8dgHOO7c+EWD5jwQo+WbiSvXt1pmrpaqbMWsqshSvZuMn5w6TZfFEHYEEsX1tDl7bhbLNL2zKq19bkHf+gXh1ZuGIdn62rYaPDm58sZ59d2jdHVaUJtJTwvAz4i5k9Y2Zj49+zwF+AHxS4bs1q/tLVDBnQA4ChB/Tk4/mfATB38SqGHhg+X3bp3Ia9enVi1sIVvF25mM7tyunesTUAxxzQk+lzqwtT+R3cW7OrGdq/GwBD+3dj8uz822HxyvXsvXN7yksNgAN7daSqeu02r6c0DfMWcm/MzEoIl+m1HfxVAZNru4yqT4fTH2wZK9IID/zgaIYM6EG3jm1YWL2Gm8e/y0efVnPb+YMoKylh7YaNXH7vG7w7cwk9urbl7kuOokfXdhjws6em8LtXZwBw7IE9ufVbAzEz3pmxhO/d/Tc2bMx5wt7iDBt+UKGr0GiXHbM7+/foSMc2ZVSv2cDv3pnHpNnLuXJoBd07lLNo5Xp+/tJMVq7fSJe2ZfzklH1p26oUd2dtzSYue+ID1mzYxOlf7Mngiq5sdGfmktX86q+zqcnyxVJL99j5h1ia6Q64/vm8K/v+TSekmm9zaDHhubWKMTwlKMbwlLp2xPBsKZftIiJFReEpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIikoPEVEUlB4ioikoPAUEUlB4SkikkKL6M9TRHZM++7ZrdBVSE1nniJStMzsfjNbaGbvJ8r+y8ymm9l7ZvaEmXVJDLvOzCrN7EMz+0qifFgsqzSzaxuybIWniBSzB4FhGWXPAwe4+0HAP4HrAMxsAHAGsH+c5i4zKzWzUuBOYDgwADgzjpuXwlNEipa7vwIszSj7s7vX9kT9BuG/9AEYAYxz93XuPhOoJHSDOQiodPcZ7r6e8N8Bjahv2QpPEWmxkv/VTvwb1chZ/BvwTHzdG5iTGDY3luUqz0tfGIlIi+XuY4GxaaY1sx8BNcBvmrRSkcJTRLY7ZjYSOBk4zj/v8b0K6JsYrU8sI095TrpsF5HtipkNA64GvubuqxODJgBnmFlrM6sA9iL8V+eTgb3MrMLMyglfKk2obzk68xSRomVmjwJDge5mNhe4gfDtemvgeTMDeMPdL3L3qWY2HviAcDn/3dr/I83MLgWeA0qB+919an3LVniKSNFy9zOzFN+XZ/ybgZuzlD8NPN2YZeuyXUQkBYWniEgKCk8RkRQUniIiKSg8RURSUHiKiKSg8BQRSUHhKSKSgsJTRCQFhaeISAoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpLCdtMZ8rDhBxW6CpLSs8+8V+gqyNY6/5BC16DZbTfhKSLFZ9DunQtdhdR02S4ikoLCU0QkBYWniEgKCk8RkRQUniIiKSg8RURSUHiKiKSg8BSRomVml5vZVDN738weNbM2ZlZhZm+aWaWZ/c7MyuO4reP7yjh8961ZtsJTRIqSmfUGvg8c5u4HAKXAGcBPgNvdvT+wDLggTnIBsCyW3x7HS03hKSLFrAxoa2ZlQDtgHvBl4LE4/CHg1Ph6RHxPHH6cmVnaBSs8RaQouXsV8FNgNiE0q4G3geXuXhNHmwv0jq97A3PitDVx/G5pl6/wFJEWy8xGmdlbib9RiWFdCWeTFUAvoD0wrLnqpo5BRKTFcvexwNgcg48HZrr7IgAzexwYDHQxs7J4dtkHqIrjVwF9gbnxMr8zsCRt3XTmKSLFajbwJTNrF+9dHgd8ALwEfCOOcx7wVHw9Ib4nDn/R3T3twhWeIlKU3P1Nwhc/fwemEPJsLHANcIWZVRLuad4XJ7kP6BbLrwCu3Zrl67JdRIqWu98A3JBRPAMYlGXctcBpTbVsnXmKiKSg8BQRSUHhKSKSgsJTRCQFhaeISAoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQY9nikjBHNyjc6GrkJrOPEVEUlB4ioikoPAUEUlB4SkikoLCU0QkBYWniEgKCk8RkRQUniIiKehH8i3ESQN25vi9u2PAC/9czJ8+WMTlQyvo1ak1AO3LS1m1fiNXTZhOqcHFR/Wjols7Ss14uXIJT0xZUNgV2MHcdfFghh/Sh0XVaxn0w/CfMx7Ybyd+8e0jaFNeSs3GTVx+7xu8/fFiAIYM6MFPRg6iVamxZMU6ho1+dvO8Ssx4dczJfLp0Naf95C8FWR9pPIVnC9C3SxuO37s71/5hOjWbnOtP7M/bcz7j9okzN4/zrYG9Wb1+IwBHVHSlVYlx5ZPTKC817viXAbw2cxmLVq4v1CrscH4zsZK7n53GPd8dsrnspnMO5dbH3uX5d6s48eDe3HTOYQz/8bN0bppRJOcAAAYPSURBVFfO7Rd+iVNvfp65S1axc6c2deZ1yUn78WFVNR3btmru1ZCtoMv2FqBPlzZ8tGgV6zc6mxw+mL+Sw/t1qTPOkRVdeW3mMgDcoXVZKSUG5WUl1Gxy1sRglebx12kLWJbxYeUOnWIAdm5XzrxlqwE4/agKJrz5CXOXrAJg0WdrN0/Ta6d2DDukDw/95Z/NVHNpKi3+zNPMznf3Bwpdj21p9rK1nHlILzq0LmV9zSYO7tOJjxev3jx8v107UL1mA/M/WwfAG7OWMWi3ztxzxoG0Li3hwUlzWanwLLhrHprEkz86gZvPHUhJCRx3/dMA9O/ZmVZlxjM3DKND2zLuenoaj77yMQC3jRzE9Y+8rbPOIlQMZ54/zjXAzEaZ2Vtm9taMiY83Z52aVFX1Wp6csoD/d+JeXH9if2YtXcMm/3z4UXt05bUZyza/779zezY5jBo3hUsem8opB+zKLh3KC1BzSbrwxH249qHJ7HvJ77n2ocncddFgAMpKjS9WdOfrY17g1Juf55qvf4H+PTsxLN4zfXfmkgLXXNJoEWeeZvZerkHArrmmc/exwFiAbzzwd881XjF48aMlvPhROIjOOqQXS1aHS8ISg8P7deHqCdM3jztkj514p+ozNjp8traGDxesZM/u7Vioe54FddYx/bnqgUkAPP76LH75nSMBqFqymqUrqli9robV62r467T5HNivK1+o6MZJh/XlxIP70Ka8lI5tW3Hv94Zw4f+8WsC1kIZqKWeeuwLfAk7J8rdDfCx3ahM+x7q3b8Xh/brwajzTPKhXJ6qq17J09YbN4y5etZ4DenYEoHVZCXvt0p5Pq9c1f6WljvlLVzNkQA8Ahh7Qk4/nfwbAn96azRH77kJpidG2vJSB/Xfmw6pqRj/6d/a5+Pfsf+ljjLzjZV5+f56Cs4i0iDNP4I9AB3d/N3OAmU1s/uo0v6uO3YMObUrZuMm59405m79ZH1zRlb8mLtkBnp22iO8e1Y/bT90PDF76aAmfLFtTiGrvsB74wdEMGdCDbh3b8OGvTuPm8e9y6d1/47bzB1FWUsLaDRv53t2vA/BhVTXPv1vFmz8dwaZNzoMvfsQHc5YXeA22H2ZWCrwFVLn7yWZWAYwDugFvA+e6+3ozaw38GjiUcFL2TXeflXq57kV9tbtZsV+278iefSbXXRspFivHj7Q00z0/bXHe4/aE/brXO18zuwI4DOgUw3M88Li7jzOz/wX+4e6/MrNLgIPc/SIzOwP4F3f/Zpp6Q8u5bBcRaTQz6wN8Fbg3vjfgy8BjcZSHgFPj6xHxPXH4cXH8VBSeItJiJX9RE/9GZYxyB3A1sCm+7wYsd/ea+H4u0Du+7g3MAYjDq+P4qbSUe54iIltI/qImk5mdDCx097fNbGizVgyFp4gUr8HA18zsJKAN0An4BdDFzMri2WUfoCqOXwX0BeaaWRnQma34NY8u20WkKLn7de7ex913B84AXnT3s4GXgG/E0c4DnoqvJ8T3xOEv+lZ8Y67wFJHtzTXAFWZWSbineV8svw/oFsuvAK7dmoXosl1Eip67TwQmxtczgEFZxlkLnNZUy9SZp4hICgpPEZEUdNkuIgVT0a19oauQms48RURSUHiKiKSg8BQRSUHhKSKSgsJTRCQFhaeISAoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIikoPEVEUlB4ioikoPAUEUlB4SkikoLCU0QkBYWniEgKCk8RKVpmNszMPjSzSjO7tjmXrfAUkaJkZqXAncBwYABwppkNaK7lKzxFpFgNAirdfYa7rwfGASOaa+EKTxEpVr2BOYn3c2NZsyhrrgVta4+df4gVug6S0vmHFLoGUiD9d2mb97g1s1HAqETRWHcfu21r1TDbTXiKyPYnBmWusKwC+ibe94llzUKX7SJSrCYDe5lZhZmVA2cAE5pr4TrzFJGi5O41ZnYp8BxQCtzv7lOba/nm7s21LBGR7YYu20VEUlB4ioikoPAUEUlB4SkikoLCU0QkBYWniEgKCk8RkRQUniIiKfx/+0N6PuzIMWYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.DataFrame(\n",
    "    data=metrics.confusion_matrix(y_test, np.rint(y_pred_stack_conv)), \n",
    "    \n",
    ")\n",
    "\n",
    "confusion_figure, confusion_axes = matplotlib.pyplot.subplots()\n",
    "confusion_figure.set_size_inches(5, 3)\n",
    "confusion_axes.set_title(\n",
    "    'Confusion matrix showing the frequency of \\n'\n",
    "    'correct and incorrect bug classification predictions.'\n",
    "    '\\n\\n'  # hack to avoid overlap with x-axis labels below\n",
    ")\n",
    "confusion_axes.xaxis.tick_top()  # move x-axis labels to top of matrix\n",
    "_ = sns.heatmap(\n",
    "    confusion_matrix,\n",
    "    annot=True,\n",
    "    fmt=\"d\",\n",
    "    cmap=sns.color_palette(\"Blues\"),\n",
    "    vmin=0,\n",
    "    ax=confusion_axes,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "tn, fp, fn, tp = metrics.confusion_matrix(y_test, np.rint(y_pred_stack_conv)).flatten().tolist()\n",
    "\n",
    "fpr_stack_conv = fp/(fp+tp)\n",
    "fnr_stack_conv = fn/(fn+tn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3532269257460097"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fpr_stack_conv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3414804469273743"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fnr_stack_conv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_test, np.rint(y_pred_stack_conv))\n",
    "precision, recall, thresholds = precision_recall_curve(y_test, np.rint(y_pred_stack_conv))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "auc_keras = auc(fpr_keras, tpr_keras)\n",
    "roc_auc = roc_auc_score(y_test, np.rint(y_pred_stack_conv))\n",
    "pr_auc = auc(recall, precision)\n",
    "f1 = f1_score(y_test, np.rint(y_pred_stack_conv))\n",
    "mcc = matthews_corrcoef(y_test, np.rint(y_pred_stack_conv))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PR AUC: 0.736\n",
      "ROC AUC: 0.653\n",
      "MCC: 0.305\n",
      "F1: 0.651\n"
     ]
    }
   ],
   "source": [
    "print('PR AUC: %.3f' % pr_auc)\n",
    "print('ROC AUC: %.3f' % roc_auc)\n",
    "print('MCC: %.3f' % mcc)\n",
    "print('F1: %.3f' % f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RF to CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_rf_s = pd.DataFrame(rf.predict(x_train))\n",
    "y_pred_rf_s = y_pred_rf_s.set_index(y_train.index)\n",
    "y_pred_rf_s = np.rint(y_pred_rf_s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_4 (Dense)              (None, 128)               256       \n",
      "_________________________________________________________________\n",
      "repeat_vector_2 (RepeatVecto (None, 3, 128)            0         \n",
      "_________________________________________________________________\n",
      "conv1d_2 (Conv1D)            (None, 1, 64)             24640     \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 1, 64)             0         \n",
      "_________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1 (None, 1, 64)             0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "lambda_2 (Lambda)            (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 128)               8320      \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 33,345\n",
      "Trainable params: 33,345\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "conv_model_stack_1 = Sequential()\n",
    "\n",
    "conv_model_stack_1.add(Dense(128, input_dim=1, activation='sigmoid'))\n",
    "conv_model_stack_1.add(RepeatVector(3))\n",
    "\n",
    "conv_model_stack_1.add(Conv1D(filters = 64, kernel_size = 3, strides = 1, activation ='sigmoid'))\n",
    "conv_model_stack_1.add(Dropout(0.5))\n",
    "conv_model_stack_1.add(MaxPooling1D(pool_size = 2, padding = 'same'))\n",
    "\n",
    "conv_model_stack_1.add(Flatten())\n",
    "\n",
    "conv_model_stack_1.add(Lambda(lambda x: x**2))\n",
    "\n",
    "conv_model_stack_1.add(Dense(128, activation='sigmoid'))\n",
    "conv_model_stack_1.add(Dropout(0.05))\n",
    "\n",
    "conv_model_stack_1.add(Dense(units=1, activation='sigmoid'))\n",
    "\n",
    "conv_model_stack_1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "conv_model_stack_1.compile(optimizer=Adam(), loss='binary_crossentropy', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "22982/22982 [==============================] - 1s 56us/step - loss: 0.3283 - acc: 0.8278\n",
      "Epoch 2/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 0.0032 - acc: 1.0000\n",
      "Epoch 3/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 0.0010 - acc: 1.0000\n",
      "Epoch 4/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 5.5445e-04 - acc: 1.0000\n",
      "Epoch 5/50\n",
      "22982/22982 [==============================] - 1s 45us/step - loss: 3.5900e-04 - acc: 1.0000\n",
      "Epoch 6/50\n",
      "22982/22982 [==============================] - 1s 46us/step - loss: 2.5353e-04 - acc: 1.0000\n",
      "Epoch 7/50\n",
      "22982/22982 [==============================] - 1s 46us/step - loss: 1.6759e-04 - acc: 1.0000\n",
      "Epoch 8/50\n",
      "22982/22982 [==============================] - 1s 46us/step - loss: 1.4490e-04 - acc: 1.0000\n",
      "Epoch 9/50\n",
      "22982/22982 [==============================] - 1s 47us/step - loss: 9.6429e-05 - acc: 1.0000\n",
      "Epoch 10/50\n",
      "22982/22982 [==============================] - 1s 46us/step - loss: 9.0841e-05 - acc: 1.0000\n",
      "Epoch 11/50\n",
      "22982/22982 [==============================] - 1s 44us/step - loss: 6.3589e-05 - acc: 1.0000\n",
      "Epoch 12/50\n",
      "22982/22982 [==============================] - 1s 44us/step - loss: 6.3862e-05 - acc: 1.0000\n",
      "Epoch 13/50\n",
      "22982/22982 [==============================] - 1s 44us/step - loss: 3.7354e-05 - acc: 1.0000\n",
      "Epoch 14/50\n",
      "22982/22982 [==============================] - 1s 44us/step - loss: 4.0092e-05 - acc: 1.0000\n",
      "Epoch 15/50\n",
      "22982/22982 [==============================] - 1s 45us/step - loss: 2.7102e-05 - acc: 1.0000\n",
      "Epoch 16/50\n",
      "22982/22982 [==============================] - 1s 52us/step - loss: 2.7711e-05 - acc: 1.0000\n",
      "Epoch 17/50\n",
      "22982/22982 [==============================] - 1s 53us/step - loss: 1.7768e-05 - acc: 1.0000\n",
      "Epoch 18/50\n",
      "22982/22982 [==============================] - 1s 53us/step - loss: 1.4096e-05 - acc: 1.0000\n",
      "Epoch 19/50\n",
      "22982/22982 [==============================] - 1s 50us/step - loss: 1.4301e-05 - acc: 1.0000\n",
      "Epoch 20/50\n",
      "22982/22982 [==============================] - 1s 45us/step - loss: 8.2947e-06 - acc: 1.0000\n",
      "Epoch 21/50\n",
      "22982/22982 [==============================] - 1s 46us/step - loss: 7.4808e-06 - acc: 1.0000\n",
      "Epoch 22/50\n",
      "22982/22982 [==============================] - 1s 46us/step - loss: 6.0320e-06 - acc: 1.0000\n",
      "Epoch 23/50\n",
      "22982/22982 [==============================] - 1s 45us/step - loss: 5.5114e-06 - acc: 1.0000\n",
      "Epoch 24/50\n",
      "22982/22982 [==============================] - 1s 45us/step - loss: 4.9823e-06 - acc: 1.0000\n",
      "Epoch 25/50\n",
      "22982/22982 [==============================] - 1s 45us/step - loss: 1.6140e-04 - acc: 1.0000\n",
      "Epoch 26/50\n",
      "22982/22982 [==============================] - 1s 44us/step - loss: 1.5167e-05 - acc: 1.0000\n",
      "Epoch 27/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 4.7928e-06 - acc: 1.0000\n",
      "Epoch 28/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 3.1310e-06 - acc: 1.0000\n",
      "Epoch 29/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 3.2006e-06 - acc: 1.0000\n",
      "Epoch 30/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 1.7615e-06 - acc: 1.0000\n",
      "Epoch 31/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 2.1195e-06 - acc: 1.0000\n",
      "Epoch 32/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 1.8951e-06 - acc: 1.0000\n",
      "Epoch 33/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 1.2887e-06 - acc: 1.0000\n",
      "Epoch 34/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 1.1595e-06 - acc: 1.0000\n",
      "Epoch 35/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 9.1407e-07 - acc: 1.0000\n",
      "Epoch 36/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 1.7766e-06 - acc: 1.0000\n",
      "Epoch 37/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 6.1367e-07 - acc: 1.0000\n",
      "Epoch 38/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 5.3898e-07 - acc: 1.0000\n",
      "Epoch 39/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 4.0967e-07 - acc: 1.0000\n",
      "Epoch 40/50\n",
      "22982/22982 [==============================] - 1s 43us/step - loss: 3.8384e-07 - acc: 1.0000\n",
      "Epoch 41/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 3.2067e-07 - acc: 1.0000\n",
      "Epoch 42/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 2.6349e-06 - acc: 1.0000\n",
      "Epoch 43/50\n",
      "22982/22982 [==============================] - 1s 40us/step - loss: 6.6525e-07 - acc: 1.0000\n",
      "Epoch 44/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 8.8782e-07 - acc: 1.0000\n",
      "Epoch 45/50\n",
      "22982/22982 [==============================] - 1s 42us/step - loss: 2.7803e-07 - acc: 1.0000\n",
      "Epoch 46/50\n",
      "22982/22982 [==============================] - 1s 41us/step - loss: 1.7985e-07 - acc: 1.0000\n",
      "Epoch 47/50\n",
      "22982/22982 [==============================] - 1s 39us/step - loss: 1.6025e-07 - acc: 1.0000\n",
      "Epoch 48/50\n",
      "22982/22982 [==============================] - 1s 55us/step - loss: 1.4502e-07 - acc: 1.0000\n",
      "Epoch 49/50\n",
      "22982/22982 [==============================] - 1s 53us/step - loss: 1.4455e-07 - acc: 1.0000\n",
      "Epoch 50/50\n",
      "22982/22982 [==============================] - 1s 55us/step - loss: 1.3459e-07 - acc: 1.0000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f3aee1d5cc0>"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conv_model_stack_1.fit(y_pred_rf_s, y_train, epochs=50, batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_stack_rf = conv_model_stack_1.predict(bin_y_pred_rf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5746/5746 [==============================] - 0s 48us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[5.459371597311283, 0.6517577445179256]"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conv_model_stack_1.evaluate(bin_y_pred_rf,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU8AAAEFCAYAAABjDmX4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxV5Z3H8c8vYQuyKajsmwgK6oyoqMWFuoLVYqe11aoVq6VqW6tWQafOSB0X6nSxM9VWXKm2UmRcaOtSraJtRUArVRGUyCKJ7LLvgd/88TzBk3DvTXISchP4vl+vvMg963Oec84355x77w9zd0REpGYK8t0AEZHGSOEpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIinsseFpZkVm9gczW2NmT9RiORea2Z/rsm35YmYnmtkHdbzMMWb2WF0uM8M6njOzS3bnOhLrqrPtqatjcE9lZoPNbK6ZrTezc/PdnprKe3ia2dfN7M3YgYvjiXJCHSz6K8CBQHt3Py/tQtz9t+5+Rh20Z7cyMzezPrmmcfe/unu/+mpTXXH3Ye4+vq6Xa2ZDzKykrpebUCfH4B7sVuCX7t7K3Z/Od2NqKq/haWbXAXcDdxAOsu7AvcDwOlh8D+BDdy+rg2U1embWJN9t2AvlPAa1T+gBzMp3I1Jz97z8AG2B9cB5OaZpTgjXT+LP3UDzOG4IUAL8AFgGLAYujeN+BGwFtsV1XAaMAR5LLLsn4ECT+HoEMA9YB8wHLkwM/1tivs8BM4A18d/PJcZNAf4L+Htczp+BDlm2rbz9oxLtPxc4C/gQ+BT498T0g4CpwOo47S+BZnHca3FbNsTt/Vpi+aOBJcCj5cPiPAfFdQyMrzsDy4EhWdo7GiiN2/UBcGocPgaYCPwmjpsFHJ2Y79DYL6vjuC/G4b3isIL4+n5gWWK+R4FrEv16eXJ/AD8BVsV9NSwxX6/YH+uAl4B7kvs9Md0+wCZgR+yz9bEPqtqezsD/xb6aD1ydpb8yHYMj4rHxc2AlcFuc9pvA7Lg9LwA9Ess5HZhDON5+Cbya6Isx5D6m2wIPEo6XUuA2oLCa/bgf8DDhvFsFPB2Hvweck5iuKbACODJLP3wLKCYca5OBznH4R7HvN8X+aZ6vLEqdYXlbMQwFysp3dJZpbgXeAA4A9gdeB/4rjhsS57817sCzgI3AvlkOrKwHWjyR1gL94rhOwIDkQZY4oFYBF8f5Loiv2ydO8o+AvkBRfD02y7aVt/8/Y/u/RTghfwe0BgbEA6tXnP4o4Li43p6Ek+2axPIc6JNh+T8m/BEqIhGeiQP7faAl4aT9SZa29gMWJQ78nsBBiX7dHPu/ELgTeCNxYhUD/w40A04hBFJ5P38MHBV//4Dwx+vQxLgjE/2aDM9tse2FwJWEE9zi+KmEQGgGnBD36y7hmeijkkrDcm1PAfBW3GfNgN6xzWdmWf4YKh5zI+I++V7cj0WEu6xiwh+ZJsDNwOtx+g6xv74S+/LaOH91w/Mp4D7C8X0AMB34djX78U/A74F947pPjsNHAb9PrHM48G6W7T+FEKwDCcfg/wKvJcYvAE7LVwbVOsPytmK4EFhSxTQfAWclXp8JLEgc+JtIhC/hCu64LAdW1gMtHlyrgS8DRZXaMILPwvNiYHql8VOBEfH3KcDNiXFXAc9n2bby9pdfCbSO7Tk2Mc1bwLlZ5r8GeCrxOlN4bgVaVBpWOSwmA+8C75Dlrz/QJ/btaUDTSuPGAC8lXvcHNsXfTyRc9RYkxj8OjIm/PwpcB3QkhOddwBXselU6hYrhWZxYXsu47R0Jj33KgJaJ8Y9R8/DMtj3HAh9Xmv4m4OEsy698zI3IMP9zwGWJ1wWEi4AewDeIwR3HGeFuosrwJDwG20LieCb8sX+lGv3YiXBVuG+GbepMCPQ28fUkYFSW7X8QuCvxuhUhsHvG1wtoxOGZz2eeK4EOVTz36QwsTLxeGIftXIZXfJ60kbCDasTdNxBuda8AFpvZn8zskGq0p7xNXRKvl9SgPSvdfXv8fVP8d2li/Kby+c2sr5n90cyWmNlawnPiDjmWDbDc3TdXMc39wGHA/7r7lkwTuHsxIazHAMvMbIKZJfdD5W1uEfdrZ2CRu+9IjE/216uEADuJcKs9BTg5/vy10nxJO9fn7hvjr63i+j5NDINwxVxT2banB9DZzFaX/xCuqg+swbIrt6cH8IvE8j4lhGQXYv+VT+ghcaq7PT0IV4yLE8u+j3AFWi5bP3Yj9OOqygt1908Ijx6+bGbtgGHAb7O0ocL54u7rCed9lyzTNyr5DM+phL+MuT6i8AnhICjXPQ5LYwPhr2u5jsmR7v6Cu59O+Ks7hxAqVbWnvE2lKdtUE78itOtgd29DOGmtink810gza0V4jvwgMMbM9su6IPffufsJhO13wuOAqnwCdDOz5HGW7K9XCVenQ+LvfwMGE8Lz1Wosv7LFwH5mltzP3XJMn7N/MlgEzHf3domf1u5+Vg2WUXmdiwi30sllFrn764Tt2dl+MzMqbk+uY3oR4fzqkFhuG3cfUM3t3C+GYybjgYuA84Cp7p7t+K9wvpjZPkB76ud82e3yFp7uvobw7OgeMzvXzFqaWVMzG2Zmd8XJHgduNrP9zaxDnD7tZ/BmAieZWXcza0u43QLAzA40s+Fx524hPMDOdNXzLNA3fryqiZl9jXBb98eUbaqJ1oTnd+vjVfGVlcYvJTyDq4lfAG+6++WEZ1y/zjSRmfUzs1PMrDnheWD5Gy1VmUa4chsV9+0Q4BxgAoC7z43Lugh41d3Xxu34MinC090XAm8S/hA0M7Pj4/qyWQq0j8dDdUwH1pnZ6PgZzkIzO8zMjqlpWxN+DdxkZgMAzKytmZV/rOlPwAAz+7d45Xs1FQMy6zHt7osJb1j+1MzamFmBmR1kZidX1aA473PAvWa2b9x3JyUmeZrwHPP7hDfWsnkcuNTM/jUeO3cA09x9QVVtaAzy+lEld/8p4ZnXzYQ3SxYB3yXsHAjvDr5JeB73LvCPOCzNul4kPAB/h/AsMRl4BbEdnxBum05m13DC3VcCZxPe4V9JeHh+truvSNOmGroe+DrhedP9hG1JGgOMj7doX61qYWY2nPCmXfl2XgcMNLMLM0zeHBhLePi/hHDrd1OG6Spw962E8BoW570X+Ia7z0lM9irh8cWixGsj7Os0LgSOJ76bTeinbI8j5hBO8Hmx3zpnmi4x/XbC/v9XwrvTK4AHCO9qp+LuTxGu4ifExzHvEfqLeFydR+j7lcDBhFvm8nlzHdMQnpk2I7wpuIrwfLJTNZt2MeH55BzC8+5rEuvdRPjEQS/gyRzb9hLwH3HaxYRPeJxfzfU3eOXvrInskczs98Acd78l322pC2Y2hfAm0QN5bsd/An3d/aJ8tiOf8v4NI5G6ZGbHxNvTAjMbSvgoTaP79kpDFp+NXwaMy3db8knhKXuajoR37dcD/wNc6e5v57VFexAz+xbh8dpz7v5avtuTT7ptFxFJQVeeIiIpKDyroarqO7EiVE0/JtSomNkUM7s8z23YreXvzGxW/DgVFjxsZqvMbLrthnJ+cT3d4/FTWNfL3t2Sx4TVonSj1WPJwbrUKMOzHkqJ1YiHklrz8t2OtMxshJn9Ld/tyDd3H+DuU+LLEwhFObq6+yCvo3J+ZrbAzE5LrPPjePxszzVfQ+fVLN2Y6Q+g76aSg7tbvYRnpq9gqhxXeurPetGDUEdhQ74bUh90/KRQ1ZffCV8He5LwIfaVhOKlEIL3ZsJ3V5cRvmnQ1isWKLiMUB3ntUzD4rTHEaolrQb+SaIkGhnKYpGllFiGdn8BeJvwrZxFxGIUldp3SWzLCuCHifFFwCNxne8DN1CpgESlde0syhHnu4fw7ZB1hG/ZHJSYdgDwIuHD+EuJZeeoXvm9XcrLJYfFac8mfPNkdezXI3LtS0I1n83A9tiXq7Ns4xRChaHpsU+fAfZLtq/S9AuIRR9if46P/Tmb8OWCXP2ZrY/GULEQxhNx29cQjrEBiXFnxX23jvB1wOvj8A6ED5OXf4/8r3xWgGQBofjJZZX65EeVtzFTX8bhBwEvx2ErCN/7bhfHPUrFMmyj2LUSUmdCsZZPCdWWvpVY5xhylMvLclxeTaj8tAL478S2jqBuy+ONoGLpxl32IeFLGckyff9MHFvly6lOrmQ7bwcRvlRT/k21n1WVb7X5qSo4CwmB9nNCaLUATkh0cjHhK4Gt4oFUfgKXb+Rv4nxFWYZ1iTvurNhpp8fX+8flZCuLNYQcJ19imsPjco+InXlupfbdH9vxL4RvoZSXQxtLOKn2I5wk7+VaH7uG58q4I5sQTp4JcVxrwjctfhD7sjWxihLVK79Xubxc5WFHxgPu2LjvLiEEQvMq9uUIEgd+jvAsJRQR2YfwrZHHqhmeYwkn2r5AV8I3YjL2ZxV9NIaK4fnNOL78D8/MxLjFwInx9335rG7pnYSvRDaNPyfy2adOkm2u0CfJbayiL/sQjuPmcT++BtydqV8qHYvl4fka4ZtYLQjfZFoOnJLY/ozl8nIcl68QjuPuhDqxybCry/J4O/urJvswQ3hWJ1eynbdTgYvj762IFdbyFZ7Hx523S81N4C/AVYnX/Qh/UZokNrJ3hoMkOWx0ecckhr1AOOlzlcUaQhXhmWGeu4GfV2pL18T46cD58fd5wNDEuJG51seu4flAYtxZhG+4QCgJ9naWZVRVfi9TebnKw35FDNzEsA8IXzfNtS9HUL3wHJt43T+uvzDT/qBiEFWoeQlcnq0/q+ijMWQvL9cu7ofyq5SPgW8TS6clpruVcNXcJ8Mykm2u0CdUDM+sfZlhmecmt4cc4Un4Q70daJ0YfyfwSGL7M5bLy3FcJo/jq4C/JLavLsvj7eyvmu5DKoZndXIl23n7GuEuIWMB8rr+qeqZZzdgoWf+bwQylYsrryNYLlP5rOSwHsB5lUp8nUAIzqxlsarDzI41s1fMbLmZrSGUm6tcwi1b+bgKpcDYtQxdVbIttxshJDOpqvxepvJylYf1AH5QqT+7xeXk2pfVVblPmlJ1WTzYtT9zlVXL1Uc7xaIcY83so/id8AVxVHl7vkz4w7XQzF6NRUIg3LoWA382s3lmdmM12p+pjRn7MhaZmWBmpbFdj1G9PoLPSuqtSwyrquRhiyqeV1beZ52zjIO6K49XrX2YRXVyJdv5dRmhEPkcM5thZmenbEO1VBWei4DuWXZOpnJxZVSsR+kZ5ksOW0S48kyW49rH3ceSuyxWpuVW9jvCs6Nu7t6WcKtWVQm3chVKgRG2rS4sInvlo6rK71XVl+XLv71Sf7Z098fJvS+r05+wa59sIzx3qlAaLX7sZv/EtIsJt+uZllNZrj5K+jrhNvM0QmGOnuWrB3D3Ge4+nPAY5GnCs0LcfZ27/8DdewNfBK4zs1Orsb7KbczWl3cQ+vNwD6UDL6LicZerrz8hHPOtE8NqW/Kw8j7LdUzVpjxe5eVk24dVHWvVyZXMC3af6+4XEPb5j4FJFiql7RZVhed0QqeNNbN9zKyFmQ2O4x4HrjWzXhbqQt5BKM9fkyubx4BzzOzMeCXRIn4MqavnLotVnVJirQl/xTeb2SDCyVZdEwllwvY1s66E50J14Y9AJzO7xsyam1lrMzs2jquL8nv3A1fEq26L++wL8WTMtS+XAl3NrFkVy7/IzPpbqJd5KzDJw0dsPiRcAX3BzJoSnpU1T8yX7M8uhMpZ2eTqo6TWhOddKwnBfUf5CAvl6C40s7buvo3wBsKOOO5sM+sTT/41hNvk6pTXS8rVl60Jb4asidt6Q6V5s5YO9FBZ6nXgzrjMIwhXU7X5bOsNsd+7EUrIVa7GlVSb8nhJufbhUqCnVazxmpQ6V8zsIjPb30MR7dVxcE33bbXlDM94YpxDeAj+MeEZx9fi6IcI7x6+RijPtZkahkw8WIYT3okrL0l3Q6JdGctiefVKiV0F3Gpm6whBNLEGTfsR4XZhPqEm4qM12a5s4u3Y6YQ+XQLMBT4fR9e6/J67v0n4P2l+SXi3tJjwLKqqffky4Z3bJWaWq7zeo4RnuksIbwRcHZe9htDfDxCukjbE5Ze7Nb6eT/hP2SaRvUxcrj5K+g1hH5US3lV/o9L4i4EF8db5CkKpOghl3V4iBNxU4F53fyXHNmdqY66+/BGh1uUaQuBULtl2J+GP5Gozuz7D4i8gXEV/Qvg/iG7xUNotrWcI5epmxvY8mG1Cr0V5vErLybUPy///+pVmlqnsYG1yZSgwy8zWE2rVnu+hfF75F1lOrOZyqkXfbZd6Z2ZXEg7sk/Pdlj2ZmTnhfx4ozndb9kSN8htG0riYWSczG2yhTFw/wkdYnsp3u0RqQ98qkPrQjPCfj5X/r5gTCJ9lFGm0dNsuIpKCbttFRFJQeIqIpKDwFBFJQeEpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIikoPEVEUlB4ioikoPAUEUlB4SkikoLCU0QkBYWniEgKCk8RkRQUniIiKSg8RURSUHiKiKSg8BQRSUHhKSKSgsJTRCQFhaeISAoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIikoPBsBMxtqZh+YWbGZ3Zjv9kj1mdlDZrbMzN7Ld1ukbik8GzgzKwTuAYYB/YELzKx/flslNfAIMDTfjZC6p/Bs+AYBxe4+z923AhOA4Xluk1STu78GfJrvdkjdU3g2fF2ARYnXJXGYiOSRwlNEJAWFZ8NXCnRLvO4ah4lIHik8G74ZwMFm1svMmgHnA5Pz3CaRvZ7Cs4Fz9zLgu8ALwGxgorvPym+rpLrM7HFgKtDPzErM7LJ8t0nqhrl7vtsgItLo6MpTRCQFhaeISAoKTxGRFBSeIiIpKDxFRFJQeDYSZjYy322Q9LT/9jwKz8ZDJ1/jpv23h1F4ioiksMd8SL7VVx/ZMzYki23zptC095B8N2O3GDrsiHw3YbebN+VJeg/5t3w3Y7eZdOlASzNfVeft+okjUi23PujKs5HYU4Nzb7EnB+feSuEpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIikoPEVEUlB4ioikoPAUEUlB4SkikoLCU0QkBYWniEgKCk8RkRQUniIiKSg8RURSUHiKiKSg8BQRSUHhKSKSgsJTRCQFhaeISAoKTxFptMzsITNbZmbvVRr+PTObY2azzOyuxPCbzKzYzD4wszMTw4fGYcVmdmN11t2k7jZDRKTePQL8EvhN+QAz+zwwHPgXd99iZgfE4f2B84EBQGfgJTPrG2e7BzgdKAFmmNlkd38/14oVniLSaLn7a2bWs9LgK4Gx7r4lTrMsDh8OTIjD55tZMTAojit293kAZjYhTpszPHXbLiJ7mr7AiWY2zcxeNbNj4vAuwKLEdCVxWLbhOSk8RaTBMrORZvZm4mdkNWZrAuwHHAfcAEw0M6vrtum2XUQaLHcfB4yr4WwlwJPu7sB0M9sBdABKgW6J6brGYeQYnpXCU0TypmffKu+O03ga+DzwSnxDqBmwApgM/M7MfkZ4w+hgYDpgwMFm1osQmucDX69qJQpPEWm0zOxxYAjQwcxKgFuAh4CH4seXtgKXxKvQWWY2kfBGUBnwHXffHpfzXeAFoBB4yN1nVbVuhaeINFrufkGWURdlmf524PYMw58Fnq3JuvWGkYhICgpPEZEUFJ4iIikoPEVEUlB4ioikoPAUEUlBH1XKo3uvHMywgV1ZvmYzg65/BoDDe+zHL751PC2aFVK2fQfXPvAGb320gjZFTXng6pPo1n4fmhQav/jDLB6bUgzAf114FEMHdqXAjJff/YQbHp6ez83aK1w1uDtHdWvLms1lXPf0bABaNSvk2iG9OKB1M5at28rPpsxnw9btdG7bnO+c0IPe7Vvy+D8+YfJ7y3Yup2WzQq4c3J3u7Ypw4N6/LeTD5RvytFVSE7ryzKPfTinm3DterDDstouO4s5JM/ncqMncNvFtbrvoaABGDj2EOSWrOX7UZIaNeZ47vnEMTQsLOLbv/hzX7wCOvX4yx/zgGQYe1IET+3fMx+bsVV4p/pTbXiyuMOzcIzry7uJ1fO//3ufdxev40hEHArB+y3YemlZSITTLffPYrswsWcv3n3qf65+ZTcmazfXSfqm9BhOeZnaImY02s/+JP6PN7NB8t2t3+vvspaxav7XCMHdoU9QUgLYtm7F41cadw1u3CMP3adGUVeu3ULZjB+7QolkhzZoU0LxpAU0LC1i2ZlP9bsheaPbS9azfsr3CsGO6t2VK8UoAphSv5Jju7QBYu7mMj1ZsZPsOrzB9y6YFHHpgK/4yN8xTtsPZuLXiMqXhahC37WY2GrgAmED4rimEL+c/bmYT3H1s3hpXz0aPn87TPzyd2y8+hoICOPXm8KWH+56fzcRRp1J831dpVdSUS37+Ku4wfe5yXpu1hOJxX8MMxj0/mw9K1+R5K/ZO7Vo0YfWmMgBWbyqjXYvcp9cBrZuzdnMZ3zmhBz33K+KjlRt5eFoJW8p21EdzpZYaypXnZcAx7j7W3R+LP2MJhUovyzZTslzVtnlT6qutu9XlZ/TjxvEzOOSqJ7hx/AzuvWIwAKf9SxfeWfgpfb49kc/dMJmfXnYsrYua0vvA1vTr0pZ+V0yk77cnctJhnfjcIQfkeSsEwKsYX2hG7/Yt+fOc5dwweQ5bynbwpcMPrJe2Se01lPDcQahyUlmnOC4jdx/n7ke7+9FNew/ZTU2rX18/uQ/PTFsIwJNTF3BUnw4AXPT5PkyOw+ctXcfCZevp27kt5wzqzoy5y9mwpYwNW8p48e1SBvVVeObD6s1ltCsKV5vtipqwZnNZzulXbtzKyg1bmbsiPJp5Y8EqerVvudvbKXWjoYTnNcBfzOw5MxsXf54H/gJ8P89tq1dLPt248w2fIYd14qMlawEoWbGBIYeHvy8HtG3BwZ3bsGDZOkpWbOCEQztSWGA0KTRO6H8gH5Suzlv792ZvfryGIX3aAzCkT3tmfJz78cnqTWWs3LCNzm2aA3B4pzaUrNYbRo2FhUpN+WdmBYTb9PICf6XAjPKSUVVp9dVHGsaG1MDD3z+JE/t3pH3rFixbs4nbJ85k7idruOvSQTQpKGDztu1c+8AbzJy/ko77FnHfVSfQcd+WGPDTZ97l93+dR4EZd19+HIMPPRAHXpxZyk2/mZHvTauRocOOyHcTauyak3syoGNrWrdowppN2/j924uZ/vFqfjCkFx1aNWP5+q387JX5rN+6nXZFTfjxOYdQ1LQQd2dz2Q6ueep9Nm3bQc/9irhycHeaFBSwdN0W7vnbQjY0wjeNJl06MFWl9sNufjHnefvebafXeQX4utJgwrO2GmN4StAYw1Mq2hvDs6HctouINCoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpKCwlNEJAWFp4hICgpPEZEUGkQ9TxHZOx1yUPt8NyE1XXmKSKNlZg+Z2TIzey8x7L/NbI6ZvWNmT5lZu8S4m8ys2Mw+MLMzE8OHxmHFZnZjddat8BSRxuwRYGilYS8Ch7n7EcCHwE0AZtYfOB8YEOe518wKzawQuAcYBvQHLojT5qTwFJFGy91fAz6tNOzP7l5eifoNwn/pAzAcmODuW9x9PlBMKIM5CCh293nuvpXw3wENr2rdCk8RabCS/9VO/BlZw0V8E3gu/t4FWJQYVxKHZRuek94wEpEGy93HAePSzGtmPwTKgN/WaaMihaeI7HHMbARwNnCqf1bxvRTolpisaxxGjuFZ6bZdRPYoZjYUGAV80d03JkZNBs43s+Zm1gs4mPBfnc8ADjazXmbWjPCm0uSq1qMrTxFptMzscWAI0MHMSoBbCO+uNwdeNDOAN9z9CnefZWYTgfcJt/PfKf8/0szsu8ALQCHwkLvPqmrdCk8RabTc/YIMgx/MMf3twO0Zhj8LPFuTdeu2XUQkBYWniEgKCk8RkRQUniIiKSg8RURSUHiKiKSg8BQRSUHhKSKSgsJTRCQFhaeISAoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpLCHlMMeeiwI/LdBEnp+efeyXcTpLYuHZjvFtS7PSY8RaTxGdSzbb6bkJpu20VEUlB4ioikoPAUEUlB4SkikoLCU0QkBYWniEgKCk8RkRQUniLSaJnZtWY2y8zeM7PHzayFmfUys2lmVmxmvzezZnHa5vF1cRzfszbrVniKSKNkZl2Aq4Gj3f0woBA4H/gx8HN37wOsAi6Ls1wGrIrDfx6nS03hKSKNWROgyMyaAC2BxcApwKQ4fjxwbvx9eHxNHH+qmVnaFSs8RaRRcvdS4CfAx4TQXAO8Bax297I4WQnQJf7eBVgU5y2L07dPu36Fp4g0WGY20szeTPyMTIzbl3A12QvoDOwDDK2vtqkwiIg0WO4+DhiXZfRpwHx3Xw5gZk8Cg4F2ZtYkXl12BUrj9KVAN6Ak3ua3BVambZuuPEWksfoYOM7MWsZnl6cC7wOvAF+J01wCPBN/nxxfE8e/7O6eduUKTxFplNx9GuGNn38A7xLybBwwGrjOzIoJzzQfjLM8CLSPw68DbqzN+nXbLiKNlrvfAtxSafA8YFCGaTcD59XVunXlKSKSgsJTRCQFhaeISAoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpKCwlNEJAV9PVNE8ubIjm3z3YTUdOUpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIikoPEVEUlB4ioikoPAUEUlBH5JvIM7qvz+n9e2AAS99uII/vb+cnvsVMfL47jQtNHa4c//URRSv2MiAjq0YdepBLFu3BYBpC1cz6Z9L8rsBe5l7rxzMsIFdWb5mM4OuD/854+E99uMX3zqeFs0KKdu+g2sfeIO3PlrBif07MmHUKSxcth6AydMWMvb//snBndow/tohO5fZ84BW3DZxJvc++34+NklqSOHZAHRr14LT+nbgxj/MoWyHc/MZfXhr0VouProLT8xczNulazmyaxsuProLtzw/F4A5S9dz50sf5bnle6/fTinmvudnc/93Ttw57LaLjuLOSTN5cWYpZxzZhdsuOpphP3oegNdnL+W8Hw3KO/0AAAXzSURBVP+lwjLmLl7L50ZNBqDAjLn3fZU/TF9YfxshtaLb9gaga7sWzF2+ga3bnR0O7y9Zz7E92uFAUbNCAFo2LeTTjdvy21DZ6e+zl7Jq/dYKw9yhTVFTANq2bMbiVRurvbwhh3di3pK1LFqxoU7bKbtPg7/yNLNL3f3hfLdjd/p41WYuGNiZVs0L2Vq2gyO7tuGjFRt5eFoJN5/Rh28c0wUDfvinD3fO03f/ffjJ8ENYtXEb42eUUrJ6c/42QAAYPX46T//wdG6/+BgKCuDUm5/dOW5Q3/2ZetcXWbxqIz989E1ml6yuMO9XBvdi0t/n13eTpRYaw5Xnj7KNMLORZvammb05b8qT9dmmOlW6ZjNPv7uU/zjjYG4+ow8LPt3EDoczD+nAI9NLuGLiezwyvYSrTugOwLyVG7nyife4/pk5PDt7OaNP7Z3nLRCAy8/ox43jZ3DIVU9w4/gZ3HvFYABmzl9J/6smcfyoyfz6+dk8fsMpFeZrWljAF47qxlNvLMhDqyWtBhGeZvZOlp93gQOzzefu49z9aHc/uveQf6vHFte9l+euZPQf5vCfz81lw5btLF67mZP7tGfawnCFMnXBavp02AeATdt2sLlsBwBvl6yl0IzWzQvz1nYJvn5yH56ZFp5ZPjl1AUf16QDAuk3b2LClDIA/v11K08IC2rduvnO+M47swsz5K1m2RncPjUmDCE9CQH4DOCfDz8o8tqvetGkRnqB02Kcpx/Zox1/nrWLVxm0M6NgKgMM7tWbx2vDueruiz5629OnQEjNj3Zbt9d9oqWDJpxs5sX9HAIYc1omPlqwF4IC2RTunOeqgDhQUwMr4SQmA8wb35gndsjc6DeWZ5x+BVu4+s/IIM5tS/82pfzd8vjetWhSyfYfzwBuL2Lh1O7/++0IuPbYbhQWwbbtz3+vhqua4nvtyZr8ObHdna5lz96s68erbw98/iRP7d6R96xZ88KvzuH3iTL573+vcdekgmhQUsHnbdr5331QAvnRcDy4/ox9l251NW7cz4u5Xdy6nZfMmfP6ITlw97vV8bUqjZ2aFwJtAqbufbWa9gAlAe+At4GJ332pmzYHfAEcRLsq+5u4LUq/X3Wvd+IbgKw//Y8/YkL3Q88+9k+8mSC2tnzjC0sz34uwVOc/b0w/tUOVyzew64GigTQzPicCT7j7BzH4N/NPdf2VmVwFHuPsVZnY+8CV3/1qadkPDuW0XEakxM+sKfAF4IL424BRgUpxkPHBu/H14fE0cf2qcPhWFp4g0WMlP1MSfkZUmuRsYBeyIr9sDq929LL4uAbrE37sAiwDi+DVx+lQayjNPEZFduPs4YFymcWZ2NrDM3d8ysyH12jAUniLSeA0GvmhmZwEtgDbAL4B2ZtYkXl12BUrj9KVAN6DEzJoAbanFp3l02y4ijZK73+TuXd29J3A+8LK7Xwi8AnwlTnYJ8Ez8fXJ8TRz/stfiHXOFp4jsaUYD15lZMeGZ5oNx+INA+zj8OuDG2qxEt+0i0ui5+xRgSvx9HjAowzSbgfPqap268hQRSUHhKSKSgm7bRSRverXfJ99NSE1XniIiKSg8RURSUHiKiKSg8BQRSUHhKSKSgsJTRCQFhaeISAoKTxGRFBSeIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpKCwlNEJAWFp4hICgpPEZEUFJ4iIikoPEVEUlB4ioikoPAUEUlB4SkikoLCU0QkBYWniDRaZjbUzD4ws2Izu7E+163wFJFGycwKgXuAYUB/4AIz619f61d4ikhjNQgodvd57r4VmAAMr6+VKzxFpLHqAixKvC6Jw+pFk/pa0e426dKBlu82SEqXDsx3CyRP+hxQlPO8NbORwMjEoHHuPm73tqp69pjwFJE9TwzKbGFZCnRLvO4ah9UL3baLSGM1AzjYzHqZWTPgfGByfa1cV54i0ii5e5mZfRd4ASgEHnL3WfW1fnP3+lqXiMgeQ7ftIiIpKDxFRFJQeIqIpKDwFBFJQeEpIpKCwlNEJAWFp4hICgpPEZEU/h+nnXnkNBl99AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "confusion_matrix = pd.DataFrame(\n",
    "    data=metrics.confusion_matrix(y_test, np.rint(y_pred_stack_rf)), \n",
    "    \n",
    ")\n",
    "\n",
    "confusion_figure, confusion_axes = matplotlib.pyplot.subplots()\n",
    "confusion_figure.set_size_inches(5, 3)\n",
    "confusion_axes.set_title(\n",
    "    'Confusion matrix showing the frequency of \\n'\n",
    "    'correct and incorrect bug classification predictions.'\n",
    "    '\\n\\n'  # hack to avoid overlap with x-axis labels below\n",
    ")\n",
    "confusion_axes.xaxis.tick_top()  # move x-axis labels to top of matrix\n",
    "_ = sns.heatmap(\n",
    "    confusion_matrix,\n",
    "    annot=True,\n",
    "    fmt=\"d\",\n",
    "    cmap=sns.color_palette(\"Blues\"),\n",
    "    vmin=0,\n",
    "    ax=confusion_axes,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "tn, fp, fn, tp = metrics.confusion_matrix(y_test, np.rint(y_pred_stack_rf)).flatten().tolist()\n",
    "\n",
    "fpr_stack_rf = fp/(fp+tp)\n",
    "fnr_stack_rf = fn/(fn+tn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.353637312913331"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fpr_stack_rf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.342847198050818"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fnr_stack_rf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr_keras, tpr_keras, thresholds_keras = roc_curve(y_test, np.rint(y_pred_stack_rf))\n",
    "precision, recall, thresholds = precision_recall_curve(y_test, np.rint(y_pred_stack_rf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "auc_keras = auc(fpr_keras, tpr_keras)\n",
    "roc_auc = roc_auc_score(y_test, np.rint(y_pred_stack_rf))\n",
    "pr_auc = auc(recall, precision)\n",
    "f1 = f1_score(y_test, np.rint(y_pred_stack_rf))\n",
    "mcc = matthews_corrcoef(y_test, np.rint(y_pred_stack_rf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PR AUC: 0.736\n",
      "ROC AUC: 0.652\n",
      "MCC: 0.304\n",
      "F1: 0.650\n"
     ]
    }
   ],
   "source": [
    "print('PR AUC: %.3f' % pr_auc)\n",
    "print('ROC AUC: %.3f' % roc_auc)\n",
    "print('MCC: %.3f' % mcc)\n",
    "print('F1: %.3f' % f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}