result_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-10T08:28:39.630256Z",
     "start_time": "2019-01-10T08:28:39.201181Z"
    }
   },
   "outputs": [],
   "source": [
    "#%load_ext autoreload\n",
    "#%autoreload 2\n",
    "\n",
    "import pandas as pd\n",
    "import pickle\n",
    "from glob import glob\n",
    "from sleep_eval import evaluation_summary\n",
    "from scipy.stats import ttest_ind\n",
    "\n",
    "TASK = 2\n",
    "\n",
    "baselines = [\"gt\", \"always1\", \"always0\", \"binterval\", \"sleep\"] # binterval is the \"Manual Annotation alg\" and \"sleep\" is the \"device algo\"\n",
    "defaultalgs = [\"sazonov\", \"cole\", \"sadeh\", \"oakley\", \"kripke\", \"webster\"] # \"sazonov2\", \"time_based\",\n",
    "defaultml = [\"ExtraTrees\", \"SGD_perceptron\", \"SGD_log\", \"SGD_hinge\"] # \"SGD_huber\"\n",
    "defaultdl = [\"LSTM_20_raw\", \"LSTM_50_raw\", \"LSTM_100_raw\", \"CNN_20_raw\", \"CNN_50_raw\", \"CNN_100_raw\"]\n",
    "\n",
    "summary = pd.read_csv(\"./summaries/task%d_summary.csv\" % (TASK))\n",
    "summary = summary.set_index(\"Unnamed: 0\")\n",
    "\n",
    "with open(\"./results/task%d_results.pkl\" % (TASK), \"rb\") as f:\n",
    "    results = pickle.load(f)\n",
    "    \n",
    "metrics = [\"Accuracy\", \"Specificity\", \"Precision\", \"Recall\", \"F1\"]\n",
    "\n",
    "def get_group_average(t, metrics):\n",
    "\n",
    "    for m in metrics:\n",
    "        t[m] = t[m].apply(lambda x: np.float(x.split(\"+-\")[0]))\n",
    "    t = pd.concat((t.mean(), t.std()), axis=1)\n",
    "    t.columns = [\"Mean\", \"Std\"]\n",
    "    t = t.apply(lambda x: \"%.1f +- %.1f\" % (x[\"Mean\"],x[\"Std\"]), axis=1)\n",
    "    return t.to_frame(\"Group Average\").T.to_latex()\n",
    "\n",
    "def pvalue(results, alg1, alg2, metric):\n",
    "    return ttest_ind(results[alg1][metric], results[alg2][metric])[1]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-09T07:13:12.045914Z",
     "start_time": "2019-01-09T07:13:12.035270Z"
    }
   },
   "outputs": [],
   "source": [
    "bls = [alg for alg in baselines]\n",
    "    \n",
    "print summary.loc[bls].sort_values(by=\"Accuracy\", ascending=False)[metrics].to_latex()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-09T07:13:22.843067Z",
     "start_time": "2019-01-09T07:13:22.801557Z"
    }
   },
   "outputs": [],
   "source": [
    "traditional = [alg for alg in defaultalgs]\n",
    "print summary.loc[traditional].sort_values(by=\"Accuracy\", ascending=False)[metrics].to_latex()\n",
    "\n",
    "t = summary.loc[traditional].sort_values(by=\"Accuracy\", ascending=False)[metrics]\n",
    "print(get_group_average(t, metrics))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-09T07:13:33.624794Z",
     "start_time": "2019-01-09T07:13:33.577433Z"
    }
   },
   "outputs": [],
   "source": [
    "traditionalresc = [\"resc_\" + alg for alg in defaultalgs]\n",
    "print(summary.loc[traditionalresc].sort_values(by=\"Accuracy\", ascending=False)[metrics].to_latex())\n",
    "\n",
    "t = summary.loc[traditionalresc].sort_values(by=\"Accuracy\", ascending=False)[metrics]\n",
    "print(get_group_average(t, metrics))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-09T07:13:41.872982Z",
     "start_time": "2019-01-09T07:13:41.830026Z"
    }
   },
   "outputs": [],
   "source": [
    "ml = [alg for alg in defaultml]\n",
    "print summary.loc[ml].sort_values(by=\"Accuracy\", ascending=False)[metrics].to_latex()\n",
    "\n",
    "\n",
    "t = summary.loc[ml].sort_values(by=\"Accuracy\", ascending=False)[metrics]\n",
    "print(get_group_average(t, metrics))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-09T07:13:50.911493Z",
     "start_time": "2019-01-09T07:13:50.864690Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "mlresc = [\"resc_\" + alg for alg in defaultml]\n",
    "print summary.loc[mlresc].sort_values(by=\"Accuracy\", ascending=False)[metrics].to_latex()\n",
    "\n",
    "t = summary.loc[mlresc].sort_values(by=\"Accuracy\", ascending=False)[metrics]\n",
    "print(get_group_average(t, metrics))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-09T07:13:59.349575Z",
     "start_time": "2019-01-09T07:13:59.296384Z"
    }
   },
   "outputs": [],
   "source": [
    "dl = [alg for alg in defaultdl]\n",
    "print summary.loc[dl].sort_values(by=\"Accuracy\", ascending=False)[metrics].to_latex()\n",
    "\n",
    "t = summary.loc[dl].sort_values(by=\"Accuracy\", ascending=False)[metrics]\n",
    "print(get_group_average(t, metrics))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-09T07:14:08.295624Z",
     "start_time": "2019-01-09T07:14:08.231242Z"
    }
   },
   "outputs": [],
   "source": [
    "dlresc = [\"resc_\" + alg for alg in defaultdl]\n",
    "print summary.loc[dlresc].sort_values(by=\"Accuracy\", ascending=False)[metrics].to_latex()\n",
    "\n",
    "t = summary.loc[dlresc].sort_values(by=\"Accuracy\", ascending=False)[metrics]\n",
    "print(get_group_average(t, metrics))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-10T08:20:39.976475Z",
     "start_time": "2019-01-10T08:20:39.945074Z"
    }
   },
   "outputs": [],
   "source": [
    "# T-tests made for Task 1\n",
    "# binterval is the manual algorithm\n",
    "# sleep is the device algorithm\n",
    "\n",
    "pvalue(results, \"sleep\", \"kripke\", \"F1\")\n",
    "pvalue(results, \"kripke\", \"binterval\", \"F1\")\n",
    "pvalue(results, \"SGD_hinge\", \"SGD_perceptron\", \"Recall\")\n",
    "for m in [\"F1\", \"Accuracy\"]:\n",
    "    print pvalue(results, \"sleep\", \"ExtraTrees\", m)\n",
    "    \n",
    "for m in [\"F1\", \"Accuracy\"]:\n",
    "    print pvalue(results, \"binterval\", \"ExtraTrees\", m)\n",
    "\n",
    "print (\"Compare with device:\")\n",
    "for alg in defaultdl:\n",
    "    print \"Alg: %s, p=%.3f\" % (alg, pvalue(results, \"sleep\", alg, \"F1\"))\n",
    "\n",
    "print (\"Compare with manual:\")\n",
    "for alg in defaultdl:\n",
    "    print \"Alg: %s, p=%.3f\" % (alg, pvalue(results, \"binterval\", alg, \"F1\"))\n",
    "    \n",
    "print \"p=%.3f\" % (pvalue(results, \"LSTM_20_raw\", \"LSTM_50_raw\", \"Accuracy\"))\n",
    "print \"p=%.3f\" % (pvalue(results, \"LSTM_50_raw\", \"LSTM_100_raw\", \"Accuracy\"))\n",
    "print \"p=%.3f\" % (pvalue(results, \"LSTM_20_raw\", \"LSTM_100_raw\", \"Accuracy\"))\n",
    "\n",
    "print \"p=%.3f\" % (pvalue(results, \"CNN_20_raw\", \"CNN_100_raw\", \"Accuracy\"))\n",
    "\n",
    "print \"p=%.3f\" % (pvalue(results, \"LSTM_20_raw\", \"LSTM_100_raw\", \"F1\"))\n",
    "print \"p=%.3f\" % (pvalue(results, \"CNN_20_raw\", \"CNN_100_raw\", \"F1\"))\n",
    "\n",
    "print \"p=%.3f\" % (pvalue(results, \"CNN_100_raw\", \"LSTM_100_raw\", \"F1\"))\n",
    "print \"p=%.3f\" % (pvalue(results, \"CNN_100_raw\", \"LSTM_100_raw\", \"Accuracy\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-01-10T08:33:41.531125Z",
     "start_time": "2019-01-10T08:33:41.490284Z"
    }
   },
   "outputs": [],
   "source": [
    "# T-tests made for Task 2\n",
    "print (\"Compare with device:\")\n",
    "for alg in defaultdl:\n",
    "    print \"Alg: %s, p=%.3f\" % (alg, pvalue(results, \"sleep\", alg, \"F1\"))\n",
    "\n",
    "print (\"Compare with manual:\")\n",
    "for alg in defaultdl:\n",
    "    print \"Alg: %s, p=%.3f\" % (alg, pvalue(results, \"binterval\", alg, \"F1\"))\n",
    "\n",
    "    \n",
    "print (\"Compare with device:\")\n",
    "for alg in defaultdl:\n",
    "    print \"Alg: %s, p=%.3f\" % (alg, pvalue(results, \"sleep\", alg, \"Accuracy\"))\n",
    "\n",
    "print (\"Compare with manual:\")\n",
    "for alg in defaultdl:\n",
    "    print \"Alg: %s, p=%.3f\" % (alg, pvalue(results, \"binterval\", alg, \"Accuracy\"))\n",
    "    \n",
    "    \n",
    "print \"p=%.3f\" % (pvalue(results, \"CNN_100_raw\", \"LSTM_100_raw\", \"F1\"))\n",
    "print \"p=%.3f\" % (pvalue(results, \"CNN_100_raw\", \"LSTM_100_raw\", \"Accuracy\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15rc1"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}