notebooks/results.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "from functools import reduce\n",
    "\n",
    "from nlinec import get_models_dir, get_results_dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "ACCURACY_FILE = 'test_accuracy.csv'\n",
    "ACCURACY_FILE_HYPOTHESIS_ONLY = 'test_accuracy_ho.csv'\n",
    "MNLI_ACCURACY_FILE = 'mnli_accuracy.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['nlinec-AE-2', 'nlinec-B-2', 'nlinec-C-2', 'nlinec-D-1', 'nlinec-D-2', 'nlinec-D-3', 'nlinec-E-2', 'roberta-large-mnli']\n"
     ]
    }
   ],
   "source": [
    "# Print all models\n",
    "models = sorted(os.listdir(get_results_dir()))\n",
    "print(models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the accuracy files\n",
    "accuracy = {}\n",
    "accuracy_ho = {}\n",
    "mnli_accuracy = {}\n",
    "for model in models:\n",
    "    if os.path.exists(os.path.join(get_results_dir(), model, ACCURACY_FILE)):\n",
    "        accuracy[model] = pd.read_csv(os.path.join(get_results_dir(), model, ACCURACY_FILE), index_col=0).rename(columns={'accuracy': model})\n",
    "\n",
    "    if os.path.exists(os.path.join(get_results_dir(), model, ACCURACY_FILE_HYPOTHESIS_ONLY)):\n",
    "        accuracy_ho[model] = pd.read_csv(os.path.join(get_results_dir(), model, ACCURACY_FILE_HYPOTHESIS_ONLY), index_col=0).rename(columns={'accuracy': model})\n",
    "\n",
    "    if os.path.exists(os.path.join(get_results_dir(), model, MNLI_ACCURACY_FILE)):\n",
    "        mnli_accuracy[model] = pd.read_csv(os.path.join(get_results_dir(), model, MNLI_ACCURACY_FILE), index_col=0).rename(columns={'accuracy': model})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merge all accuracy dataframes on the 'granularity' column\n",
    "accuracy_df = reduce(lambda left, right: pd.merge(left, right, on='granularity'), accuracy.values()).T.rename(columns={0: 'Coarse', 1: 'Medium', 2: 'Fine'})\n",
    "accuracy_ho_df = reduce(lambda left, right: pd.merge(left, right, on='granularity'), accuracy_ho.values()).T.rename(columns={0: 'Coarse HO', 1: 'Medium HO', 2: 'Fine HO'})\n",
    "mnli_accuracy_df = pd.concat(mnli_accuracy.values(), axis=1).T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Coarse</th>\n",
       "      <th>Medium</th>\n",
       "      <th>Fine</th>\n",
       "      <th>Multi-NLI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>nlinec-C-2</th>\n",
       "      <td>0.67</td>\n",
       "      <td>0.69</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nlinec-D-1</th>\n",
       "      <td>0.88</td>\n",
       "      <td>0.47</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nlinec-D-2</th>\n",
       "      <td>0.61</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.71</td>\n",
       "      <td>0.82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nlinec-D-3</th>\n",
       "      <td>0.53</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nlinec-E-2</th>\n",
       "      <td>0.16</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>roberta-large-mnli</th>\n",
       "      <td>0.46</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.44</td>\n",
       "      <td>0.90</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Coarse  Medium  Fine  Multi-NLI\n",
       "nlinec-C-2            0.67    0.69  0.70       0.74\n",
       "nlinec-D-1            0.88    0.47  0.63       0.84\n",
       "nlinec-D-2            0.61    0.70  0.71       0.82\n",
       "nlinec-D-3            0.53    0.63  0.73       0.81\n",
       "nlinec-E-2            0.16    0.07  0.12       0.70\n",
       "roberta-large-mnli    0.46    0.35  0.44       0.90"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Combine the accuracy and mnli_accuracy dataframes\n",
    "# Rename the Granularity columns from 0, 1, 2 to 'Coarse', 'Medium', 'Fine'\n",
    "# Rename the 'accuracy' column from the mnli_accuracy dataframe to 'MNLI Accuracy'\n",
    "# Remove the 'granularity' row from the resulting dataframe\n",
    "combined_df = pd.concat([accuracy_df, mnli_accuracy_df], axis=1).rename(columns={'accuracy': 'Multi-NLI'}).drop('granularity', axis=0)\n",
    "combined_df.round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hypothesis-Only Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Coarse HO</th>\n",
       "      <th>Medium HO</th>\n",
       "      <th>Fine HO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>nlinec-C-2</th>\n",
       "      <td>0.53</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nlinec-D-1</th>\n",
       "      <td>0.84</td>\n",
       "      <td>0.36</td>\n",
       "      <td>0.54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nlinec-D-2</th>\n",
       "      <td>0.42</td>\n",
       "      <td>0.59</td>\n",
       "      <td>0.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nlinec-D-3</th>\n",
       "      <td>0.38</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nlinec-E-2</th>\n",
       "      <td>0.24</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>roberta-large-mnli</th>\n",
       "      <td>0.22</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.36</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Coarse HO  Medium HO  Fine HO\n",
       "nlinec-C-2               0.53       0.59     0.67\n",
       "nlinec-D-1               0.84       0.36     0.54\n",
       "nlinec-D-2               0.42       0.59     0.60\n",
       "nlinec-D-3               0.38       0.50     0.64\n",
       "nlinec-E-2               0.24       0.05     0.14\n",
       "roberta-large-mnli       0.22       0.19     0.36"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_ho_df.drop('granularity', axis=0).round(2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fsem",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}