notebooks/predict.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/psaegert/miniconda3/envs/fsem/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import torch\n",
    "import evaluate\n",
    "\n",
    "from tqdm import tqdm\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "from datasets import load_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nlinec import get_positive_data, get_all_types, get_results_dir, get_models_dir, combine_premise_hypothesis\n",
    "from nlinec.predict import predict_probabilities"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify the dataset to predict and a file to save the predictions to\n",
    "SPLIT = \"dev\"\n",
    "DATASET = f'g_{SPLIT}.json'\n",
    "MODEL = \"nlinec-E-2\"\n",
    "HYPOTHESIS_ONLY = False\n",
    "\n",
    "SAVE_MODEL_TO = os.path.join(get_models_dir(), MODEL)\n",
    "SAVE_PREDICTIONS_TO = os.path.join(get_results_dir(), MODEL, f\"{SPLIT}_predictions\" + (\"_ho\" if HYPOTHESIS_ONLY else \"\") + \".csv\")\n",
    "\n",
    "# Specify the parameters for the prediction\n",
    "SAVE_EVERY = 100_000\n",
    "\n",
    "# Use the GPU if available\n",
    "DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# Make sure the directory to save the predictions to exists\n",
    "os.makedirs(os.path.dirname(SAVE_PREDICTIONS_TO), exist_ok=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Models & Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained('roberta-large-mnli')\n",
    "\n",
    "if MODEL == 'roberta-large-mnli':\n",
    "    model = AutoModelForSequenceClassification.from_pretrained(MODEL).to(DEVICE)\n",
    "else:\n",
    "    model = AutoModelForSequenceClassification.from_pretrained(SAVE_MODEL_TO).to(DEVICE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>full_type</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/other</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/location/country</td>\n",
       "      <td>country</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/location</td>\n",
       "      <td>location</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/other/scientific</td>\n",
       "      <td>scientific</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/location/city</td>\n",
       "      <td>city</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>/location/geography/body_of_water</td>\n",
       "      <td>body_of_water</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>/location/geograpy/island</td>\n",
       "      <td>island</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>/location/geograpy</td>\n",
       "      <td>geograpy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>/other/legal</td>\n",
       "      <td>legal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>/other/product/mobile_phone</td>\n",
       "      <td>mobile_phone</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>89 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                            full_type           type\n",
       "0                              /other          other\n",
       "1                   /location/country        country\n",
       "2                           /location       location\n",
       "3                   /other/scientific     scientific\n",
       "4                      /location/city           city\n",
       "..                                ...            ...\n",
       "84  /location/geography/body_of_water  body_of_water\n",
       "85          /location/geograpy/island         island\n",
       "86                 /location/geograpy       geograpy\n",
       "87                       /other/legal          legal\n",
       "88        /other/product/mobile_phone   mobile_phone\n",
       "\n",
       "[89 rows x 2 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make entailment predictions for all types and filter out the relevant ones later in the analysis\n",
    "all_types = get_all_types(granularity=-1)\n",
    "all_types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading g_dev.json: 2202it [00:00, 221829.17it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mention_span</th>\n",
       "      <th>full_type</th>\n",
       "      <th>sentence</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Friday</td>\n",
       "      <td>[/other]</td>\n",
       "      <td>Japan's wholesale prices in September rose 3.3...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>September</td>\n",
       "      <td>[/other]</td>\n",
       "      <td>Japan's wholesale prices in September rose 3.3...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Japan</td>\n",
       "      <td>[/location, /location/country]</td>\n",
       "      <td>Japan's wholesale prices in September rose 3.3...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>the Bank of Japan</td>\n",
       "      <td>[/location, /location/structure, /organization...</td>\n",
       "      <td>Japan's wholesale prices in September rose 3.3...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.3 %</td>\n",
       "      <td>[/other]</td>\n",
       "      <td>Japan's wholesale prices in September rose 3.3...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2197</th>\n",
       "      <td>the Treasury 's</td>\n",
       "      <td>[/organization, /organization/government]</td>\n",
       "      <td>The non-callable issue, which can be put back ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2198</th>\n",
       "      <td>$ 500 million of Remic mortgage securities</td>\n",
       "      <td>[/other]</td>\n",
       "      <td>$ 500 million of Remic mortgage securities of...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2199</th>\n",
       "      <td>the Treasury 's</td>\n",
       "      <td>[/organization, /organization/government]</td>\n",
       "      <td>The issue, which is puttable back to the compa...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2200</th>\n",
       "      <td>200 basis points</td>\n",
       "      <td>[/other]</td>\n",
       "      <td>Among classes for which details were available...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2201</th>\n",
       "      <td>$ 200 million</td>\n",
       "      <td>[/other, /other/currency]</td>\n",
       "      <td>$ 200 million of stripped mortgage securities...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2202 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    mention_span  \\\n",
       "0                                         Friday   \n",
       "1                                      September   \n",
       "2                                          Japan   \n",
       "3                              the Bank of Japan   \n",
       "4                                          3.3 %   \n",
       "...                                          ...   \n",
       "2197                             the Treasury 's   \n",
       "2198  $ 500 million of Remic mortgage securities   \n",
       "2199                             the Treasury 's   \n",
       "2200                            200 basis points   \n",
       "2201                               $ 200 million   \n",
       "\n",
       "                                              full_type  \\\n",
       "0                                              [/other]   \n",
       "1                                              [/other]   \n",
       "2                        [/location, /location/country]   \n",
       "3     [/location, /location/structure, /organization...   \n",
       "4                                              [/other]   \n",
       "...                                                 ...   \n",
       "2197          [/organization, /organization/government]   \n",
       "2198                                           [/other]   \n",
       "2199          [/organization, /organization/government]   \n",
       "2200                                           [/other]   \n",
       "2201                          [/other, /other/currency]   \n",
       "\n",
       "                                               sentence  label  \n",
       "0     Japan's wholesale prices in September rose 3.3...      2  \n",
       "1     Japan's wholesale prices in September rose 3.3...      2  \n",
       "2     Japan's wholesale prices in September rose 3.3...      2  \n",
       "3     Japan's wholesale prices in September rose 3.3...      2  \n",
       "4     Japan's wholesale prices in September rose 3.3...      2  \n",
       "...                                                 ...    ...  \n",
       "2197  The non-callable issue, which can be put back ...      2  \n",
       "2198   $ 500 million of Remic mortgage securities of...      2  \n",
       "2199  The issue, which is puttable back to the compa...      2  \n",
       "2200  Among classes for which details were available...      2  \n",
       "2201   $ 200 million of stripped mortgage securities...      2  \n",
       "\n",
       "[2202 rows x 4 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the dataset\n",
    "data = get_positive_data(DATASET)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If some predictions already exist, load them\n",
    "if os.path.exists(SAVE_PREDICTIONS_TO):\n",
    "    # Load the predictions from file\n",
    "    print(\"Loading predictions from file\")\n",
    "    predictions_df = pd.read_csv(SAVE_PREDICTIONS_TO, index_col=0)\n",
    "else:\n",
    "    # Create a dataframe with the same index as the data\n",
    "    predictions_df = pd.DataFrame(columns=list(all_types['full_type']), index=data.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Progress: 0.00%\n"
     ]
    }
   ],
   "source": [
    "# Find out which predictions still need to be made\n",
    "todo = predictions_df.isna().any(axis=1)\n",
    "print(f'Progress: {(~todo).mean() * 100:.2%}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NEC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2202/2202 [03:37<00:00, 10.12it/s]\n"
     ]
    }
   ],
   "source": [
    "# Make predictions for the remaining rows\n",
    "with torch.no_grad():  # Disable gradient calculation for speed\n",
    "    # Keep track of how many predictions have been made since the last save\n",
    "    new_predictions_counter = 0\n",
    "\n",
    "    # Iterate over all rows in the dev data\n",
    "    for row in tqdm(data.loc[todo, :].itertuples(), total=todo.sum()):\n",
    "\n",
    "        # Predict the type of the mention and store the prediction\n",
    "        entailment_probabilities = predict_probabilities(\n",
    "            model,\n",
    "            tokenizer,\n",
    "            row.sentence,\n",
    "            row.mention_span,\n",
    "            all_types['type'],\n",
    "            hypothesis_only=HYPOTHESIS_ONLY)[0, :, -1]  # -1 is the entailment class\n",
    "\n",
    "        # Store the prediction\n",
    "        predictions_df.loc[row.Index, :] = entailment_probabilities\n",
    "\n",
    "        # Save the predictions to file every SAVE_EVERY predictions\n",
    "        new_predictions_counter += 1\n",
    "        if new_predictions_counter >= SAVE_EVERY:\n",
    "            \n",
    "            # Save the predictions to file\n",
    "            predictions_df.to_csv(SAVE_PREDICTIONS_TO)\n",
    "            new_predictions_counter = 0\n",
    "\n",
    "# Save the remaining predictions to file\n",
    "predictions_df.to_csv(SAVE_PREDICTIONS_TO)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multi-NLI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = load_dataset(\"multi_nli\", split='validation_matched')\n",
    "metric = evaluate.load('accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    predictions = []\n",
    "    for instance in tqdm(dataset):\n",
    "        input_text = combine_premise_hypothesis(instance[\"premise\"], instance[\"hypothesis\"])\n",
    "        tokenized_input_text = tokenizer(input_text, max_length=model.config.max_position_embeddings, padding=\"max_length\", return_tensors='pt')\n",
    "        outputs = model(tokenized_input_text['input_ids'].to(DEVICE), tokenized_input_text['attention_mask'].to(DEVICE))\n",
    "        predictions.append(outputs[0].argmax(dim=1).cpu().numpy()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy on MultiNLI: 0.701\n"
     ]
    }
   ],
   "source": [
    "# This took an embarrassingly long time to figure out. Why is the model's label mapping different from the dataset's label mapping?\n",
    "# https://huggingface.co/datasets/multi_nli/viewer/default/validation_matched\n",
    "dataset_id2label = {'ENTAILMENT': 0, 'NEUTRAL': 1, 'CONTRADICTION': 2}\n",
    "model_id2label = model.config.id2label\n",
    "\n",
    "# Convert the model's predictions to the dataset's labels\n",
    "model_labels_for_dataset = [dataset_id2label[model_id2label[logit]] for logit in predictions]\n",
    "\n",
    "# Compute the accuracy\n",
    "accuracy = metric.compute(references=dataset['label'], predictions=model_labels_for_dataset)\n",
    "\n",
    "print(f'Accuracy on MultiNLI: {accuracy[\"accuracy\"]:.3f}')\n",
    "\n",
    "# Save the accuracy\n",
    "pd.DataFrame(accuracy, index=['accuracy']).rename(columns={'accuracy': MODEL}).to_csv(os.path.join(get_results_dir(), f'{MODEL}', 'mnli_accuracy.csv'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fsem",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}