Reproduce CRAFT-MD results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f0b405c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1279cbe0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set OpenAI API Key\n",
    "import openai\n",
    "deployment_name = \"fm-endo\"\n",
    "openai.api_base = f\"https://{deployment_name}.openai.azure.com/\"\n",
    "openai.api_key = pd.read_csv(\"/home/shj622/rajpurkarlab/home/shj622/craft-md-v2/keys/azure_fmendo.txt\", \n",
    "                             header=None).iloc[0,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "483c4570",
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.utils import get_choices"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c6ad83ae",
   "metadata": {},
   "source": [
    "## Generate clinical LLM agent responses for vignette and conversational formats (multi-turn, single-turn and summarized)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01667361",
   "metadata": {},
   "source": [
    "### OpenAI Models (GPT-3.5, GPT-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a24582f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.craftmd import craftmd_gpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45ae0412",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"gpt4_1106\"\n",
    "dataset = pd.read_csv(\"./data/usmle_derm_dataset.csv\", index_col=0)\n",
    "\n",
    "cases = [(dataset.loc[idx,\"case_id\"], \n",
    "          dataset.loc[idx,\"case_vignette\"], \n",
    "          dataset.loc[idx,\"category\"],\n",
    "          get_choices(dataset,idx)) for idx in dataset.index]\n",
    "\n",
    "path_dir = f\"./results/{model_name}\"\n",
    "\n",
    "case = cases[0]\n",
    "craftmd_gpt(case, path_dir, model_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3b2206c",
   "metadata": {},
   "source": [
    "### Open-source Models (Mistral-v1, Mistral-v2, LLaMA2-7b)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b08b549c",
   "metadata": {},
   "source": [
    "These models require GPU resources. We performed all evaluations on Quadro RTX 8000 48gb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92aea9aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.craftmd import craftmd_opensource\n",
    "from src.models import get_model_and_tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c45e9315",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # To download open-source models, if not already installed in your conda environment\n",
    "# from huggingface_hub import login\n",
    "# login(token = \"insert_huggingface_token\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a995f6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"mistral-v2\"\n",
    "dataset = pd.read_csv(\"./data/usmle_derm_dataset.csv\", index_col=0)\n",
    "\n",
    "cases = [(dataset.loc[idx,\"case_id\"], \n",
    "          dataset.loc[idx,\"case_vignette\"], \n",
    "          dataset.loc[idx,\"category\"],\n",
    "          get_choices(dataset,idx)) for idx in dataset.index]\n",
    "\n",
    "path_dir = f\"./results/{model_name}\"\n",
    "\n",
    "case = cases[0]\n",
    "\n",
    "model, tokenizer = get_model_and_tokenizer(model_name)\n",
    "\n",
    "if tokenizer.pad_token_id is None:\n",
    "    tokenizer.pad_token_id = tokenizer.eos_token_id \n",
    "\n",
    "craftmd_opensource(case, path_dir, model, tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15e8f4d1",
   "metadata": {},
   "source": [
    "### Multimodal LLM (GPT-4V)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a820ee2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.craftmd import craftmd_multimodal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2147dc3",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"gpt4v\"\n",
    "dataset = pd.read_csv(\"./data/nejmai_dataset.csv\", index_col=0)\n",
    "\n",
    "cases = [(dataset.loc[idx,\"case_id\"],\n",
    "          dataset.loc[idx,\"case_vignette\"],\n",
    "          get_choices(dataset,idx)) for idx in dataset.index]\n",
    "\n",
    "path_dir = f\"./results/{model_name}\"\n",
    "img_dir = f\"../../craft-md-v2/data/nejm/imgs/\"\n",
    "\n",
    "craftmd_multimodal(cases[0], img_dir, path_dir, deployment_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "866c7b7e",
   "metadata": {},
   "source": [
    "# Evaluate using Grader-AI agent"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ebde4a6",
   "metadata": {},
   "source": [
    "Below code works for both open-source and GPT models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5acc40d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.graderai_eval import graderai_evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24cc2022",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"gpt4_1106\"\n",
    "dataset = pd.read_csv(\"./data/usmle_derm_dataset.csv\", index_col=0)\n",
    "\n",
    "experiment_names = [\"vignette_frq\", \"multiturn_frq\", \"singleturn_frq\", \"summarized_frq\"]\n",
    "path_dir = f\"./results/{model_name}\"\n",
    "\n",
    "graderai_evaluation(\"case_0\", dataset, path_dir, experiment_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3348dfa",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc9f37d8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42c269a3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1fe903b7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fbfc648",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}