1.3_bloatectomy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.3 Remove duplicate text chunks for each admission\n",
    "\n",
    "# THIS MUST BE RUN WITH PYTHON 3.7.x or the regex won't work\n",
    "+ custom tokenizer to split on `. ` + `space` or `\\n` \n",
    "+ saves the concatenation of unique tokens in db  `transfused_notes_unique` `ctrl_notes_unique` for analysis\n",
    "\n",
    "+ last ran on AWS large instance Windows OS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "last ran:  2019-12-25 10:37:27.773955\n",
      "Python Version: 3.7.3 (\n",
      "operating system: darwin\n",
      "pandas version: 0.24.2\n",
      "psycopg2 version: 2.7.6.1\n",
      "tqdm version: 4.32.1\n",
      "scipy version: 1.2.1\n"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "from datetime import datetime\n",
    "import time\n",
    "import re\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "import sys\n",
    "import psycopg2\n",
    "from sqlalchemy import create_engine, update, event\n",
    "from tqdm import tnrange, tqdm_notebook\n",
    "\n",
    "from time import sleep\n",
    "from importlib_metadata import version\n",
    "\n",
    "\n",
    "conn = psycopg2.connect(\"dbname=mimic user=xxxxxxxxxxx options=--search_path=mimiciii\");\n",
    "engine = create_engine('postgresql://xxxxxxxxxxx@localhost/mimic')\n",
    "path1 = './replication/'\n",
    "    \n",
    "\n",
    "cur = conn.cursor();\n",
    "cur.execute(\"\"\"SET search_path = mimiciii;\"\"\")\n",
    "\n",
    "libraries = ['pandas','psycopg2','tqdm','scipy']\n",
    "print('last ran: ',datetime.now() )\n",
    "print(\"Python Version:\", sys.version[0:7])\n",
    "print( \"operating system:\", sys.platform)\n",
    "\n",
    "for lib in libraries:\n",
    "    print(lib + ' version: ' + version(lib))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove duplicate text from Transfused\n",
    "+  Total transfused hadm_ids = 21443\n",
    "\n",
    "### 1.3.1 create a new table `transfused_notes_unique` in the Postgres database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "cur.execute(\"\"\"DROP TABLE IF EXISTS mimiciii.transfused_notes_unique;\n",
    "\n",
    "CREATE TABLE mimiciii.transfused_notes_unique\n",
    "(hadm_id int,\n",
    " text varchar);\"\"\")\n",
    "\n",
    "conn.commit();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.3.2 get list of **hadm_id**s from the transfused admissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21443"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xf = pd.read_sql(\"\"\"\n",
    "SELECT hadm_id\n",
    "FROM mimiciii.transfused_notes_sink \"\"\", engine)\n",
    "\n",
    "xf_ids = xf.hadm_id.unique()\n",
    "len(xf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "this function helps us execute multiple calls to the database using pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "@event.listens_for(engine, 'before_cursor_execute')\n",
    "def receive_before_cursor_execute(conn, cursor, statement, params, context, executemany):\n",
    "    #print(\"FUNC call\")\n",
    "    if executemany:\n",
    "        cursor.fast_executemany = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Below are functions modified from the bloatectomy library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mark_duplicates(input_tokens):\n",
    "    '''\n",
    "    Function uses a set() and list to generates each token with html tags added to duplicate tokens.\n",
    "    INPUT: input_tokens = string of tokenized text (can be sentences, paragraphs, words etc)\n",
    "           \n",
    "    OUTPUT: a single token at a time (generator) until the end of the input_tokens.\n",
    "    '''\n",
    "\n",
    "    # create hash of tokens    \n",
    "    tokens_set = set()\n",
    "    tokens_set_add = tokens_set.add\n",
    "    \n",
    "    for token in input_tokens:\n",
    "        \n",
    "        #skip any empty tokens\n",
    "        if token == '':\n",
    "            pass\n",
    "        \n",
    "        elif token not in tokens_set:\n",
    "            tokens_set_add(token)\n",
    "            yield token\n",
    "                           \n",
    "        else:\n",
    "            pass\n",
    "        \n",
    "        \n",
    "def tokenize2(t):\n",
    "    \n",
    "\n",
    "    tok_new = []\n",
    "    \n",
    "    # find any \\n followed by an uppercase letter, a number, or a dash \n",
    "    sent_token =re.split(r\"(?=\\n\\s*[A-Z1-9#-]+.*)\", t)\n",
    "    \n",
    "    # replace \\n with a space with a space\n",
    "    sent_token = [re.sub(r\"$\\n+\",\"\",i) for i in sent_token] # remove from end\n",
    "    sent_token = [re.sub(r\"^\\n\", \"\", i) for i in sent_token] #remove from front\n",
    "        # line feeds + whitespace or not\n",
    "    sent_token = [re.sub(r\"\\s+\\n\\s+\", \" \", i) for i in sent_token] \n",
    "    sent_token = [re.sub(r\"\\s+\\n\", \" \", i) for i in sent_token]\n",
    "    sent_token = [re.sub(r\"\\n\\s+\", \" \", i) for i in sent_token]\n",
    "    sent_token = [re.sub(r\"\\n\", \" \", i) for i in sent_token]\n",
    "    \n",
    "    #remove front/end whitespace\n",
    "    sent_token = [i.strip(' ') for i in sent_token] \n",
    "\n",
    "    \n",
    "    for i in sent_token:\n",
    "        if i != '':\n",
    "            tok_new.append(i)\n",
    "        \n",
    "    return tok_new\n",
    "\n",
    "def save_tokens(token): \n",
    "    for enum_num, enum_token in enumerate(token):\n",
    "        yield str(enum_num), enum_token\n",
    "\n",
    "def bloatectomy(input_text):\n",
    "    '''Function to tokenize,remove duplicates, and output a string. \n",
    "   Tokenization is done at each period followed by a space, or a newline. \n",
    "     \n",
    "    INPUT: input_text = text to be tokenized\n",
    "    OUTPUT: an string with duplicate tokens removed. \n",
    "    '''\n",
    "    # tokenize 1\n",
    "        \n",
    "    tok = re.split(r\"(.+?\\.[\\s\\n]+)\", input_text, flags=re.DOTALL)\n",
    "    # whitespace around tokens can cause a duplicate to be missed\n",
    "    tok = [i.strip(' ') for i in tok]\n",
    "\n",
    "    #tokenize 2\n",
    "    new_tok = []\n",
    "    \n",
    "    for num, t in enumerate(tok):\n",
    "        \n",
    "        n_tok = tokenize2(t)        \n",
    "        \n",
    "        new_tok.extend(n_tok)\n",
    "    \n",
    "    # save numbered list \n",
    "    numbered_output = list(save_tokens(new_tok))     \n",
    "    \n",
    "    # detect and mark/remove duplicates\n",
    "    u_set = list(mark_duplicates(new_tok))     \n",
    "   \n",
    "    uniq =str(\"\\n \".join(u_set))\n",
    "    \n",
    "    return uniq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test the funtions on a sample of text (any string will work)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "fake_text = '''Assessment and Plan\n",
    "61 yo male Hep C cirrhosis and HCC presents with probable lower GIB and\n",
    "renal failure of unclear duration.\n",
    "Abd pain:\n",
    "-other labs: PT / PTT / INR:16.6//                         1.5, CK / CKMB /\n",
    "\n",
    "Troponin-T:4390/40/0.21, ALT / AST:258/575, Alk Phos / T Bili:232/5.9,\n",
    "ICU Care\n",
    "-other labs: PT / PTT / INR:16.6//                         1.5, CK / CKMB /\n",
    "   Communication:                                              Comments:\n",
    "icu Care\n",
    "Assessment and Plan \n",
    "Chief Complaint:\n",
    "61 yo male Hep C cirrhosis and HCC presents with probable lower GIB and\n",
    "renal failure of unclear duration.\n",
    "# Abd pain:'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Assessment and Plan\n",
      " 61 yo male Hep C cirrhosis and HCC presents with probable lower GIB and renal failure of unclear duration.\n",
      " Abd pain:\n",
      " -other labs: PT / PTT / INR:16.6//                         1.5, CK / CKMB /\n",
      " Troponin-T:4390/40/0.21, ALT / AST:258/575, Alk Phos / T Bili:232/5.9,\n",
      " ICU Care\n",
      " Communication:                                              Comments: icu Care\n",
      " Chief Complaint:\n",
      " # Abd pain:\n"
     ]
    }
   ],
   "source": [
    "print(bloatectomy(fake_text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test on a single admission's notes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "hadm_id_sample = #an hadm_id from the database\n",
    "pt_all_xf = pd.read_sql(\"\"\"SELECT * FROM mimiciii.transfused_notes_sink WHERE hadm_id IN (hadm_id_sample);\"\"\", engine)\n",
    "print(bloatectomy(pt_all_xf))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.3.3 Remove Duplicates and save as `transfused_notes_sink`\n",
    "+ Split each admission's notes into sentences, strip whitespace from edges, remove duplicates, join, store in new table\n",
    "should take approx 19 minutes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "813acae7d55c4fb2928dc70beb0a61d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=21443), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "total- 27.955767035484314 minutes\n"
     ]
    }
   ],
   "source": [
    "s = time.time()\n",
    "\n",
    "for j in tqdm_notebook(xf_ids):\n",
    "    \n",
    "    sql = \"\"\"\n",
    "    \n",
    "    SELECT hadm_id, text\n",
    "    FROM mimiciii.transfused_notes_sink \n",
    "        WHERE hadm_id in ({0})\n",
    "    \"\"\"\n",
    "    \n",
    "    # run sql query above to pull all notes for one admission (already in order by date)\n",
    "    sql = sql.format(j)\n",
    "    xnotes = pd.read_sql(sql, engine)\n",
    "    \n",
    "    # split into tokens based on a period followed by a space or a newline \\n (the period and \\n are included in the token)\n",
    "    input_notes = xnotes.text.tolist()[0]\n",
    "    \n",
    "    unique_tokens = bloatectomy(input_notes)\n",
    "     \n",
    "    # save as a new dataframe\n",
    "    xtext2 = [(j, unique_tokens)]\n",
    "    xfulltext=pd.DataFrame(xtext2, columns=['hadm_id', 'text'])\n",
    "    \n",
    "    # append user and single note to the new table in database\n",
    "    table_name = 'transfused_notes_unique'\n",
    "    \n",
    "    \n",
    "    z = time.time()\n",
    "    \n",
    "    xfulltext.to_sql(table_name, con=engine, if_exists='append', chunksize=1, index=False, schema='mimiciii')\n",
    "    \n",
    "    #print('per hadmid-',(time.time() - z)/60,'min')\n",
    "\n",
    "print('total-', (time.time() - s)/60,'minutes')\n",
    "\n",
    "    \n",
    "conn.commit()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Print counts of notes and admissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " total notes count  xf admissions with notes\n",
      "             21443                     21443\n"
     ]
    }
   ],
   "source": [
    "cur.execute(\"\"\" SELECT COUNT(*), COUNT(DISTINCT hadm_id) FROM transfused_notes_unique;\"\"\")\n",
    "\n",
    "print( pd.DataFrame(cur.fetchall(), columns=[ 'total notes count', 'xf admissions with notes']).to_string(index=False))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove duplicate text from Control (non-transfused)\n",
    "+  unique control admissions = 27,888\n",
    "\n",
    "### 1.3.4 Create new table `ctrl_notes_unique`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "cur.execute(\"\"\"DROP TABLE IF EXISTS mimiciii.ctrl_notes_unique;\n",
    "\n",
    "CREATE TABLE mimiciii.ctrl_notes_unique\n",
    "(hadm_id int,\n",
    " text varchar);\"\"\")\n",
    "\n",
    "conn.commit();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.3.5 Load ctrl hadm_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "xf = pd.read_sql(\"\"\"\n",
    "SELECT hadm_id\n",
    "FROM mimiciii.ctrl_notes_sink \"\"\", engine)\n",
    "\n",
    "# get list of ids\n",
    "cxf_ids = xf.hadm_id.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.3.6 Bloatectomize the control notes and save into `ctrl_notes_unique`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "this will take about 15 minutes to run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c014be87e0fa4ff8847669b2a432ff28",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=27888), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "total- 38.29251765012741 minutes\n"
     ]
    }
   ],
   "source": [
    "s = time.time()\n",
    "\n",
    "for j in tqdm_notebook(cxf_ids):\n",
    "   \n",
    "    \n",
    "    sql = \"\"\"\n",
    "    SELECT hadm_id, text\n",
    "    FROM mimiciii.ctrl_notes_sink \n",
    "        WHERE hadm_id in ({0})\n",
    "    \"\"\"\n",
    "    \n",
    "    # run sql query above to pull all notes for one admission (already in order by date)\n",
    "    sql = sql.format(j)\n",
    "    cnotes = pd.read_sql(sql, engine)\n",
    "    \n",
    "     # split into tokens based on a period followed by a space or a newline \\n (the period and \\n are included in the token)\n",
    "    input_cnotes = cnotes.text.tolist()[0]\n",
    "    \n",
    "    unique_tokens = bloatectomy(input_cnotes)\n",
    "    \n",
    "    \n",
    "    # save as a new dataframe\n",
    "    xtext2 = [(j, unique_tokens)]\n",
    "    xfulltext=pd.DataFrame(xtext2, columns=['hadm_id', 'text'])\n",
    "    \n",
    "    # append user and single note to the new table in database\n",
    "    table_name = 'ctrl_notes_unique'\n",
    "    \n",
    "    \n",
    "    z = time.time()\n",
    "    \n",
    "    xfulltext.to_sql(table_name, con=engine, if_exists='append', chunksize=1, index=False, schema='mimiciii')\n",
    "    \n",
    "   # print('per hadmid-',time.time() - z)\n",
    "\n",
    "print('total-', (time.time() - s)/60,'minutes')\n",
    "\n",
    "    \n",
    "conn.commit()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Print note and admissions counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " total notes count  ctrl admissions with notes\n",
      "             27888                       27888\n"
     ]
    }
   ],
   "source": [
    "cur.execute(\"\"\" SELECT COUNT(*), COUNT(DISTINCT hadm_id) FROM ctrl_notes_unique;\"\"\")\n",
    "\n",
    "print( pd.DataFrame(cur.fetchall(), columns=[ 'total notes count', 'ctrl admissions with notes']).to_string(index=False))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "conn.commit()\n",
    "cur.close()\n",
    "conn.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}