transformer learner.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(action='once')\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pylab as plt\n",
    "import seaborn as sns\n",
    "import datetime\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\"\n",
    "import pickle\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "from exp.misc import *\n",
    "from exp.ProcessData import *\n",
    "from exp.PytorchModels import *\n",
    "from exp.LearnerClass import *\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.utils.data as D\n",
    "import torch.nn.functional as F\n",
    "import copy\n",
    "from torchvision import transforms\n",
    "import PIL.Image\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import torchvision.transforms.functional as TF\n",
    "from types import MethodType\n",
    "import sandesh\n",
    "import pydicom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params=json_to_parameters('config.json')\n",
    "num_folds=5\n",
    "SEED=220\n",
    "folds=[1]\n",
    "device = device_by_name('RTX')\n",
    "add_seed=220221\n",
    "model_type='tf_efficientnet_b5_ns'\n",
    "basic_version = 'pdint'\n",
    "basic_name_tamplate='image_{}'\n",
    "num_heads=2\n",
    "num_layers=4\n",
    "ffdim=2048\n",
    "dropout=0.1\n",
    "name_tamplate='transformer'+f'_{num_heads}_{num_layers}_{ffdim}_'+'{}'\n",
    "model_version='pdint128n03f02'\n",
    "gamma=0.2\n",
    "fnoise=0.3\n",
    "val_max_len=512\n",
    "train_max_len=128\n",
    "torch.cuda.set_device(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(params.path.data+'full_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def masked_bce_with_logits(y_pred,y_true,mask=None,reduction='mean',gamma=0.):\n",
    "    if mask is None:\n",
    "        out = F.binary_cross_entropy_with_logits(y_pred,y_true)\n",
    "    else:\n",
    "        yp=torch.where(mask,y_pred,-20*torch.ones_like(y_pred))\n",
    "        yt=torch.where(mask,y_true,torch.zeros_like(y_pred))\n",
    "        if gamma==0:\n",
    "            loss = F.binary_cross_entropy_with_logits(yp,yt,reduction='none')\n",
    "        else:\n",
    "            loss = binary_focal_loss(yp,yt,gamma=gamma,reduction='none')\n",
    "        if reduction=='mean':\n",
    "            out=loss.sum()/mask.sum()\n",
    "        if reduction=='sum':\n",
    "            out=loss.sum()\n",
    "        else:\n",
    "            out=loss\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols=  ['rv_lv_ratio_gte_1',\n",
    "        'rv_lv_ratio_lt_1',\n",
    "        'leftsided_pe',\n",
    "        'chronic_pe',\n",
    "        'negative_exam_for_pe',\n",
    "        'rightsided_pe',\n",
    "        'acute_and_chronic_pe',\n",
    "        'central_pe',\n",
    "        'indeterminate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "OutputMap={'dummy0':0,\n",
    "           'dummy1':1,\n",
    "           'dummy2':2,\n",
    "           'dummy3':3,\n",
    "           'true_filling_defect_not_pe':0,\n",
    "           'qa_motion':1,\n",
    "           'qa_contrast':2,\n",
    "           'flow_artifact':3,\n",
    "           'rv_lv_ratio_gte_1':4,\n",
    "           'rv_lv_ratio_lt_1':5,\n",
    "           'leftsided_pe':6,\n",
    "           'chronic_pe':7,\n",
    "           'negative_exam_for_pe':8,\n",
    "           'rightsided_pe':9,\n",
    "           'acute_and_chronic_pe':10,\n",
    "           'central_pe':11,\n",
    "           'indeterminate':12}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lvs=[OutputMap['rv_lv_ratio_gte_1'],\n",
    "     OutputMap['rv_lv_ratio_lt_1'],\n",
    "     OutputMap['negative_exam_for_pe'],\n",
    "     OutputMap['indeterminate']]\n",
    "chron=[OutputMap['chronic_pe'],\n",
    "       OutputMap['acute_and_chronic_pe'],\n",
    "       OutputMap['dummy0'],\n",
    "       OutputMap['negative_exam_for_pe'],\n",
    "       OutputMap['indeterminate']]\n",
    "\n",
    "def calc_pred(y):\n",
    "    lv = F.log_softmax(y[:,lvs],dim=1)\n",
    "    chn = F.log_softmax(y[:,chron],dim=1)\n",
    "    out = y\n",
    "    out[:,lvs[:2]]=lv[:,:2]\n",
    "    out[:,chron[:2]]=chn[:,:2]\n",
    "    out[:,lvs[-2:]]=(lv[:,-2:]+chn[:,-2:])/2\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "WEIGHTS=torch.tensor([0.0,0.0,0.0,0.0,\n",
    "                      0.2346625767,\n",
    "                      0.0782208589,\n",
    "                      0.06257668712,\n",
    "                      0.1042944785,\n",
    "                      0.0736196319,\n",
    "                      0.06257668712,\n",
    "                      0.1042944785,\n",
    "                      0.1877300613,\n",
    "                      0.09202453988])\n",
    "class MyLoss():\n",
    "    def __init__(self,weight_image=0.07361963,weights_series=WEIGHTS,weight_gen_img=0.1,\n",
    "                 conf_weight=0.3,mean_len=246.,eq_series=1.,gamma=0.,image_series=0.5,eth=0.,use_max=False,do_calc=False):\n",
    "        self.weight_image=weight_image\n",
    "        self.weights_series=weights_series\n",
    "        self.weight_gen_img=weight_gen_img\n",
    "        self.mean_len=mean_len\n",
    "        self.eq_series=eq_series/2\n",
    "        self.conf_weight=conf_weight\n",
    "        self.gamma=gamma\n",
    "        self.image_series=image_series\n",
    "        self.eth=eth\n",
    "        self.use_max=use_max\n",
    "        self.calc = calc_pred if do_calc else lambda x:x\n",
    "    def __call__(self,y_pred,y_true):\n",
    "        wim=2*self.image_series\n",
    "        wis=2*(1-self.image_series)\n",
    "        mask=y_true[0]>=0\n",
    "        mi=(y_true[0]*mask).sum(1).to(torch.float32)\n",
    "        ni=mask.sum(1).to(torch.float32)\n",
    "        qi=mi/ni\n",
    "        wi=self.weight_image*qi\n",
    "        yp= y_pred[...,0]\n",
    "        yprs=y_pred.max(1)[0] if self.use_max else y_pred[:,-1,:]\n",
    "        l0=masked_bce_with_logits(yp,torch.clamp(y_true[0],self.eth,1-self.eth),mask=mask,reduction='none',gamma=self.gamma)\n",
    "        bc2=binary_focal_loss(self.calc(yprs[:,1:]),torch.clamp(y_true[1],self.eth,1-self.eth),gamma=self.gamma,reduction='none')\n",
    "        npe=yprs[:,[9,13]].max(1)[0]\n",
    "\n",
    "        conf=torch.stack([yp.max(1)[0]*npe,\n",
    "                           npe*y_pred[:,-1,[5,6]].max(1)[0],\n",
    "                           y_pred[:,-1,5]*y_pred[:,-1,6]*(npe<0).to(float),\n",
    "                           npe*y_pred[:,-1,[7,10,12]].max(1)[0],\n",
    "                           npe*y_pred[:,-1,[8,11]].max(1)[0],\n",
    "                           y_pred[:,-1,8]*y_pred[:,-1,11]*(npe<0)],1)\n",
    "        bconf = binary_focal_loss(yprs[:,1:],torch.clamp(y_true[1],self.eth,1-self.eth),gamma=self.gamma,reduction='mean')*bc2.shape[0]*self.conf_weight\n",
    "        m= ((wis*bc2*self.weights_series).sum()*self.eq_series+bconf+wim*(l0.sum(1)*wi).sum())/(wim*(wi*ni).sum()+wis*self.weights_series.sum()*bc2.shape[0]*self.eq_series)    \n",
    "        li3=self.weight_gen_img*l0.sum()/mask.sum()\n",
    "        return m+li3\n",
    "        \n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyMetric():\n",
    "    def __init__(self,weight_image=0.07361963,weights_series=WEIGHTS,weight_gen_img=0.1,mean_len=246.,eq_series=1.,do_calc=False,use_max=False,image_series=0.5):\n",
    "        self.weight_image=weight_image\n",
    "        self.weights_series=weights_series\n",
    "        self.weight_gen_img=weight_gen_img\n",
    "        self.mean_len=mean_len\n",
    "        self.image_series=image_series\n",
    "        self.eq_series=eq_series/2\n",
    "        self.use_max=use_max\n",
    "        self.calc = calc_pred if do_calc else lambda x:x\n",
    "    def __call__(self,y_pred,y_true):\n",
    "        wim=2*self.image_series\n",
    "        wis=2*(1-self.image_series)\n",
    "        yt0=torch.tensor(y_true[0])\n",
    "        mask=yt0>=0\n",
    "        mi=(yt0>0).sum(1).to(torch.float32)\n",
    "        ni=(yt0>=0).sum(1).to(torch.float32)\n",
    "        qi=mi/ni\n",
    "        wi=self.weight_image*qi\n",
    "        yp= torch.tensor(y_pred[...,0])\n",
    "        l0=masked_bce_with_logits(yp,yt0,mask=mask,reduction='none')\n",
    "        yprs=y_pred.max(1) if self.use_max else y_pred[:,-1,:]\n",
    "#         yprs=y_pred.max(1)\n",
    "        li1=float((l0.sum(1)*wi).sum()/(ni*wi).sum())\n",
    "        bc2=F.binary_cross_entropy_with_logits(self.calc(torch.tensor(yprs[:,1:])),\n",
    "                                                     torch.tensor(y_true[1],dtype=torch.float32),reduction='none')\n",
    "        li2=float((bc2.mean(0)[4:]*self.weights_series[4:]).sum())\n",
    "        li21=float((bc2.mean(0)[:4]*self.weights_series[:4]).sum())\n",
    "        li3=float(self.weight_gen_img*l0.sum()/mask.sum())\n",
    "        m= (wis*(bc2[:,4:]*self.weights_series[4:]).sum()*self.eq_series+wim*(l0.sum(1)*wi).sum())/(wim*(wi*ni).sum()+wis*self.weights_series[4:].sum()*bc2.shape[0]*self.eq_series)\n",
    "        metric_dict={'metric':float(m),'images':li1,'series':li2,'li21':li21,'li3':li3}\n",
    "        return metric_dict \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "epoch_in_rep=10\n",
    "reps =2\n",
    "num_epochs=epoch_in_rep*reps\n",
    "batch_size=32\n",
    "reps_lr=[1e-4*batch_size/32,3e-5*batch_size/32,1e-5*batch_size/32]\n",
    "linear_embd=OrderedDict([('slice',[16,16])]) #OrderedDict([('extra',[16,16]),('slice',[16,16])])\n",
    "extra=None #{'col':'kvp','norm':100.,'noise':0.03}\n",
    "extra_val=None #{'col':'kvp','norm':100.,'noise':0.}\n",
    "for SEED in [220]:\n",
    "    val_folds, train_folds, patients_val = create_folds(df,num_folds,SEED)\n",
    "    all_patients=[]\n",
    "    for p in patients_val:\n",
    "        all_patients.extend(p)\n",
    "    for fold in folds:\n",
    "        torch.manual_seed(SEED+fold+add_seed)\n",
    "        np.random.seed(SEED+fold+add_seed)\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "        basic_name=params.model_format.format(model_type,basic_name_tamplate.format(SEED),basic_version,fold).split('.')[0]+'.pkl'\n",
    "        with open(params.path.features+basic_name,'rb') as f:\n",
    "            features0=pickle.load(f)\n",
    "        print('nans=',np.isnan(features0).sum())\n",
    "        features=torch.tensor(np.nan_to_num(features0),dtype=torch.float32)\n",
    "        validate_ds=PatientFeaturesDataset(features[:1],df,patients_val[fold],max_len=val_max_len,\n",
    "                                           rand_split=False,rep=1,new_z=False,extra=extra_val)\n",
    "        train_ds=PatientFeaturesDataset(features,df,\n",
    "                                        list(set(all_patients).difference(patients_val[fold])),\n",
    "                                        max_len=train_max_len,rand_split=True,rep=1,new_z=False,fnoise=fnoise,extra=extra)\n",
    "        eq_series_val =len(patients_val[fold])/len(validate_ds) \n",
    "        eq_series_train =len( list(set(all_patients).difference(patients_val[fold])))/len(train_ds) \n",
    "        print(eq_series_val,eq_series_train )\n",
    "        epoch_size=len(train_ds)\n",
    "        model = get_transformer_model(dim_feedforward=ffdim,n_heads=num_heads,linear_embd=linear_embd,freeze=False,\n",
    "                                      n_encoders=num_layers,dropout=dropout,use_src_mask=False,res=False).to(device)\n",
    "        name=params.model_format.format(model_type,name_tamplate.format(SEED),model_version,fold)\n",
    "        my_loss=MyLoss(weights_series=WEIGHTS.to(device),eq_series=eq_series_train,\n",
    "                       weight_gen_img=0.1,conf_weight=0.,gamma=gamma,image_series=0.5,eth=0.,use_max=False,do_calc=False)\n",
    "        my_metric=MyMetric(eq_series=eq_series_val,image_series=0.5,use_max=False,do_calc=False)\n",
    "        learner = Learner(model,None,loss_func=my_loss,name=name,scheduler=None,device=device)\n",
    "        learner.metric=my_metric\n",
    "        learner.optimizer = torch.optim.Adam(learner.model.parameters(), lr=1e-4)\n",
    "\n",
    "        def new_get_y(self,batch):\n",
    "            return batch[-3],batch[-2]\n",
    "        def run_model(self,model,batch):\n",
    "            mask = batch[1]==-1\n",
    "            return model(*(x.to(self.device) for x in batch[:3+(extra is not None)]),mask=mask.to(device))\n",
    "        def calc_loss(self,y_pred,y_true):\n",
    "            return self.loss_func(y_pred,tuple(y.to(self.device) for y in y_true))\n",
    "        def on_epoch_begin(self,*args,**kargs):\n",
    "            train_ds.reset()\n",
    "        learner.get_y=MethodType(new_get_y, learner)\n",
    "        learner.run_model=MethodType(run_model, learner)\n",
    "        learner.calc_loss=MethodType(calc_loss, learner)\n",
    "        learner.on_epoch_begin=MethodType(on_epoch_begin, learner)\n",
    "        train_dl_args={'shuffle':True,'batch_size':batch_size}\n",
    "        for t in range(reps):\n",
    "            learner.scheduler = torch.optim.lr_scheduler.OneCycleLR(learner.optimizer, pct_start=0.01,final_div_factor= 10,\n",
    "                                                                    max_lr=reps_lr[t], steps_per_epoch=epoch_size//batch_size+1, \n",
    "                                                                    epochs=num_epochs//reps)\n",
    "\n",
    "            learner.fit(num_epochs//reps,train_ds,validate_ds,batch_size=batch_size,eval_batch=batch_size,path=params.path.models,\n",
    "                        train_dl_args=train_dl_args,num_workers=12,send_log=False)\n",
    "        sandesh.send({'name':learner.name,'best_metric':learner.best_metric})\n",
    "        learner.save_model(params.path.models)\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}