Spacy - NER- Individual Data Processing + Model Training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Download \n",
    "\n",
    "# pip install -U spacy\n",
    "\n",
    "# !python -m spacy download en"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training data structure for Sapcy\n",
    "```\n",
    "dummy_train_data = [\n",
    "    (\"Uber blew through $1 million a week\", [(0, 4, 'ORG')]),\n",
    "    (\"Android Pay expands to Canada\", [(0, 11, 'PRODUCT'), (23, 30, 'GPE')]),\n",
    "    (\"Spotify steps up Asia expansion\", [(0, 8, \"ORG\"), (17, 21, \"LOC\")]),\n",
    "    (\"Google Maps launches location sharing\", [(0, 11, \"PRODUCT\")]),\n",
    "    (\"Google rebrands its business apps\", [(0, 6, \"ORG\")]),\n",
    "    (\"look what i found on google! 😂\", [(21, 27, \"PRODUCT\")])]\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"/home/ubuntu/Grakn/Spacy Play/pubmed_result_abstract-annotated_individual.json\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(filename, encoding='utf-8') as train_data:\n",
    "# \ttrain = json.load(train_data)\n",
    "    \n",
    "data = []\n",
    "with open(filename) as f:\n",
    "    for line in f:\n",
    "        data.append(json.loads(line))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- data is a list containg dictionary of annotated text \n",
    "- It is a list of dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 2,\n",
       " 'text': 'The BRAF inhibitor dabrafenib (Tafinlar(Â®)) and the MEK inhibitor trametinib (Mekinist(Â®)) are indicated, as monotherapy or in combination with each other, for the treatment of patients with unresectable or metastatic melanoma with a BRAF (V600) mutation.',\n",
       " 'annotations': [{'label': 7, 'start_offset': 4, 'end_offset': 8, 'user': 1},\n",
       "  {'label': 8, 'start_offset': 19, 'end_offset': 29, 'user': 1},\n",
       "  {'label': 9, 'start_offset': 53, 'end_offset': 56, 'user': 1},\n",
       "  {'label': 8, 'start_offset': 67, 'end_offset': 77, 'user': 1},\n",
       "  {'label': 10, 'start_offset': 220, 'end_offset': 228, 'user': 1},\n",
       "  {'label': 7, 'start_offset': 236, 'end_offset': 240, 'user': 1},\n",
       "  {'label': 11, 'start_offset': 242, 'end_offset': 246, 'user': 1}],\n",
       " 'meta': {}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dictionary for label used while annotation\n",
    "\n",
    "\n",
    "label_dic = {'7':'gene','8':'drug','9':'protien','10':'diesease','11':'varient'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for k,v in label_dic.items():\n",
    "#     print(k,v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Iterating through all the dictionary of the list of dictionary of annotations\n",
    "\n",
    "for individual_dictionary in data:\n",
    "    for dic in individual_dictionary['annotations']:\n",
    "        # Going to find the label in the dictionary\n",
    "        temp_key = str(dic['label'])\n",
    "        #print(label_dic[temp_key])\n",
    "        dic['label'] = label_dic[temp_key]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': 2,\n",
       "  'text': 'The BRAF inhibitor dabrafenib (Tafinlar(Â®)) and the MEK inhibitor trametinib (Mekinist(Â®)) are indicated, as monotherapy or in combination with each other, for the treatment of patients with unresectable or metastatic melanoma with a BRAF (V600) mutation.',\n",
       "  'annotations': [{'label': 'gene',\n",
       "    'start_offset': 4,\n",
       "    'end_offset': 8,\n",
       "    'user': 1},\n",
       "   {'label': 'drug', 'start_offset': 19, 'end_offset': 29, 'user': 1},\n",
       "   {'label': 'protien', 'start_offset': 53, 'end_offset': 56, 'user': 1},\n",
       "   {'label': 'drug', 'start_offset': 67, 'end_offset': 77, 'user': 1},\n",
       "   {'label': 'diesease', 'start_offset': 220, 'end_offset': 228, 'user': 1},\n",
       "   {'label': 'gene', 'start_offset': 236, 'end_offset': 240, 'user': 1},\n",
       "   {'label': 'varient', 'start_offset': 242, 'end_offset': 246, 'user': 1}],\n",
       "  'meta': {}},\n",
       " {'id': 4,\n",
       "  'text': 'Dabrafenib plus trametinib significantly prolonged progression-free survival (PFS) and overall survival (OS), improved objective response rates (ORRs) and preserved health-related quality of life (HR-QOL) to a greater extent than dabrafenib (in the double-blind COMBI-d study) and vemurafenib (in the open-label COMBI-v study) in two large, randomized, phase III studies in treatment-naÃ¯ve patients with unresectable or metastatic melanoma with BRAF (V600E/K) mutation.',\n",
       "  'annotations': [{'label': 'drug',\n",
       "    'start_offset': 0,\n",
       "    'end_offset': 10,\n",
       "    'user': 1},\n",
       "   {'label': 'drug', 'start_offset': 16, 'end_offset': 26, 'user': 1},\n",
       "   {'label': 'drug', 'start_offset': 230, 'end_offset': 241, 'user': 1},\n",
       "   {'label': 'diesease', 'start_offset': 432, 'end_offset': 441, 'user': 1},\n",
       "   {'label': 'gene', 'start_offset': 446, 'end_offset': 450, 'user': 1},\n",
       "   {'label': 'varient', 'start_offset': 452, 'end_offset': 459, 'user': 1}],\n",
       "  'meta': {}},\n",
       " {'id': 5,\n",
       "  'text': 'Limited treatment benefit with the combination was also seen in patients who had progressed on prior BRAF inhibitor therapy, as indicated by ORRs of â‰¤ 15Â % and stable disease in â‰¤ 50Â % of patients in small phase I and II studies.Limited treatment benefit with the combination was also seen in patients who had progressed on prior BRAF inhibitor therapy, as indicated by ORRs of â‰¤ 15Â % and stable disease in â‰¤ 50Â % of patients in small phase I and II studies.',\n",
       "  'annotations': [{'label': 'gene',\n",
       "    'start_offset': 101,\n",
       "    'end_offset': 106,\n",
       "    'user': 1},\n",
       "   {'label': 'gene', 'start_offset': 336, 'end_offset': 341, 'user': 1}],\n",
       "  'meta': {}},\n",
       " {'id': 6,\n",
       "  'text': 'Combination therapy did not increase overall toxicity relative to dabrafenib or vemurafenib monotherapy, with most adverse events (AEs) mild or moderate in severity and generally manageable.',\n",
       "  'annotations': [{'label': 'drug',\n",
       "    'start_offset': 66,\n",
       "    'end_offset': 77,\n",
       "    'user': 1},\n",
       "   {'label': 'drug', 'start_offset': 80, 'end_offset': 92, 'user': 1}],\n",
       "  'meta': {}},\n",
       " {'id': 9,\n",
       "  'text': 'This article reviews the therapeutic efficacy and tolerability of combination treatment with dabrafenib and trametinib in this indication and summarizes relevant pharmacological data.',\n",
       "  'annotations': [{'label': 'drug',\n",
       "    'start_offset': 93,\n",
       "    'end_offset': 104,\n",
       "    'user': 1},\n",
       "   {'label': 'drug', 'start_offset': 108, 'end_offset': 119, 'user': 1}],\n",
       "  'meta': {}},\n",
       " {'id': 10,\n",
       "  'text': 'Fewer skin-related AEs (e.g. cutaneous malignancies, hyperkeratinosis and hand-foot syndrome) were reported with combination therapy than with dabrafenib or vemurafenib, probably because of reduced paradoxical activation of the MAPK pathway.',\n",
       "  'annotations': [{'label': 'drug',\n",
       "    'start_offset': 143,\n",
       "    'end_offset': 154,\n",
       "    'user': 1},\n",
       "   {'label': 'drug', 'start_offset': 157, 'end_offset': 168, 'user': 1}],\n",
       "  'meta': {}},\n",
       " {'id': 11,\n",
       "  'text': 'Thus, dabrafenib plus trametinib provides an important treatment option for patients with BRAF (V600) mutation-positive unresectable or metastatic melanoma.',\n",
       "  'annotations': [{'label': 'drug',\n",
       "    'start_offset': 6,\n",
       "    'end_offset': 17,\n",
       "    'user': 1},\n",
       "   {'label': 'drug', 'start_offset': 22, 'end_offset': 33, 'user': 1},\n",
       "   {'label': 'gene', 'start_offset': 90, 'end_offset': 95, 'user': 1},\n",
       "   {'label': 'varient', 'start_offset': 96, 'end_offset': 100, 'user': 1},\n",
       "   {'label': 'diesease', 'start_offset': 147, 'end_offset': 155, 'user': 1}],\n",
       "  'meta': {}}]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating data lsit as per Spacy format\n",
    "# Iterating through all the dictionary for transforming in spacy data\n",
    "\n",
    "\n",
    "temp_data= []\n",
    "for d in data:\n",
    "    #print(d[\"text\"]) # print text of all dictionary\n",
    "    #print(d.keys()) # print keys of dictionary\n",
    "    temp_annotation = d['annotations']\n",
    "    temp_list = [] # for storing tuples\n",
    "    temp_data_list = [] # for storing list of tuples\n",
    "    for label_dic in temp_annotation:\n",
    "        temp_list.append((label_dic['start_offset'],label_dic['end_offset'],label_dic['label']))\n",
    "        #emp_list.append()\n",
    "        #emp_list.append()\n",
    "        #t(label_dic['start_offset'],label_dic['end_offset'],label_dic['label'])\n",
    "    temp_data.append((d['text'],{\"entities\":temp_list}))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('The BRAF inhibitor dabrafenib (Tafinlar(Â®)) and the MEK inhibitor trametinib (Mekinist(Â®)) are indicated, as monotherapy or in combination with each other, for the treatment of patients with unresectable or metastatic melanoma with a BRAF (V600) mutation.',\n",
       "  {'entities': [(4, 8, 'gene'),\n",
       "    (19, 29, 'drug'),\n",
       "    (53, 56, 'protien'),\n",
       "    (67, 77, 'drug'),\n",
       "    (220, 228, 'diesease'),\n",
       "    (236, 240, 'gene'),\n",
       "    (242, 246, 'varient')]}),\n",
       " ('Dabrafenib plus trametinib significantly prolonged progression-free survival (PFS) and overall survival (OS), improved objective response rates (ORRs) and preserved health-related quality of life (HR-QOL) to a greater extent than dabrafenib (in the double-blind COMBI-d study) and vemurafenib (in the open-label COMBI-v study) in two large, randomized, phase III studies in treatment-naÃ¯ve patients with unresectable or metastatic melanoma with BRAF (V600E/K) mutation.',\n",
       "  {'entities': [(0, 10, 'drug'),\n",
       "    (16, 26, 'drug'),\n",
       "    (230, 241, 'drug'),\n",
       "    (432, 441, 'diesease'),\n",
       "    (446, 450, 'gene'),\n",
       "    (452, 459, 'varient')]}),\n",
       " ('Limited treatment benefit with the combination was also seen in patients who had progressed on prior BRAF inhibitor therapy, as indicated by ORRs of â‰¤ 15Â % and stable disease in â‰¤ 50Â % of patients in small phase I and II studies.Limited treatment benefit with the combination was also seen in patients who had progressed on prior BRAF inhibitor therapy, as indicated by ORRs of â‰¤ 15Â % and stable disease in â‰¤ 50Â % of patients in small phase I and II studies.',\n",
       "  {'entities': [(101, 106, 'gene'), (336, 341, 'gene')]}),\n",
       " ('Combination therapy did not increase overall toxicity relative to dabrafenib or vemurafenib monotherapy, with most adverse events (AEs) mild or moderate in severity and generally manageable.',\n",
       "  {'entities': [(66, 77, 'drug'), (80, 92, 'drug')]}),\n",
       " ('This article reviews the therapeutic efficacy and tolerability of combination treatment with dabrafenib and trametinib in this indication and summarizes relevant pharmacological data.',\n",
       "  {'entities': [(93, 104, 'drug'), (108, 119, 'drug')]}),\n",
       " ('Fewer skin-related AEs (e.g. cutaneous malignancies, hyperkeratinosis and hand-foot syndrome) were reported with combination therapy than with dabrafenib or vemurafenib, probably because of reduced paradoxical activation of the MAPK pathway.',\n",
       "  {'entities': [(143, 154, 'drug'), (157, 168, 'drug')]}),\n",
       " ('Thus, dabrafenib plus trametinib provides an important treatment option for patients with BRAF (V600) mutation-positive unresectable or metastatic melanoma.',\n",
       "  {'entities': [(6, 17, 'drug'),\n",
       "    (22, 33, 'drug'),\n",
       "    (90, 95, 'gene'),\n",
       "    (96, 100, 'varient'),\n",
       "    (147, 155, 'diesease')]})]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparing training data as per Spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_DATA = temp_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('The BRAF inhibitor dabrafenib (Tafinlar(Â®)) and the MEK inhibitor trametinib (Mekinist(Â®)) are indicated, as monotherapy or in combination with each other, for the treatment of patients with unresectable or metastatic melanoma with a BRAF (V600) mutation.',\n",
       "  {'entities': [(4, 8, 'gene'),\n",
       "    (19, 29, 'drug'),\n",
       "    (53, 56, 'protien'),\n",
       "    (67, 77, 'drug'),\n",
       "    (220, 228, 'diesease'),\n",
       "    (236, 240, 'gene'),\n",
       "    (242, 246, 'varient')]}),\n",
       " ('Dabrafenib plus trametinib significantly prolonged progression-free survival (PFS) and overall survival (OS), improved objective response rates (ORRs) and preserved health-related quality of life (HR-QOL) to a greater extent than dabrafenib (in the double-blind COMBI-d study) and vemurafenib (in the open-label COMBI-v study) in two large, randomized, phase III studies in treatment-naÃ¯ve patients with unresectable or metastatic melanoma with BRAF (V600E/K) mutation.',\n",
       "  {'entities': [(0, 10, 'drug'),\n",
       "    (16, 26, 'drug'),\n",
       "    (230, 241, 'drug'),\n",
       "    (432, 441, 'diesease'),\n",
       "    (446, 450, 'gene'),\n",
       "    (452, 459, 'varient')]}),\n",
       " ('Limited treatment benefit with the combination was also seen in patients who had progressed on prior BRAF inhibitor therapy, as indicated by ORRs of â‰¤ 15Â % and stable disease in â‰¤ 50Â % of patients in small phase I and II studies.Limited treatment benefit with the combination was also seen in patients who had progressed on prior BRAF inhibitor therapy, as indicated by ORRs of â‰¤ 15Â % and stable disease in â‰¤ 50Â % of patients in small phase I and II studies.',\n",
       "  {'entities': [(101, 106, 'gene'), (336, 341, 'gene')]}),\n",
       " ('Combination therapy did not increase overall toxicity relative to dabrafenib or vemurafenib monotherapy, with most adverse events (AEs) mild or moderate in severity and generally manageable.',\n",
       "  {'entities': [(66, 77, 'drug'), (80, 92, 'drug')]}),\n",
       " ('This article reviews the therapeutic efficacy and tolerability of combination treatment with dabrafenib and trametinib in this indication and summarizes relevant pharmacological data.',\n",
       "  {'entities': [(93, 104, 'drug'), (108, 119, 'drug')]}),\n",
       " ('Fewer skin-related AEs (e.g. cutaneous malignancies, hyperkeratinosis and hand-foot syndrome) were reported with combination therapy than with dabrafenib or vemurafenib, probably because of reduced paradoxical activation of the MAPK pathway.',\n",
       "  {'entities': [(143, 154, 'drug'), (157, 168, 'drug')]}),\n",
       " ('Thus, dabrafenib plus trametinib provides an important treatment option for patients with BRAF (V600) mutation-positive unresectable or metastatic melanoma.',\n",
       "  {'entities': [(6, 17, 'drug'),\n",
       "    (22, 33, 'drug'),\n",
       "    (90, 95, 'gene'),\n",
       "    (96, 100, 'varient'),\n",
       "    (147, 155, 'diesease')]})]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TRAIN_DATA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  -- Training Model --"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import unicode_literals, print_function\n",
    "\n",
    "import plac\n",
    "import random\n",
    "from pathlib import Path\n",
    "import spacy\n",
    "from spacy.util import minibatch, compounding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_model_custom = \"/home/ubuntu/Grakn/Spacy Play/spacy_en_core_web_lg\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/ubuntu/Grakn/Spacy Play/spacy_en_core_web_lg'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_model_custom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# @plac.annotations(\n",
    "# model=(\"en\", \"option\", \"m\", str),\n",
    "# output_dir=(\"Optional output directory\", \"option\", \"o\", Path),\n",
    "# n_iter=(\"5\", \"option\", \"n\", int)\n",
    "#     )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# @plac.annotations(\n",
    "#     model=(\"Model name. Defaults to blank 'en' model.\", \"option\", \"m\", str),\n",
    "#     new_model_name=(\"New model name for model meta.\", \"option\", \"nm\", str),\n",
    "#     output_dir=(\"Optional output directory\", \"option\", \"o\", Path),\n",
    "#     n_iter=(\"Number of training iterations\", \"option\", \"n\", int),)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main(model=None, output_dir=None, n_iter=1000):\n",
    "    \"\"\"Load the model, set up the pipeline and train the entity recognizer.\"\"\"\n",
    "    if model is not None:\n",
    "        nlp = spacy.load(model)  # load existing spaCy model\n",
    "        print(\"Loaded model '%s'\" % model)\n",
    "    else:\n",
    "        nlp = spacy.blank(\"en\")  # create blank Language class\n",
    "        print(\"Created blank 'en' model\")\n",
    "\n",
    "    # create the built-in pipeline components and add them to the pipeline\n",
    "    # nlp.create_pipe works for built-ins that are registered with spaCy\n",
    "    if \"ner\" not in nlp.pipe_names:\n",
    "        ner = nlp.create_pipe(\"ner\")\n",
    "        nlp.add_pipe(ner, last=True)\n",
    "    # otherwise, get it so we can add labels\n",
    "    else:\n",
    "        ner = nlp.get_pipe(\"ner\")\n",
    "\n",
    "    # add labels\n",
    "    for _, annotations in TRAIN_DATA:\n",
    "        for ent in annotations.get(\"entities\"):\n",
    "            ner.add_label(ent[2])\n",
    "\n",
    "    # get names of other pipes to disable them during training\n",
    "    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != \"ner\"]\n",
    "    with nlp.disable_pipes(*other_pipes):  # only train NER\n",
    "        # reset and initialize the weights randomly – but only if we're\n",
    "        # training a new model\n",
    "        if model is None:\n",
    "            nlp.begin_training()\n",
    "        for itn in range(n_iter):\n",
    "            random.shuffle(TRAIN_DATA)\n",
    "            losses = {}\n",
    "            # batch up the examples using spaCy's minibatch\n",
    "            batches = minibatch(TRAIN_DATA, size=compounding(4.0, 32.0, 1.001))\n",
    "            for batch in batches:\n",
    "                texts, annotations = zip(*batch)\n",
    "                nlp.update(\n",
    "                    texts,  # batch of texts\n",
    "                    annotations,  # batch of annotations\n",
    "                    drop=0.5,  # dropout - make it harder to memorise data\n",
    "                    losses=losses,\n",
    "                )\n",
    "            print(\"Losses\", losses)\n",
    "\n",
    "    # test the trained model\n",
    "    for text, _ in TRAIN_DATA:\n",
    "        doc = nlp(text)\n",
    "        print(\"Entities\", [(ent.text, ent.label_) for ent in doc.ents])\n",
    "        print(\"Tokens\", [(t.text, t.ent_type_, t.ent_iob) for t in doc])\n",
    "\n",
    "    # save model to output directory\n",
    "    if output_dir is not None:\n",
    "        output_dir = Path(output_dir)\n",
    "        if not output_dir.exists():\n",
    "            output_dir.mkdir()\n",
    "        nlp.to_disk(output_dir)\n",
    "        print(\"Saved model to\", output_dir)\n",
    "\n",
    "        # test the saved model\n",
    "        print(\"Loading from\", output_dir)\n",
    "        nlp2 = spacy.load(output_dir)\n",
    "        for text, _ in TRAIN_DATA:\n",
    "            doc = nlp2(text)\n",
    "            print(\"Entities\", [(ent.text, ent.label_) for ent in doc.ents])\n",
    "            print(\"Tokens\", [(t.text, t.ent_type_, t.ent_iob) for t in doc])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded model 'en_core_web_lg'\n",
      "Losses {'ner': 351.10849845409393}\n",
      "Losses {'ner': 388.565584897995}\n",
      "Losses {'ner': 347.4417562484741}\n",
      "Losses {'ner': 331.8790317773819}\n",
      "Losses {'ner': 352.67305040359497}\n",
      "Losses {'ner': 320.6880597472191}\n",
      "Losses {'ner': 345.4755458831787}\n",
      "Losses {'ner': 346.2539955377579}\n",
      "Losses {'ner': 317.91983294487}\n",
      "Losses {'ner': 330.30823731422424}\n",
      "Losses {'ner': 299.71725249290466}\n",
      "Losses {'ner': 284.6068466901779}\n",
      "Losses {'ner': 298.05564427375793}\n",
      "Losses {'ner': 296.43757581710815}\n",
      "Losses {'ner': 300.38855504989624}\n",
      "Losses {'ner': 307.24784541130066}\n",
      "Losses {'ner': 286.82343339920044}\n",
      "Losses {'ner': 274.07502818107605}\n",
      "Losses {'ner': 260.9445958137512}\n",
      "Losses {'ner': 297.78058433532715}\n",
      "Losses {'ner': 275.33467388153076}\n",
      "Losses {'ner': 284.6750916838646}\n",
      "Losses {'ner': 281.7085689306259}\n",
      "Losses {'ner': 272.0712777376175}\n",
      "Losses {'ner': 272.44746148586273}\n",
      "Losses {'ner': 260.43229842185974}\n",
      "Losses {'ner': 267.4916616678238}\n",
      "Losses {'ner': 266.34694027900696}\n",
      "Losses {'ner': 301.0447463989258}\n",
      "Losses {'ner': 266.41826152801514}\n",
      "Losses {'ner': 286.664466381073}\n",
      "Losses {'ner': 272.54417872428894}\n",
      "Losses {'ner': 288.84376096725464}\n",
      "Losses {'ner': 259.39520287513733}\n",
      "Losses {'ner': 272.7353265285492}\n",
      "Losses {'ner': 289.3734040260315}\n",
      "Losses {'ner': 255.3381543159485}\n",
      "Losses {'ner': 269.13178730010986}\n",
      "Losses {'ner': 260.596192240715}\n",
      "Losses {'ner': 273.10447669029236}\n",
      "Losses {'ner': 268.2996599674225}\n",
      "Losses {'ner': 261.63798332214355}\n",
      "Losses {'ner': 267.07644391059875}\n",
      "Losses {'ner': 270.57161045074463}\n",
      "Losses {'ner': 271.25006890296936}\n",
      "Losses {'ner': 266.643146276474}\n",
      "Losses {'ner': 291.09550499916077}\n",
      "Losses {'ner': 265.040679872036}\n",
      "Losses {'ner': 262.79738116264343}\n",
      "Losses {'ner': 259.7715172767639}\n",
      "Losses {'ner': 252.8828227519989}\n",
      "Losses {'ner': 258.5402410030365}\n",
      "Losses {'ner': 249.9797396659851}\n",
      "Losses {'ner': 267.3889787197113}\n",
      "Losses {'ner': 262.82293176651}\n",
      "Losses {'ner': 250.860098361969}\n",
      "Losses {'ner': 241.6019824743271}\n",
      "Losses {'ner': 263.22106862068176}\n",
      "Losses {'ner': 245.0732444524765}\n",
      "Losses {'ner': 239.95013737678528}\n",
      "Losses {'ner': 268.01032972335815}\n",
      "Losses {'ner': 242.43794703483582}\n",
      "Losses {'ner': 257.08318424224854}\n",
      "Losses {'ner': 243.6196712255478}\n",
      "Losses {'ner': 248.27758407592773}\n",
      "Losses {'ner': 253.51355636119843}\n",
      "Losses {'ner': 260.2730462551117}\n",
      "Losses {'ner': 252.49681890010834}\n",
      "Losses {'ner': 258.9052759408951}\n",
      "Losses {'ner': 256.2167854309082}\n",
      "Losses {'ner': 248.04207491874695}\n",
      "Losses {'ner': 252.22408258914948}\n",
      "Losses {'ner': 246.05276131629944}\n",
      "Losses {'ner': 254.23356342315674}\n",
      "Losses {'ner': 253.41880333423615}\n",
      "Losses {'ner': 247.95990109443665}\n",
      "Losses {'ner': 243.44134402275085}\n",
      "Losses {'ner': 251.78856468200684}\n",
      "Losses {'ner': 264.80216643214226}\n",
      "Losses {'ner': 251.05107271671295}\n",
      "Losses {'ner': 244.2574384212494}\n",
      "Losses {'ner': 253.9089698791504}\n",
      "Losses {'ner': 260.78633081912994}\n",
      "Losses {'ner': 242.03559637069702}\n",
      "Losses {'ner': 253.5729856491089}\n",
      "Losses {'ner': 249.27516198158264}\n",
      "Losses {'ner': 256.4872636795044}\n",
      "Losses {'ner': 259.0110731124878}\n",
      "Losses {'ner': 250.96792662143707}\n",
      "Losses {'ner': 259.86155247688293}\n",
      "Losses {'ner': 258.902321100235}\n",
      "Losses {'ner': 244.30454754829407}\n",
      "Losses {'ner': 240.68974089622498}\n",
      "Losses {'ner': 242.9562051296234}\n",
      "Losses {'ner': 246.94941800832748}\n",
      "Losses {'ner': 251.24092841148376}\n",
      "Losses {'ner': 250.98434615135193}\n",
      "Losses {'ner': 250.3318145275116}\n",
      "Losses {'ner': 257.80034482479095}\n",
      "Losses {'ner': 252.2353571653366}\n",
      "Losses {'ner': 243.62421041727066}\n",
      "Losses {'ner': 236.03328132629395}\n",
      "Losses {'ner': 252.35795843601227}\n",
      "Losses {'ner': 247.05872237682343}\n",
      "Losses {'ner': 251.6349766254425}\n",
      "Losses {'ner': 244.66817665100098}\n",
      "Losses {'ner': 250.7888435125351}\n",
      "Losses {'ner': 231.18045365810394}\n",
      "Losses {'ner': 244.45131301879883}\n",
      "Losses {'ner': 247.90811705589294}\n",
      "Losses {'ner': 229.09687852859497}\n",
      "Losses {'ner': 244.35624623298645}\n",
      "Losses {'ner': 262.6579910516739}\n",
      "Losses {'ner': 251.08382487297058}\n",
      "Losses {'ner': 251.11936116218567}\n",
      "Losses {'ner': 239.00441813468933}\n",
      "Losses {'ner': 240.92825531959534}\n",
      "Losses {'ner': 256.7052392959595}\n",
      "Losses {'ner': 226.4341198205948}\n",
      "Losses {'ner': 252.78174448013306}\n",
      "Losses {'ner': 241.5910371541977}\n",
      "Losses {'ner': 240.26030945777893}\n",
      "Losses {'ner': 250.19630867242813}\n",
      "Losses {'ner': 237.45089602470398}\n",
      "Losses {'ner': 254.7498803138733}\n",
      "Losses {'ner': 244.13677334785461}\n",
      "Losses {'ner': 252.1931664943695}\n",
      "Losses {'ner': 249.91896557807922}\n",
      "Losses {'ner': 243.14377903938293}\n",
      "Losses {'ner': 246.39439070224762}\n",
      "Losses {'ner': 237.93176865577698}\n",
      "Losses {'ner': 240.56062245368958}\n",
      "Losses {'ner': 236.70585894584656}\n",
      "Losses {'ner': 238.049094080925}\n",
      "Losses {'ner': 259.788689494133}\n",
      "Losses {'ner': 233.80747628211975}\n",
      "Losses {'ner': 242.98405689001083}\n",
      "Losses {'ner': 248.36528325080872}\n",
      "Losses {'ner': 248.30304503440857}\n",
      "Losses {'ner': 244.1240187883377}\n",
      "Losses {'ner': 250.7458953857422}\n",
      "Losses {'ner': 242.20729160308838}\n",
      "Losses {'ner': 249.63269352912903}\n",
      "Losses {'ner': 250.17612886428833}\n",
      "Losses {'ner': 243.04559004306793}\n",
      "Losses {'ner': 229.48291024565697}\n",
      "Losses {'ner': 241.0949625968933}\n",
      "Losses {'ner': 233.02057480812073}\n",
      "Losses {'ner': 233.82934057712555}\n",
      "Losses {'ner': 225.80043411254883}\n",
      "Losses {'ner': 237.31512594223022}\n",
      "Losses {'ner': 230.65840196609497}\n",
      "Losses {'ner': 244.7888150215149}\n",
      "Losses {'ner': 246.54975032806396}\n",
      "Losses {'ner': 254.46129047870636}\n",
      "Losses {'ner': 235.91242861747742}\n",
      "Losses {'ner': 234.84024167060852}\n",
      "Losses {'ner': 241.46039295196533}\n",
      "Losses {'ner': 242.15761375427246}\n",
      "Losses {'ner': 240.94945192337036}\n",
      "Losses {'ner': 253.25329566001892}\n",
      "Losses {'ner': 238.33684188127518}\n",
      "Losses {'ner': 245.03205633163452}\n",
      "Losses {'ner': 238.1612800359726}\n",
      "Losses {'ner': 250.8303918838501}\n",
      "Losses {'ner': 248.11632251739502}\n",
      "Losses {'ner': 229.98072504997253}\n",
      "Losses {'ner': 221.68017053604126}\n",
      "Losses {'ner': 235.5959117412567}\n",
      "Losses {'ner': 244.00410318374634}\n",
      "Losses {'ner': 230.950585603714}\n",
      "Losses {'ner': 247.8761613368988}\n",
      "Losses {'ner': 233.05747365951538}\n",
      "Losses {'ner': 240.85335683822632}\n",
      "Losses {'ner': 247.3069224357605}\n",
      "Losses {'ner': 241.67041969299316}\n",
      "Losses {'ner': 237.52977967262268}\n",
      "Losses {'ner': 247.29099082946777}\n",
      "Losses {'ner': 243.43687999248505}\n",
      "Losses {'ner': 239.35908031463623}\n",
      "Losses {'ner': 234.52209496498108}\n",
      "Losses {'ner': 231.49410045146942}\n",
      "Losses {'ner': 239.50333654880524}\n",
      "Losses {'ner': 232.3284431695938}\n",
      "Losses {'ner': 239.14097833633423}\n",
      "Losses {'ner': 241.65515226125717}\n",
      "Losses {'ner': 233.4107882976532}\n",
      "Losses {'ner': 240.85810565948486}\n",
      "Losses {'ner': 243.59119033813477}\n",
      "Losses {'ner': 245.3553370833397}\n",
      "Losses {'ner': 240.38884449005127}\n",
      "Losses {'ner': 246.35944628715515}\n",
      "Losses {'ner': 240.72322392463684}\n",
      "Losses {'ner': 232.57397508621216}\n",
      "Losses {'ner': 236.9492472410202}\n",
      "Losses {'ner': 240.7534475326538}\n",
      "Losses {'ner': 244.46789121627808}\n",
      "Losses {'ner': 239.98352003097534}\n",
      "Losses {'ner': 242.3748073577881}\n",
      "Losses {'ner': 236.77786791324615}\n",
      "Losses {'ner': 240.89625573158264}\n",
      "Losses {'ner': 235.41416317224503}\n",
      "Losses {'ner': 242.3837662935257}\n",
      "Losses {'ner': 247.19479632377625}\n",
      "Losses {'ner': 241.96106386184692}\n",
      "Losses {'ner': 226.8866810798645}\n",
      "Losses {'ner': 241.53404808044434}\n",
      "Losses {'ner': 242.9791224002838}\n",
      "Losses {'ner': 244.04319429397583}\n",
      "Losses {'ner': 232.57610154151917}\n",
      "Losses {'ner': 232.78224289417267}\n",
      "Losses {'ner': 241.46750390529633}\n",
      "Losses {'ner': 250.3661060333252}\n",
      "Losses {'ner': 240.72015273571014}\n",
      "Losses {'ner': 236.82265067100525}\n",
      "Losses {'ner': 217.07106828689575}\n",
      "Losses {'ner': 230.2342039346695}\n",
      "Losses {'ner': 242.81835794448853}\n",
      "Losses {'ner': 233.6255248785019}\n",
      "Losses {'ner': 241.9578139781952}\n",
      "Losses {'ner': 231.26910972595215}\n",
      "Losses {'ner': 227.05696040391922}\n",
      "Losses {'ner': 228.66403436660767}\n",
      "Losses {'ner': 241.57196414470673}\n",
      "Losses {'ner': 237.31072568893433}\n",
      "Losses {'ner': 237.27237284183502}\n",
      "Losses {'ner': 245.4099566936493}\n",
      "Losses {'ner': 247.4124300479889}\n",
      "Losses {'ner': 236.29479885101318}\n",
      "Losses {'ner': 238.19594383239746}\n",
      "Losses {'ner': 232.86639094352722}\n",
      "Losses {'ner': 228.9029312133789}\n",
      "Losses {'ner': 247.06744462251663}\n",
      "Losses {'ner': 245.5352246761322}\n",
      "Losses {'ner': 236.39042735099792}\n",
      "Losses {'ner': 246.87600684165955}\n",
      "Losses {'ner': 251.09105467796326}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Losses {'ner': 233.6366627216339}\n",
      "Losses {'ner': 235.8771733045578}\n",
      "Losses {'ner': 221.96025329828262}\n",
      "Losses {'ner': 227.77365100383759}\n",
      "Losses {'ner': 230.28813433647156}\n",
      "Losses {'ner': 224.94705414772034}\n",
      "Losses {'ner': 256.2351031303406}\n",
      "Losses {'ner': 240.87209904193878}\n",
      "Losses {'ner': 254.76977908611298}\n",
      "Losses {'ner': 246.39050960540771}\n",
      "Losses {'ner': 243.85203969478607}\n",
      "Losses {'ner': 234.20369482040405}\n",
      "Losses {'ner': 230.91706705093384}\n",
      "Losses {'ner': 230.25865185260773}\n",
      "Losses {'ner': 240.6725445985794}\n",
      "Losses {'ner': 224.6598494052887}\n",
      "Losses {'ner': 231.51966953277588}\n",
      "Losses {'ner': 246.90274953842163}\n",
      "Losses {'ner': 232.99806886911392}\n",
      "Losses {'ner': 231.121839761734}\n",
      "Losses {'ner': 234.4996476173401}\n",
      "Losses {'ner': 230.86529922485352}\n",
      "Losses {'ner': 239.97690451145172}\n",
      "Losses {'ner': 231.09726428985596}\n",
      "Losses {'ner': 234.0229851603508}\n",
      "Losses {'ner': 242.33097195625305}\n",
      "Losses {'ner': 239.91440272331238}\n",
      "Losses {'ner': 240.11331939697266}\n",
      "Losses {'ner': 244.03682577610016}\n",
      "Losses {'ner': 241.21195769309998}\n",
      "Losses {'ner': 232.4873025417328}\n",
      "Losses {'ner': 238.53507113456726}\n",
      "Losses {'ner': 226.0170982480049}\n",
      "Losses {'ner': 224.1049718260765}\n",
      "Losses {'ner': 219.3982012271881}\n",
      "Losses {'ner': 227.8427391052246}\n",
      "Losses {'ner': 236.35972547531128}\n",
      "Losses {'ner': 226.52055096626282}\n",
      "Losses {'ner': 239.67364692687988}\n",
      "Losses {'ner': 234.48469507694244}\n",
      "Losses {'ner': 233.9701339006424}\n",
      "Losses {'ner': 233.29974365234375}\n",
      "Losses {'ner': 243.35298788547516}\n",
      "Losses {'ner': 237.4665197134018}\n",
      "Losses {'ner': 238.834627866745}\n",
      "Losses {'ner': 231.95517754554749}\n",
      "Losses {'ner': 236.37009239196777}\n",
      "Losses {'ner': 251.84419465065002}\n",
      "Losses {'ner': 229.45359659194946}\n",
      "Losses {'ner': 233.55554962158203}\n",
      "Losses {'ner': 240.14527714252472}\n",
      "Losses {'ner': 224.9765067100525}\n",
      "Losses {'ner': 233.9547589868307}\n",
      "Losses {'ner': 247.68487334251404}\n",
      "Losses {'ner': 243.76189160346985}\n",
      "Losses {'ner': 224.14297938346863}\n",
      "Losses {'ner': 230.1812343597412}\n",
      "Losses {'ner': 236.9798126220703}\n",
      "Losses {'ner': 232.50518095493317}\n",
      "Losses {'ner': 234.1200371980667}\n",
      "Losses {'ner': 216.03526306152344}\n",
      "Losses {'ner': 231.75966180860996}\n",
      "Losses {'ner': 244.5027916431427}\n",
      "Losses {'ner': 240.9880690574646}\n",
      "Losses {'ner': 222.9988068342209}\n",
      "Losses {'ner': 236.31207919120789}\n",
      "Losses {'ner': 235.84850239753723}\n",
      "Losses {'ner': 244.05627846717834}\n",
      "Losses {'ner': 251.15494298934937}\n",
      "Losses {'ner': 233.9399664402008}\n",
      "Losses {'ner': 239.610675573349}\n",
      "Losses {'ner': 228.4536918401718}\n",
      "Losses {'ner': 238.59856033325195}\n",
      "Losses {'ner': 239.53534626960754}\n",
      "Losses {'ner': 223.936105966568}\n",
      "Losses {'ner': 236.24115753173828}\n",
      "Losses {'ner': 227.4807825088501}\n",
      "Losses {'ner': 241.3072350025177}\n",
      "Losses {'ner': 233.21117520332336}\n",
      "Losses {'ner': 248.31842064857483}\n",
      "Losses {'ner': 243.95403242111206}\n",
      "Losses {'ner': 236.11946594715118}\n",
      "Losses {'ner': 250.62886881828308}\n",
      "Losses {'ner': 237.78873616456985}\n",
      "Losses {'ner': 233.22273802757263}\n",
      "Losses {'ner': 228.69819927215576}\n",
      "Losses {'ner': 237.09377336502075}\n",
      "Losses {'ner': 237.33975517749786}\n",
      "Losses {'ner': 233.8224309682846}\n",
      "Losses {'ner': 230.3493001461029}\n",
      "Losses {'ner': 229.8147071003914}\n",
      "Losses {'ner': 254.62864768505096}\n",
      "Losses {'ner': 241.56528639793396}\n",
      "Losses {'ner': 220.25677287578583}\n",
      "Losses {'ner': 226.246741771698}\n",
      "Losses {'ner': 226.1831818819046}\n",
      "Losses {'ner': 235.3731918334961}\n",
      "Losses {'ner': 216.71386981010437}\n",
      "Losses {'ner': 249.60384058952332}\n",
      "Losses {'ner': 236.88661789894104}\n",
      "Losses {'ner': 247.50324189662933}\n",
      "Losses {'ner': 231.90006923675537}\n",
      "Losses {'ner': 239.6462733745575}\n",
      "Losses {'ner': 230.12218856811523}\n",
      "Losses {'ner': 245.54376077651978}\n",
      "Losses {'ner': 226.0217044353485}\n",
      "Losses {'ner': 231.9499695301056}\n",
      "Losses {'ner': 247.01179957389832}\n",
      "Losses {'ner': 228.28574228286743}\n",
      "Losses {'ner': 220.6819007396698}\n",
      "Losses {'ner': 235.91805934906006}\n",
      "Losses {'ner': 244.04237604141235}\n",
      "Losses {'ner': 225.50314044952393}\n",
      "Losses {'ner': 226.98916172981262}\n",
      "Losses {'ner': 241.94583189487457}\n",
      "Losses {'ner': 232.36570417881012}\n",
      "Losses {'ner': 239.91035401821136}\n",
      "Losses {'ner': 230.77001953125}\n",
      "Losses {'ner': 253.84757924079895}\n",
      "Losses {'ner': 234.78328597545624}\n",
      "Losses {'ner': 236.35221815109253}\n",
      "Losses {'ner': 239.32206672430038}\n",
      "Losses {'ner': 235.32421255111694}\n",
      "Losses {'ner': 235.34339880943298}\n",
      "Losses {'ner': 228.27819991111755}\n",
      "Losses {'ner': 236.9393446445465}\n",
      "Losses {'ner': 238.64708185195923}\n",
      "Losses {'ner': 232.57940113544464}\n",
      "Losses {'ner': 227.71037888526917}\n",
      "Losses {'ner': 240.5723011493683}\n",
      "Losses {'ner': 234.56759977340698}\n",
      "Losses {'ner': 226.37988138198853}\n",
      "Losses {'ner': 222.89900398254395}\n",
      "Losses {'ner': 237.60993933677673}\n",
      "Losses {'ner': 241.70588731765747}\n",
      "Losses {'ner': 245.76040744781494}\n",
      "Losses {'ner': 228.54488170146942}\n",
      "Losses {'ner': 228.3243373632431}\n",
      "Losses {'ner': 220.84474778175354}\n",
      "Losses {'ner': 244.08528804779053}\n",
      "Losses {'ner': 240.1474391222}\n",
      "Losses {'ner': 221.11735606193542}\n",
      "Losses {'ner': 234.44419634342194}\n",
      "Losses {'ner': 239.7900459766388}\n",
      "Losses {'ner': 237.33865690231323}\n",
      "Losses {'ner': 234.74978256225586}\n",
      "Losses {'ner': 229.35316395759583}\n",
      "Losses {'ner': 227.92600905895233}\n",
      "Losses {'ner': 238.16227102279663}\n",
      "Losses {'ner': 229.7946012020111}\n",
      "Losses {'ner': 231.68246495723724}\n",
      "Losses {'ner': 231.81660848855972}\n",
      "Losses {'ner': 225.300430893898}\n",
      "Losses {'ner': 238.3484046459198}\n",
      "Losses {'ner': 224.24907517433167}\n",
      "Losses {'ner': 230.6500644683838}\n",
      "Losses {'ner': 233.42420530319214}\n",
      "Losses {'ner': 235.55183684825897}\n",
      "Losses {'ner': 235.93273282051086}\n",
      "Losses {'ner': 238.24618291854858}\n",
      "Losses {'ner': 228.43344068527222}\n",
      "Losses {'ner': 222.3228656053543}\n",
      "Losses {'ner': 232.0071771144867}\n",
      "Losses {'ner': 235.47277069091797}\n",
      "Losses {'ner': 225.63306510448456}\n",
      "Losses {'ner': 237.53270936012268}\n",
      "Losses {'ner': 237.6215214729309}\n",
      "Losses {'ner': 234.35657501220703}\n",
      "Losses {'ner': 231.4015622138977}\n",
      "Losses {'ner': 227.35610628128052}\n",
      "Losses {'ner': 218.95799565315247}\n",
      "Losses {'ner': 238.55650305747986}\n",
      "Losses {'ner': 236.85231113433838}\n",
      "Losses {'ner': 230.98352766036987}\n",
      "Losses {'ner': 235.45287036895752}\n",
      "Losses {'ner': 218.37250208854675}\n",
      "Losses {'ner': 237.86408853530884}\n",
      "Losses {'ner': 237.8549473285675}\n",
      "Losses {'ner': 233.66789257526398}\n",
      "Losses {'ner': 237.5775966644287}\n",
      "Losses {'ner': 238.60709083080292}\n",
      "Losses {'ner': 242.5357825756073}\n",
      "Losses {'ner': 231.6301145553589}\n",
      "Losses {'ner': 226.95177745819092}\n",
      "Losses {'ner': 225.68374061584473}\n",
      "Losses {'ner': 237.6118335723877}\n",
      "Losses {'ner': 243.0496146082878}\n",
      "Losses {'ner': 224.0696929693222}\n",
      "Losses {'ner': 247.89892446994781}\n",
      "Losses {'ner': 230.5277419090271}\n",
      "Losses {'ner': 228.1725155711174}\n",
      "Losses {'ner': 235.41434240341187}\n",
      "Losses {'ner': 233.82428884506226}\n",
      "Losses {'ner': 233.06393837928772}\n",
      "Losses {'ner': 224.9031046628952}\n",
      "Losses {'ner': 221.02138304710388}\n",
      "Losses {'ner': 232.52153980731964}\n",
      "Losses {'ner': 228.4177747964859}\n",
      "Losses {'ner': 236.2888526916504}\n",
      "Losses {'ner': 223.74837911128998}\n",
      "Losses {'ner': 222.1071457862854}\n",
      "Losses {'ner': 222.69920992851257}\n",
      "Losses {'ner': 241.59051881730556}\n",
      "Losses {'ner': 214.85926949977875}\n",
      "Losses {'ner': 226.05049538612366}\n",
      "Losses {'ner': 234.83548855781555}\n",
      "Losses {'ner': 229.40283739566803}\n",
      "Losses {'ner': 229.6191577911377}\n",
      "Losses {'ner': 242.01049423217773}\n",
      "Losses {'ner': 215.6742568090558}\n",
      "Losses {'ner': 235.73826432228088}\n",
      "Losses {'ner': 237.49150079488754}\n",
      "Losses {'ner': 230.48982548713684}\n",
      "Losses {'ner': 252.83140790462494}\n",
      "Losses {'ner': 232.23471009731293}\n",
      "Losses {'ner': 225.3821575641632}\n",
      "Losses {'ner': 233.16683411598206}\n",
      "Losses {'ner': 235.1311366558075}\n",
      "Losses {'ner': 235.23406565189362}\n",
      "Losses {'ner': 220.26603746414185}\n",
      "Losses {'ner': 232.5239770412445}\n",
      "Losses {'ner': 231.51955366134644}\n",
      "Losses {'ner': 233.92973911762238}\n",
      "Losses {'ner': 222.23240447044373}\n",
      "Losses {'ner': 234.75236028432846}\n",
      "Losses {'ner': 235.03328347206116}\n",
      "Losses {'ner': 244.24441742897034}\n",
      "Losses {'ner': 230.94468069076538}\n",
      "Losses {'ner': 227.66441702842712}\n",
      "Losses {'ner': 223.17853391170502}\n",
      "Losses {'ner': 238.4419800043106}\n",
      "Losses {'ner': 223.55610513687134}\n",
      "Losses {'ner': 232.8647530078888}\n",
      "Losses {'ner': 230.54333984851837}\n",
      "Losses {'ner': 238.4959671497345}\n",
      "Losses {'ner': 230.9300286769867}\n",
      "Losses {'ner': 238.188232421875}\n",
      "Losses {'ner': 232.59092843532562}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Losses {'ner': 217.31947553157806}\n",
      "Losses {'ner': 226.26286268234253}\n",
      "Losses {'ner': 239.99363887310028}\n",
      "Losses {'ner': 235.0160572528839}\n",
      "Losses {'ner': 233.71570873260498}\n",
      "Losses {'ner': 219.91142201423645}\n",
      "Losses {'ner': 233.9765277504921}\n",
      "Losses {'ner': 239.34053707122803}\n",
      "Losses {'ner': 220.54632222652435}\n",
      "Losses {'ner': 232.84493350982666}\n",
      "Losses {'ner': 241.22601437568665}\n",
      "Losses {'ner': 239.0030117034912}\n",
      "Losses {'ner': 229.91782879829407}\n",
      "Losses {'ner': 229.58780145645142}\n",
      "Losses {'ner': 227.85076141357422}\n",
      "Losses {'ner': 250.0526304244995}\n",
      "Losses {'ner': 231.4900209903717}\n",
      "Losses {'ner': 226.82943320274353}\n",
      "Losses {'ner': 228.06738567352295}\n",
      "Losses {'ner': 226.17665362358093}\n",
      "Losses {'ner': 246.86148929595947}\n",
      "Losses {'ner': 224.99926924705505}\n",
      "Losses {'ner': 229.31083691120148}\n",
      "Losses {'ner': 232.47655057907104}\n",
      "Losses {'ner': 242.5118328332901}\n",
      "Losses {'ner': 215.92603611946106}\n",
      "Losses {'ner': 225.43117141723633}\n",
      "Losses {'ner': 225.2126805782318}\n",
      "Losses {'ner': 231.40691661834717}\n",
      "Losses {'ner': 240.40659356117249}\n",
      "Losses {'ner': 227.1015031337738}\n",
      "Losses {'ner': 213.54374241828918}\n",
      "Losses {'ner': 241.2693793773651}\n",
      "Losses {'ner': 219.48833191394806}\n",
      "Losses {'ner': 234.48637008666992}\n",
      "Losses {'ner': 231.2047734260559}\n",
      "Losses {'ner': 221.18342900276184}\n",
      "Losses {'ner': 236.5946090221405}\n",
      "Losses {'ner': 234.7660105228424}\n",
      "Losses {'ner': 227.63172352313995}\n",
      "Losses {'ner': 226.79068732261658}\n",
      "Losses {'ner': 234.65489852428436}\n",
      "Losses {'ner': 236.0026547908783}\n",
      "Losses {'ner': 232.23536324501038}\n",
      "Losses {'ner': 238.6355823278427}\n",
      "Losses {'ner': 236.89193177223206}\n",
      "Losses {'ner': 237.09793257713318}\n",
      "Losses {'ner': 221.38135480880737}\n",
      "Losses {'ner': 227.10850954055786}\n",
      "Losses {'ner': 227.50972217321396}\n",
      "Losses {'ner': 235.43993723392487}\n",
      "Losses {'ner': 227.8540482521057}\n",
      "Losses {'ner': 221.64669287204742}\n",
      "Losses {'ner': 234.58693957328796}\n",
      "Losses {'ner': 228.29034316539764}\n",
      "Losses {'ner': 219.91360986232758}\n",
      "Losses {'ner': 234.18029880523682}\n",
      "Losses {'ner': 229.18190574645996}\n",
      "Losses {'ner': 234.2949140071869}\n",
      "Losses {'ner': 235.5141246318817}\n",
      "Losses {'ner': 232.94136595726013}\n",
      "Losses {'ner': 234.60469675064087}\n",
      "Losses {'ner': 235.4009280204773}\n",
      "Losses {'ner': 243.31314170360565}\n",
      "Losses {'ner': 239.68124496936798}\n",
      "Losses {'ner': 226.67416095733643}\n",
      "Losses {'ner': 215.09606647491455}\n",
      "Losses {'ner': 226.88817512989044}\n",
      "Losses {'ner': 218.1072827577591}\n",
      "Losses {'ner': 230.53020524978638}\n",
      "Losses {'ner': 225.17447018623352}\n",
      "Losses {'ner': 237.39911425113678}\n",
      "Losses {'ner': 234.58020734786987}\n",
      "Losses {'ner': 236.8982805609703}\n",
      "Losses {'ner': 216.89029705524445}\n",
      "Losses {'ner': 234.03006744384766}\n",
      "Losses {'ner': 237.3463146686554}\n",
      "Losses {'ner': 227.54864990711212}\n",
      "Losses {'ner': 236.82855224609375}\n",
      "Losses {'ner': 232.5555636882782}\n",
      "Losses {'ner': 224.5311814546585}\n",
      "Losses {'ner': 222.25943183898926}\n",
      "Losses {'ner': 231.87919878959656}\n",
      "Losses {'ner': 226.0765198469162}\n",
      "Losses {'ner': 239.04764699935913}\n",
      "Losses {'ner': 229.89751505851746}\n",
      "Losses {'ner': 231.74445497989655}\n",
      "Losses {'ner': 230.40880799293518}\n",
      "Losses {'ner': 233.59008359909058}\n",
      "Losses {'ner': 230.15752410888672}\n",
      "Losses {'ner': 237.85227584838867}\n",
      "Losses {'ner': 224.2988116145134}\n",
      "Losses {'ner': 231.84819388389587}\n",
      "Losses {'ner': 231.74053120613098}\n",
      "Losses {'ner': 216.5359125137329}\n",
      "Losses {'ner': 217.26924526691437}\n",
      "Losses {'ner': 236.49304008483887}\n",
      "Losses {'ner': 233.7075707912445}\n",
      "Losses {'ner': 234.63729310035706}\n",
      "Losses {'ner': 229.5271577835083}\n",
      "Losses {'ner': 221.6455557346344}\n",
      "Losses {'ner': 235.09732103347778}\n",
      "Losses {'ner': 217.55693876743317}\n",
      "Losses {'ner': 234.90364861488342}\n",
      "Losses {'ner': 236.2039337158203}\n",
      "Losses {'ner': 234.40767002105713}\n",
      "Losses {'ner': 222.32072949409485}\n",
      "Losses {'ner': 230.72283136844635}\n",
      "Losses {'ner': 218.98447799682617}\n",
      "Losses {'ner': 238.13388288021088}\n",
      "Losses {'ner': 233.61290550231934}\n",
      "Losses {'ner': 227.88202214241028}\n",
      "Losses {'ner': 248.70359861850739}\n",
      "Losses {'ner': 223.13816499710083}\n",
      "Losses {'ner': 215.87724328041077}\n",
      "Losses {'ner': 225.1511161327362}\n",
      "Losses {'ner': 216.57358050346375}\n",
      "Losses {'ner': 241.6015658378601}\n",
      "Losses {'ner': 238.63917636871338}\n",
      "Losses {'ner': 227.3639042377472}\n",
      "Losses {'ner': 231.43074941635132}\n",
      "Losses {'ner': 225.51078486442566}\n",
      "Losses {'ner': 238.1449192762375}\n",
      "Losses {'ner': 234.42111492156982}\n",
      "Losses {'ner': 233.9919891357422}\n",
      "Losses {'ner': 236.63592326641083}\n",
      "Losses {'ner': 211.21504986286163}\n",
      "Losses {'ner': 234.89239716529846}\n",
      "Losses {'ner': 223.96294569969177}\n",
      "Losses {'ner': 235.80484628677368}\n",
      "Losses {'ner': 236.2138932943344}\n",
      "Losses {'ner': 218.86517214775085}\n",
      "Losses {'ner': 230.74796617031097}\n",
      "Losses {'ner': 228.10111784934998}\n",
      "Losses {'ner': 226.7437286376953}\n",
      "Losses {'ner': 237.3081170320511}\n",
      "Losses {'ner': 217.31546258926392}\n",
      "Losses {'ner': 237.3634955883026}\n",
      "Losses {'ner': 217.67489099502563}\n",
      "Losses {'ner': 227.7922933101654}\n",
      "Losses {'ner': 224.97795367240906}\n",
      "Losses {'ner': 229.9430320262909}\n",
      "Losses {'ner': 224.35942387580872}\n",
      "Losses {'ner': 230.95757365226746}\n",
      "Losses {'ner': 228.07740533351898}\n",
      "Losses {'ner': 231.7992353439331}\n",
      "Losses {'ner': 231.58376002311707}\n",
      "Losses {'ner': 233.5836580991745}\n",
      "Losses {'ner': 228.2115854024887}\n",
      "Losses {'ner': 242.09370493888855}\n",
      "Losses {'ner': 226.3155393600464}\n",
      "Losses {'ner': 222.33852088451385}\n",
      "Losses {'ner': 237.6827701330185}\n",
      "Losses {'ner': 231.55309599637985}\n",
      "Losses {'ner': 233.67928266525269}\n",
      "Losses {'ner': 234.35350632667542}\n",
      "Losses {'ner': 233.3773124217987}\n",
      "Losses {'ner': 225.47093772888184}\n",
      "Losses {'ner': 221.79010581970215}\n",
      "Losses {'ner': 223.13822412490845}\n",
      "Losses {'ner': 240.67302656173706}\n",
      "Losses {'ner': 227.9901578426361}\n",
      "Losses {'ner': 229.59358632564545}\n",
      "Losses {'ner': 224.39101207256317}\n",
      "Losses {'ner': 228.89546728134155}\n",
      "Losses {'ner': 223.9524827003479}\n",
      "Losses {'ner': 221.14220345020294}\n",
      "Losses {'ner': 229.37087213993073}\n",
      "Losses {'ner': 233.39276504516602}\n",
      "Losses {'ner': 237.23294925689697}\n",
      "Losses {'ner': 223.62021780014038}\n",
      "Losses {'ner': 226.58376336097717}\n",
      "Losses {'ner': 225.9006745815277}\n",
      "Losses {'ner': 234.25578093528748}\n",
      "Losses {'ner': 237.88928651809692}\n",
      "Losses {'ner': 231.38085532188416}\n",
      "Losses {'ner': 223.93113100528717}\n",
      "Losses {'ner': 248.51014590263367}\n",
      "Losses {'ner': 218.55119466781616}\n",
      "Losses {'ner': 229.5038961172104}\n",
      "Losses {'ner': 236.03359937667847}\n",
      "Losses {'ner': 229.0502588748932}\n",
      "Losses {'ner': 234.62123346328735}\n",
      "Losses {'ner': 236.66094374656677}\n",
      "Losses {'ner': 226.90976405143738}\n",
      "Losses {'ner': 223.493337392807}\n",
      "Losses {'ner': 229.21567630767822}\n",
      "Losses {'ner': 228.15833687782288}\n",
      "Losses {'ner': 217.09241795539856}\n",
      "Losses {'ner': 225.71932125091553}\n",
      "Losses {'ner': 238.2604718208313}\n",
      "Losses {'ner': 225.27829432487488}\n",
      "Losses {'ner': 224.71515198796988}\n",
      "Losses {'ner': 229.57903718948364}\n",
      "Losses {'ner': 224.12082636356354}\n",
      "Losses {'ner': 244.55849647521973}\n",
      "Losses {'ner': 233.4465652704239}\n",
      "Losses {'ner': 237.04866993427277}\n",
      "Losses {'ner': 225.20777773857117}\n",
      "Losses {'ner': 218.6595811843872}\n",
      "Losses {'ner': 214.62854647636414}\n",
      "Losses {'ner': 234.52183496952057}\n",
      "Losses {'ner': 233.35893654823303}\n",
      "Losses {'ner': 235.29499399662018}\n",
      "Losses {'ner': 231.7948021888733}\n",
      "Losses {'ner': 232.2324149608612}\n",
      "Losses {'ner': 226.1449375152588}\n",
      "Losses {'ner': 226.11666226387024}\n",
      "Losses {'ner': 230.88155317306519}\n",
      "Losses {'ner': 224.9408210515976}\n",
      "Losses {'ner': 230.9451940059662}\n",
      "Losses {'ner': 235.58747386932373}\n",
      "Losses {'ner': 217.73581719398499}\n",
      "Losses {'ner': 236.52205228805542}\n",
      "Losses {'ner': 226.08472073078156}\n",
      "Losses {'ner': 223.89594331383705}\n",
      "Losses {'ner': 231.8710902929306}\n",
      "Losses {'ner': 216.70196044445038}\n",
      "Losses {'ner': 233.6622064113617}\n",
      "Losses {'ner': 238.3745230436325}\n",
      "Losses {'ner': 223.1033480167389}\n",
      "Losses {'ner': 228.89955615997314}\n",
      "Losses {'ner': 232.4482103586197}\n",
      "Losses {'ner': 236.06813645362854}\n",
      "Losses {'ner': 232.55451142787933}\n",
      "Losses {'ner': 233.69624876976013}\n",
      "Losses {'ner': 230.44201970100403}\n",
      "Losses {'ner': 231.04834055900574}\n",
      "Losses {'ner': 229.97133994102478}\n",
      "Losses {'ner': 221.86905765533447}\n",
      "Losses {'ner': 229.2554633617401}\n",
      "Losses {'ner': 240.98541963100433}\n",
      "Losses {'ner': 234.9305818080902}\n",
      "Losses {'ner': 232.8790955543518}\n",
      "Losses {'ner': 229.56696224212646}\n",
      "Losses {'ner': 227.92602157592773}\n",
      "Losses {'ner': 224.13146996498108}\n",
      "Losses {'ner': 220.96911785006523}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Losses {'ner': 232.46591973304749}\n",
      "Losses {'ner': 230.39528489112854}\n",
      "Losses {'ner': 239.91586256027222}\n",
      "Losses {'ner': 215.36139976978302}\n",
      "Losses {'ner': 226.99625062942505}\n",
      "Losses {'ner': 209.43849098682404}\n",
      "Losses {'ner': 230.13026082515717}\n",
      "Losses {'ner': 234.90162420272827}\n",
      "Losses {'ner': 229.4929929971695}\n",
      "Losses {'ner': 223.6851178407669}\n",
      "Losses {'ner': 237.4086912870407}\n",
      "Losses {'ner': 229.431795835495}\n",
      "Losses {'ner': 211.58354938030243}\n",
      "Losses {'ner': 215.9823590517044}\n",
      "Losses {'ner': 229.30610036849976}\n",
      "Losses {'ner': 230.9776200056076}\n",
      "Losses {'ner': 216.90763974189758}\n",
      "Losses {'ner': 227.31260538101196}\n",
      "Losses {'ner': 234.77997744083405}\n",
      "Losses {'ner': 237.55975449085236}\n",
      "Losses {'ner': 234.50929725170135}\n",
      "Losses {'ner': 221.85109400749207}\n",
      "Losses {'ner': 224.89296460151672}\n",
      "Losses {'ner': 223.93422985076904}\n",
      "Losses {'ner': 211.18497240543365}\n",
      "Losses {'ner': 222.4240279197693}\n",
      "Losses {'ner': 226.46822142601013}\n",
      "Losses {'ner': 225.44879019260406}\n",
      "Losses {'ner': 240.6982342004776}\n",
      "Losses {'ner': 238.85251569747925}\n",
      "Losses {'ner': 237.99100863933563}\n",
      "Losses {'ner': 235.6786448955536}\n",
      "Losses {'ner': 232.9965853691101}\n",
      "Losses {'ner': 226.77588325738907}\n",
      "Losses {'ner': 232.36004996299744}\n",
      "Losses {'ner': 226.19184970855713}\n",
      "Losses {'ner': 237.7822036743164}\n",
      "Losses {'ner': 230.52898144721985}\n",
      "Losses {'ner': 237.67559027671814}\n",
      "Losses {'ner': 229.93893337249756}\n",
      "Losses {'ner': 223.53183138370514}\n",
      "Losses {'ner': 208.7192347049713}\n",
      "Losses {'ner': 232.86458587646484}\n",
      "Losses {'ner': 229.60163807868958}\n",
      "Losses {'ner': 231.56785464286804}\n",
      "Losses {'ner': 234.00230276584625}\n",
      "Losses {'ner': 223.64738988876343}\n",
      "Losses {'ner': 228.71665585041046}\n",
      "Losses {'ner': 222.11167645454407}\n",
      "Losses {'ner': 229.27680087089539}\n",
      "Losses {'ner': 210.9689575433731}\n",
      "Losses {'ner': 224.4649374485016}\n",
      "Losses {'ner': 232.87370102107525}\n",
      "Losses {'ner': 233.90943598747253}\n",
      "Losses {'ner': 231.50684332847595}\n",
      "Losses {'ner': 224.89854383468628}\n",
      "Losses {'ner': 224.40969157218933}\n",
      "Losses {'ner': 220.48676586151123}\n",
      "Losses {'ner': 235.27369856834412}\n",
      "Losses {'ner': 231.23966598510742}\n",
      "Losses {'ner': 234.72202920913696}\n",
      "Losses {'ner': 230.59614849090576}\n",
      "Losses {'ner': 230.1569800376892}\n",
      "Losses {'ner': 226.98285055160522}\n",
      "Losses {'ner': 228.36933517456055}\n",
      "Losses {'ner': 231.01565584540367}\n",
      "Losses {'ner': 228.85593235492706}\n",
      "Losses {'ner': 224.14859741926193}\n",
      "Losses {'ner': 229.16187143325806}\n",
      "Losses {'ner': 225.91441416740417}\n",
      "Losses {'ner': 226.0213302373886}\n",
      "Losses {'ner': 227.45566248893738}\n",
      "Losses {'ner': 218.63451766967773}\n",
      "Losses {'ner': 215.26559591293335}\n",
      "Losses {'ner': 226.41759848594666}\n",
      "Losses {'ner': 222.29867315292358}\n",
      "Losses {'ner': 243.78914880752563}\n",
      "Losses {'ner': 227.5419843196869}\n",
      "Losses {'ner': 232.45325589179993}\n",
      "Losses {'ner': 233.64155220985413}\n",
      "Losses {'ner': 233.3603162765503}\n",
      "Losses {'ner': 225.9168300628662}\n",
      "Losses {'ner': 236.82016468048096}\n",
      "Losses {'ner': 225.07151436805725}\n",
      "Losses {'ner': 229.70201495289803}\n",
      "Losses {'ner': 226.63987386226654}\n",
      "Losses {'ner': 222.69924867153168}\n",
      "Losses {'ner': 230.73102259635925}\n",
      "Losses {'ner': 232.3481261730194}\n",
      "Losses {'ner': 229.3800469636917}\n",
      "Losses {'ner': 219.12453854084015}\n",
      "Losses {'ner': 231.24807608127594}\n",
      "Losses {'ner': 233.93626141548157}\n",
      "Losses {'ner': 219.45159459114075}\n",
      "Losses {'ner': 228.87919116020203}\n",
      "Losses {'ner': 223.44668292999268}\n",
      "Losses {'ner': 229.26880931854248}\n",
      "Losses {'ner': 231.58620953559875}\n",
      "Losses {'ner': 223.7191026210785}\n",
      "Losses {'ner': 234.2499098777771}\n",
      "Losses {'ner': 225.4614658355713}\n",
      "Losses {'ner': 217.52817797660828}\n",
      "Losses {'ner': 232.8793728351593}\n",
      "Losses {'ner': 227.45351779460907}\n",
      "Losses {'ner': 234.547833442688}\n",
      "Losses {'ner': 220.64992207288742}\n",
      "Losses {'ner': 239.966583609581}\n",
      "Losses {'ner': 236.5518925189972}\n",
      "Losses {'ner': 238.6587507724762}\n",
      "Losses {'ner': 232.4783272743225}\n",
      "Losses {'ner': 227.34736359119415}\n",
      "Losses {'ner': 235.2827763557434}\n",
      "Losses {'ner': 226.27241492271423}\n",
      "Losses {'ner': 228.91973233222961}\n",
      "Losses {'ner': 232.01823544502258}\n",
      "Losses {'ner': 224.2271659374237}\n",
      "Losses {'ner': 208.54896140098572}\n",
      "Losses {'ner': 223.7236944437027}\n",
      "Losses {'ner': 224.7783329486847}\n",
      "Losses {'ner': 220.51283288002014}\n",
      "Losses {'ner': 223.03267407417297}\n",
      "Losses {'ner': 233.58041763305664}\n",
      "Losses {'ner': 231.34884524345398}\n",
      "Losses {'ner': 227.8282654285431}\n",
      "Losses {'ner': 227.05568480491638}\n",
      "Losses {'ner': 217.87379240989685}\n",
      "Losses {'ner': 222.25895404815674}\n",
      "Losses {'ner': 236.41646814346313}\n",
      "Losses {'ner': 221.34965300559998}\n",
      "Losses {'ner': 236.71640157699585}\n",
      "Losses {'ner': 220.13518571853638}\n",
      "Losses {'ner': 229.48753023147583}\n",
      "Losses {'ner': 233.19265937805176}\n",
      "Losses {'ner': 234.54015231132507}\n",
      "Losses {'ner': 216.3504101037979}\n",
      "Losses {'ner': 227.49508237838745}\n",
      "Losses {'ner': 240.63237345218658}\n",
      "Losses {'ner': 228.24108064174652}\n",
      "Losses {'ner': 232.74596762657166}\n",
      "Losses {'ner': 221.0195118188858}\n",
      "Losses {'ner': 237.09399795532227}\n",
      "Losses {'ner': 224.28032612800598}\n",
      "Losses {'ner': 216.26869708299637}\n",
      "Losses {'ner': 230.9298621416092}\n",
      "Losses {'ner': 224.96734142303467}\n",
      "Losses {'ner': 223.8146106004715}\n",
      "Losses {'ner': 233.416574716568}\n",
      "Losses {'ner': 221.9500002861023}\n",
      "Losses {'ner': 227.29307651519775}\n",
      "Losses {'ner': 224.0288872718811}\n",
      "Losses {'ner': 230.72652006149292}\n",
      "Losses {'ner': 228.92047929763794}\n",
      "Losses {'ner': 213.1594681739807}\n",
      "Losses {'ner': 234.4955713748932}\n",
      "Losses {'ner': 225.87651872634888}\n",
      "Losses {'ner': 222.4767639040947}\n",
      "Losses {'ner': 232.28774440288544}\n",
      "Losses {'ner': 230.8722470998764}\n",
      "Losses {'ner': 221.45314979553223}\n",
      "Losses {'ner': 230.52624666690826}\n",
      "Losses {'ner': 218.195174574852}\n",
      "Losses {'ner': 239.1206043958664}\n",
      "Losses {'ner': 236.3581805229187}\n",
      "Losses {'ner': 226.244567155838}\n",
      "Losses {'ner': 225.13387191295624}\n",
      "Losses {'ner': 210.75983095169067}\n",
      "Losses {'ner': 231.93500542640686}\n",
      "Losses {'ner': 230.4928616285324}\n",
      "Losses {'ner': 227.72228860855103}\n",
      "Losses {'ner': 221.92693865299225}\n",
      "Losses {'ner': 234.16641700267792}\n",
      "Losses {'ner': 249.36118745803833}\n",
      "Losses {'ner': 230.52097177505493}\n",
      "Losses {'ner': 216.77793192863464}\n",
      "Losses {'ner': 230.6042823791504}\n",
      "Losses {'ner': 228.78093093633652}\n",
      "Losses {'ner': 227.43251538276672}\n",
      "Losses {'ner': 227.84365510940552}\n",
      "Losses {'ner': 234.81271767616272}\n",
      "Losses {'ner': 227.77856707572937}\n",
      "Losses {'ner': 230.72031593322754}\n",
      "Losses {'ner': 237.3574719429016}\n",
      "Losses {'ner': 219.16194415092468}\n",
      "Losses {'ner': 229.94487714767456}\n",
      "Losses {'ner': 224.3306361436844}\n",
      "Losses {'ner': 232.26850819587708}\n",
      "Losses {'ner': 226.52392578125}\n",
      "Losses {'ner': 219.31021630764008}\n",
      "Losses {'ner': 215.64060711860657}\n",
      "Losses {'ner': 221.93046498298645}\n",
      "Losses {'ner': 237.1225483417511}\n",
      "Losses {'ner': 224.36424803733826}\n",
      "Losses {'ner': 224.87571120262146}\n",
      "Losses {'ner': 227.26101458072662}\n",
      "Losses {'ner': 237.51490139961243}\n",
      "Losses {'ner': 223.04349267482758}\n",
      "Losses {'ner': 232.6588089466095}\n",
      "Losses {'ner': 227.2632040977478}\n",
      "Losses {'ner': 225.77198100090027}\n",
      "Losses {'ner': 224.48818063735962}\n",
      "Losses {'ner': 242.72327399253845}\n",
      "Losses {'ner': 226.542142868042}\n",
      "Losses {'ner': 250.46701216697693}\n",
      "Losses {'ner': 223.81351006031036}\n",
      "Losses {'ner': 222.42789328098297}\n",
      "Losses {'ner': 239.20377254486084}\n",
      "Losses {'ner': 223.12618494033813}\n",
      "Losses {'ner': 209.17692458629608}\n",
      "Losses {'ner': 235.48058891296387}\n",
      "Losses {'ner': 235.27759885787964}\n",
      "Losses {'ner': 234.33424925804138}\n",
      "Losses {'ner': 207.434427857399}\n",
      "Losses {'ner': 231.40548181533813}\n",
      "Losses {'ner': 224.5379746556282}\n",
      "Losses {'ner': 225.29167795181274}\n",
      "Losses {'ner': 218.6088489294052}\n",
      "Losses {'ner': 232.96180391311646}\n",
      "Losses {'ner': 217.6973408460617}\n",
      "Losses {'ner': 237.40986514091492}\n",
      "Losses {'ner': 237.85097694396973}\n",
      "Losses {'ner': 234.4577980041504}\n",
      "Losses {'ner': 233.52131247520447}\n",
      "Losses {'ner': 226.20929515361786}\n",
      "Losses {'ner': 226.89863109588623}\n",
      "Losses {'ner': 225.6846297979355}\n",
      "Losses {'ner': 235.78747868537903}\n",
      "Losses {'ner': 228.72855710983276}\n",
      "Losses {'ner': 227.70942783355713}\n",
      "Losses {'ner': 232.33290541172028}\n",
      "Losses {'ner': 229.0754474401474}\n",
      "Losses {'ner': 216.3827407360077}\n",
      "Losses {'ner': 226.98310089111328}\n",
      "Losses {'ner': 218.71267223358154}\n",
      "Losses {'ner': 231.55864918231964}\n",
      "Losses {'ner': 231.79143393039703}\n",
      "Losses {'ner': 228.9815456867218}\n",
      "Losses {'ner': 221.76495361328125}\n",
      "Losses {'ner': 213.6664823293686}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Losses {'ner': 228.01917600631714}\n",
      "Losses {'ner': 233.87990760803223}\n",
      "Losses {'ner': 234.4457609653473}\n",
      "Losses {'ner': 224.98776930570602}\n",
      "Losses {'ner': 214.5377995222807}\n",
      "Losses {'ner': 219.07737922668457}\n",
      "Losses {'ner': 218.5037897825241}\n",
      "Losses {'ner': 236.6707468032837}\n",
      "Losses {'ner': 227.2183063030243}\n",
      "Losses {'ner': 236.67265820503235}\n",
      "Losses {'ner': 244.32980036735535}\n",
      "Losses {'ner': 217.8340916633606}\n",
      "Losses {'ner': 224.1265513896942}\n",
      "Losses {'ner': 233.15836381912231}\n",
      "Losses {'ner': 222.23577427864075}\n",
      "Losses {'ner': 224.4675942659378}\n",
      "Losses {'ner': 217.7204041481018}\n",
      "Losses {'ner': 227.89754927158356}\n",
      "Losses {'ner': 216.13045293092728}\n",
      "Losses {'ner': 222.94964230060577}\n",
      "Losses {'ner': 231.70309925079346}\n",
      "Losses {'ner': 238.57020592689514}\n",
      "Losses {'ner': 229.8649287223816}\n",
      "Losses {'ner': 230.99213552474976}\n",
      "Losses {'ner': 234.4040083885193}\n",
      "Losses {'ner': 217.2306990623474}\n",
      "Losses {'ner': 215.80235528945923}\n",
      "Losses {'ner': 221.53398323059082}\n",
      "Losses {'ner': 223.19354033470154}\n",
      "Losses {'ner': 232.94142985343933}\n",
      "Losses {'ner': 232.65285778045654}\n",
      "Losses {'ner': 231.87804400920868}\n",
      "Losses {'ner': 221.33642834424973}\n",
      "Losses {'ner': 213.01274275779724}\n",
      "Losses {'ner': 234.5610272884369}\n",
      "Losses {'ner': 233.2490028142929}\n",
      "Losses {'ner': 230.20053911209106}\n",
      "Losses {'ner': 229.10562372207642}\n",
      "Losses {'ner': 237.27087926864624}\n",
      "Losses {'ner': 218.433180809021}\n",
      "Losses {'ner': 217.4446861743927}\n",
      "Losses {'ner': 228.8969805240631}\n",
      "Losses {'ner': 225.5678101181984}\n",
      "Losses {'ner': 231.2676156759262}\n",
      "Losses {'ner': 215.37510681152344}\n",
      "Losses {'ner': 217.24105668067932}\n",
      "Losses {'ner': 221.01626074314117}\n",
      "Losses {'ner': 229.05526506900787}\n",
      "Losses {'ner': 229.07695531845093}\n",
      "Entities [('trametinib', 'drug')]\n",
      "Tokens [('This', '', 2), ('article', '', 2), ('reviews', '', 2), ('the', '', 2), ('therapeutic', '', 2), ('efficacy', '', 2), ('and', '', 2), ('tolerability', '', 2), ('of', '', 2), ('combination', '', 2), ('treatment', '', 2), ('with', '', 2), ('dabrafenib', '', 2), ('and', '', 2), ('trametinib', 'drug', 3), ('in', '', 2), ('this', '', 2), ('indication', '', 2), ('and', '', 2), ('summarizes', '', 2), ('relevant', '', 2), ('pharmacological', '', 2), ('data', '', 2), ('.', '', 2)]\n",
      "Entities [('BRAF', 'gene'), ('dabrafenib', 'drug'), ('MEK', 'protien'), ('trametinib', 'drug'), ('melanoma', 'diesease'), ('BRAF', 'gene'), ('V600', 'varient')]\n",
      "Tokens [('The', '', 2), ('BRAF', 'gene', 3), ('inhibitor', '', 2), ('dabrafenib', 'drug', 3), ('(', '', 2), ('Tafinlar(Â', '', 2), ('®', '', 2), (')', '', 2), (')', '', 2), ('and', '', 2), ('the', '', 2), ('MEK', 'protien', 3), ('inhibitor', '', 2), ('trametinib', 'drug', 3), ('(', '', 2), ('Mekinist(Â', '', 2), ('®', '', 2), (')', '', 2), (')', '', 2), ('are', '', 2), ('indicated', '', 2), (',', '', 2), ('as', '', 2), ('monotherapy', '', 2), ('or', '', 2), ('in', '', 2), ('combination', '', 2), ('with', '', 2), ('each', '', 2), ('other', '', 2), (',', '', 2), ('for', '', 2), ('the', '', 2), ('treatment', '', 2), ('of', '', 2), ('patients', '', 2), ('with', '', 2), ('unresectable', '', 2), ('or', '', 2), ('metastatic', '', 2), ('melanoma', 'diesease', 3), ('with', '', 2), ('a', '', 2), ('BRAF', 'gene', 3), ('(', '', 2), ('V600', 'varient', 3), (')', '', 2), ('mutation', '', 2), ('.', '', 2)]\n",
      "Entities [('trametinib', 'drug'), ('BRAF', 'gene'), ('V600', 'varient'), ('melanoma', 'diesease')]\n",
      "Tokens [('Thus', '', 2), (',', '', 2), ('dabrafenib', '', 2), ('plus', '', 2), ('trametinib', 'drug', 3), ('provides', '', 2), ('an', '', 2), ('important', '', 2), ('treatment', '', 2), ('option', '', 2), ('for', '', 2), ('patients', '', 2), ('with', '', 2), ('BRAF', 'gene', 3), ('(', '', 2), ('V600', 'varient', 3), (')', '', 2), ('mutation', '', 2), ('-', '', 2), ('positive', '', 2), ('unresectable', '', 2), ('or', '', 2), ('metastatic', '', 2), ('melanoma', 'diesease', 3), ('.', '', 2)]\n",
      "Entities [('vemurafenib', 'drug')]\n",
      "Tokens [('Combination', '', 2), ('therapy', '', 2), ('did', '', 2), ('not', '', 2), ('increase', '', 2), ('overall', '', 2), ('toxicity', '', 2), ('relative', '', 2), ('to', '', 2), ('dabrafenib', '', 2), ('or', '', 2), ('vemurafenib', 'drug', 3), ('monotherapy', '', 2), (',', '', 2), ('with', '', 2), ('most', '', 2), ('adverse', '', 2), ('events', '', 2), ('(', '', 2), ('AEs', '', 2), (')', '', 2), ('mild', '', 2), ('or', '', 2), ('moderate', '', 2), ('in', '', 2), ('severity', '', 2), ('and', '', 2), ('generally', '', 2), ('manageable', '', 2), ('.', '', 2)]\n",
      "Entities [('Dabrafenib', 'drug'), ('trametinib', 'drug'), ('melanoma', 'diesease'), ('BRAF', 'gene'), ('V600E/K', 'varient')]\n",
      "Tokens [('Dabrafenib', 'drug', 3), ('plus', '', 2), ('trametinib', 'drug', 3), ('significantly', '', 2), ('prolonged', '', 2), ('progression', '', 2), ('-', '', 2), ('free', '', 2), ('survival', '', 2), ('(', '', 2), ('PFS', '', 2), (')', '', 2), ('and', '', 2), ('overall', '', 2), ('survival', '', 2), ('(', '', 2), ('OS', '', 2), (')', '', 2), (',', '', 2), ('improved', '', 2), ('objective', '', 2), ('response', '', 2), ('rates', '', 2), ('(', '', 2), ('ORRs', '', 2), (')', '', 2), ('and', '', 2), ('preserved', '', 2), ('health', '', 2), ('-', '', 2), ('related', '', 2), ('quality', '', 2), ('of', '', 2), ('life', '', 2), ('(', '', 2), ('HR', '', 2), ('-', '', 2), ('QOL', '', 2), (')', '', 2), ('to', '', 2), ('a', '', 2), ('greater', '', 2), ('extent', '', 2), ('than', '', 2), ('dabrafenib', '', 2), ('(', '', 2), ('in', '', 2), ('the', '', 2), ('double', '', 2), ('-', '', 2), ('blind', '', 2), ('COMBI', '', 2), ('-', '', 2), ('d', '', 2), ('study', '', 2), (')', '', 2), ('and', '', 2), ('vemurafenib', '', 2), ('(', '', 2), ('in', '', 2), ('the', '', 2), ('open', '', 2), ('-', '', 2), ('label', '', 2), ('COMBI', '', 2), ('-', '', 2), ('v', '', 2), ('study', '', 2), (')', '', 2), ('in', '', 2), ('two', '', 2), ('large', '', 2), (',', '', 2), ('randomized', '', 2), (',', '', 2), ('phase', '', 2), ('III', '', 2), ('studies', '', 2), ('in', '', 2), ('treatment', '', 2), ('-', '', 2), ('naÃ¯ve', '', 2), ('patients', '', 2), ('with', '', 2), ('unresectable', '', 2), ('or', '', 2), ('metastatic', '', 2), ('melanoma', 'diesease', 3), ('with', '', 2), ('BRAF', 'gene', 3), ('(', '', 2), ('V600E', 'varient', 3), ('/', 'varient', 1), ('K', 'varient', 1), (')', '', 2), ('mutation', '', 2), ('.', '', 2)]\n",
      "Entities [('BRAF', 'gene')]\n",
      "Tokens [('Limited', '', 2), ('treatment', '', 2), ('benefit', '', 2), ('with', '', 2), ('the', '', 2), ('combination', '', 2), ('was', '', 2), ('also', '', 2), ('seen', '', 2), ('in', '', 2), ('patients', '', 2), ('who', '', 2), ('had', '', 2), ('progressed', '', 2), ('on', '', 2), ('prior', '', 2), ('BRAF', 'gene', 3), ('inhibitor', '', 2), ('therapy', '', 2), (',', '', 2), ('as', '', 2), ('indicated', '', 2), ('by', '', 2), ('ORRs', '', 2), ('of', '', 2), ('â‰¤', '', 2), ('15Â', '', 2), ('%', '', 2), ('and', '', 2), ('stable', '', 2), ('disease', '', 2), ('in', '', 2), ('â‰¤', '', 2), ('50Â', '', 2), ('%', '', 2), ('of', '', 2), ('patients', '', 2), ('in', '', 2), ('small', '', 2), ('phase', '', 2), ('I', '', 2), ('and', '', 2), ('II', '', 2), ('studies', '', 2), ('.', '', 2), ('Limited', '', 2), ('treatment', '', 2), ('benefit', '', 2), ('with', '', 2), ('the', '', 2), ('combination', '', 2), ('was', '', 2), ('also', '', 2), ('seen', '', 2), ('in', '', 2), ('patients', '', 2), ('who', '', 2), ('had', '', 2), ('progressed', '', 2), ('on', '', 2), ('prior', '', 2), ('BRAF', '', 2), ('inhibitor', '', 2), ('therapy', '', 2), (',', '', 2), ('as', '', 2), ('indicated', '', 2), ('by', '', 2), ('ORRs', '', 2), ('of', '', 2), ('â‰¤', '', 2), ('15Â', '', 2), ('%', '', 2), ('and', '', 2), ('stable', '', 2), ('disease', '', 2), ('in', '', 2), ('â‰¤', '', 2), ('50Â', '', 2), ('%', '', 2), ('of', '', 2), ('patients', '', 2), ('in', '', 2), ('small', '', 2), ('phase', '', 2), ('I', '', 2), ('and', '', 2), ('II', '', 2), ('studies', '', 2), ('.', '', 2)]\n",
      "Entities [('vemurafenib', 'drug')]\n",
      "Tokens [('Fewer', '', 2), ('skin', '', 2), ('-', '', 2), ('related', '', 2), ('AEs', '', 2), ('(', '', 2), ('e.g.', '', 2), ('cutaneous', '', 2), ('malignancies', '', 2), (',', '', 2), ('hyperkeratinosis', '', 2), ('and', '', 2), ('hand', '', 2), ('-', '', 2), ('foot', '', 2), ('syndrome', '', 2), (')', '', 2), ('were', '', 2), ('reported', '', 2), ('with', '', 2), ('combination', '', 2), ('therapy', '', 2), ('than', '', 2), ('with', '', 2), ('dabrafenib', '', 2), ('or', '', 2), ('vemurafenib', 'drug', 3), (',', '', 2), ('probably', '', 2), ('because', '', 2), ('of', '', 2), ('reduced', '', 2), ('paradoxical', '', 2), ('activation', '', 2), ('of', '', 2), ('the', '', 2), ('MAPK', '', 2), ('pathway', '', 2), ('.', '', 2)]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved model to /home/ubuntu/Grakn/Spacy Play/spacy_en_core_web_lg\n",
      "Loading from /home/ubuntu/Grakn/Spacy Play/spacy_en_core_web_lg\n",
      "Entities [('trametinib', 'drug')]\n",
      "Tokens [('This', '', 2), ('article', '', 2), ('reviews', '', 2), ('the', '', 2), ('therapeutic', '', 2), ('efficacy', '', 2), ('and', '', 2), ('tolerability', '', 2), ('of', '', 2), ('combination', '', 2), ('treatment', '', 2), ('with', '', 2), ('dabrafenib', '', 2), ('and', '', 2), ('trametinib', 'drug', 3), ('in', '', 2), ('this', '', 2), ('indication', '', 2), ('and', '', 2), ('summarizes', '', 2), ('relevant', '', 2), ('pharmacological', '', 2), ('data', '', 2), ('.', '', 2)]\n",
      "Entities [('BRAF', 'gene'), ('dabrafenib', 'drug'), ('MEK', 'protien'), ('trametinib', 'drug'), ('melanoma', 'diesease'), ('BRAF', 'gene'), ('V600', 'varient')]\n",
      "Tokens [('The', '', 2), ('BRAF', 'gene', 3), ('inhibitor', '', 2), ('dabrafenib', 'drug', 3), ('(', '', 2), ('Tafinlar(Â', '', 2), ('®', '', 2), (')', '', 2), (')', '', 2), ('and', '', 2), ('the', '', 2), ('MEK', 'protien', 3), ('inhibitor', '', 2), ('trametinib', 'drug', 3), ('(', '', 2), ('Mekinist(Â', '', 2), ('®', '', 2), (')', '', 2), (')', '', 2), ('are', '', 2), ('indicated', '', 2), (',', '', 2), ('as', '', 2), ('monotherapy', '', 2), ('or', '', 2), ('in', '', 2), ('combination', '', 2), ('with', '', 2), ('each', '', 2), ('other', '', 2), (',', '', 2), ('for', '', 2), ('the', '', 2), ('treatment', '', 2), ('of', '', 2), ('patients', '', 2), ('with', '', 2), ('unresectable', '', 2), ('or', '', 2), ('metastatic', '', 2), ('melanoma', 'diesease', 3), ('with', '', 2), ('a', '', 2), ('BRAF', 'gene', 3), ('(', '', 2), ('V600', 'varient', 3), (')', '', 2), ('mutation', '', 2), ('.', '', 2)]\n",
      "Entities [('trametinib', 'drug'), ('BRAF', 'gene'), ('V600', 'varient'), ('melanoma', 'diesease')]\n",
      "Tokens [('Thus', '', 2), (',', '', 2), ('dabrafenib', '', 2), ('plus', '', 2), ('trametinib', 'drug', 3), ('provides', '', 2), ('an', '', 2), ('important', '', 2), ('treatment', '', 2), ('option', '', 2), ('for', '', 2), ('patients', '', 2), ('with', '', 2), ('BRAF', 'gene', 3), ('(', '', 2), ('V600', 'varient', 3), (')', '', 2), ('mutation', '', 2), ('-', '', 2), ('positive', '', 2), ('unresectable', '', 2), ('or', '', 2), ('metastatic', '', 2), ('melanoma', 'diesease', 3), ('.', '', 2)]\n",
      "Entities [('vemurafenib', 'drug')]\n",
      "Tokens [('Combination', '', 2), ('therapy', '', 2), ('did', '', 2), ('not', '', 2), ('increase', '', 2), ('overall', '', 2), ('toxicity', '', 2), ('relative', '', 2), ('to', '', 2), ('dabrafenib', '', 2), ('or', '', 2), ('vemurafenib', 'drug', 3), ('monotherapy', '', 2), (',', '', 2), ('with', '', 2), ('most', '', 2), ('adverse', '', 2), ('events', '', 2), ('(', '', 2), ('AEs', '', 2), (')', '', 2), ('mild', '', 2), ('or', '', 2), ('moderate', '', 2), ('in', '', 2), ('severity', '', 2), ('and', '', 2), ('generally', '', 2), ('manageable', '', 2), ('.', '', 2)]\n",
      "Entities [('Dabrafenib', 'drug'), ('trametinib', 'drug'), ('melanoma', 'diesease'), ('BRAF', 'gene'), ('V600E/K', 'varient')]\n",
      "Tokens [('Dabrafenib', 'drug', 3), ('plus', '', 2), ('trametinib', 'drug', 3), ('significantly', '', 2), ('prolonged', '', 2), ('progression', '', 2), ('-', '', 2), ('free', '', 2), ('survival', '', 2), ('(', '', 2), ('PFS', '', 2), (')', '', 2), ('and', '', 2), ('overall', '', 2), ('survival', '', 2), ('(', '', 2), ('OS', '', 2), (')', '', 2), (',', '', 2), ('improved', '', 2), ('objective', '', 2), ('response', '', 2), ('rates', '', 2), ('(', '', 2), ('ORRs', '', 2), (')', '', 2), ('and', '', 2), ('preserved', '', 2), ('health', '', 2), ('-', '', 2), ('related', '', 2), ('quality', '', 2), ('of', '', 2), ('life', '', 2), ('(', '', 2), ('HR', '', 2), ('-', '', 2), ('QOL', '', 2), (')', '', 2), ('to', '', 2), ('a', '', 2), ('greater', '', 2), ('extent', '', 2), ('than', '', 2), ('dabrafenib', '', 2), ('(', '', 2), ('in', '', 2), ('the', '', 2), ('double', '', 2), ('-', '', 2), ('blind', '', 2), ('COMBI', '', 2), ('-', '', 2), ('d', '', 2), ('study', '', 2), (')', '', 2), ('and', '', 2), ('vemurafenib', '', 2), ('(', '', 2), ('in', '', 2), ('the', '', 2), ('open', '', 2), ('-', '', 2), ('label', '', 2), ('COMBI', '', 2), ('-', '', 2), ('v', '', 2), ('study', '', 2), (')', '', 2), ('in', '', 2), ('two', '', 2), ('large', '', 2), (',', '', 2), ('randomized', '', 2), (',', '', 2), ('phase', '', 2), ('III', '', 2), ('studies', '', 2), ('in', '', 2), ('treatment', '', 2), ('-', '', 2), ('naÃ¯ve', '', 2), ('patients', '', 2), ('with', '', 2), ('unresectable', '', 2), ('or', '', 2), ('metastatic', '', 2), ('melanoma', 'diesease', 3), ('with', '', 2), ('BRAF', 'gene', 3), ('(', '', 2), ('V600E', 'varient', 3), ('/', 'varient', 1), ('K', 'varient', 1), (')', '', 2), ('mutation', '', 2), ('.', '', 2)]\n",
      "Entities [('BRAF', 'gene')]\n",
      "Tokens [('Limited', '', 2), ('treatment', '', 2), ('benefit', '', 2), ('with', '', 2), ('the', '', 2), ('combination', '', 2), ('was', '', 2), ('also', '', 2), ('seen', '', 2), ('in', '', 2), ('patients', '', 2), ('who', '', 2), ('had', '', 2), ('progressed', '', 2), ('on', '', 2), ('prior', '', 2), ('BRAF', 'gene', 3), ('inhibitor', '', 2), ('therapy', '', 2), (',', '', 2), ('as', '', 2), ('indicated', '', 2), ('by', '', 2), ('ORRs', '', 2), ('of', '', 2), ('â‰¤', '', 2), ('15Â', '', 2), ('%', '', 2), ('and', '', 2), ('stable', '', 2), ('disease', '', 2), ('in', '', 2), ('â‰¤', '', 2), ('50Â', '', 2), ('%', '', 2), ('of', '', 2), ('patients', '', 2), ('in', '', 2), ('small', '', 2), ('phase', '', 2), ('I', '', 2), ('and', '', 2), ('II', '', 2), ('studies', '', 2), ('.', '', 2), ('Limited', '', 2), ('treatment', '', 2), ('benefit', '', 2), ('with', '', 2), ('the', '', 2), ('combination', '', 2), ('was', '', 2), ('also', '', 2), ('seen', '', 2), ('in', '', 2), ('patients', '', 2), ('who', '', 2), ('had', '', 2), ('progressed', '', 2), ('on', '', 2), ('prior', '', 2), ('BRAF', '', 2), ('inhibitor', '', 2), ('therapy', '', 2), (',', '', 2), ('as', '', 2), ('indicated', '', 2), ('by', '', 2), ('ORRs', '', 2), ('of', '', 2), ('â‰¤', '', 2), ('15Â', '', 2), ('%', '', 2), ('and', '', 2), ('stable', '', 2), ('disease', '', 2), ('in', '', 2), ('â‰¤', '', 2), ('50Â', '', 2), ('%', '', 2), ('of', '', 2), ('patients', '', 2), ('in', '', 2), ('small', '', 2), ('phase', '', 2), ('I', '', 2), ('and', '', 2), ('II', '', 2), ('studies', '', 2), ('.', '', 2)]\n",
      "Entities [('vemurafenib', 'drug')]\n",
      "Tokens [('Fewer', '', 2), ('skin', '', 2), ('-', '', 2), ('related', '', 2), ('AEs', '', 2), ('(', '', 2), ('e.g.', '', 2), ('cutaneous', '', 2), ('malignancies', '', 2), (',', '', 2), ('hyperkeratinosis', '', 2), ('and', '', 2), ('hand', '', 2), ('-', '', 2), ('foot', '', 2), ('syndrome', '', 2), (')', '', 2), ('were', '', 2), ('reported', '', 2), ('with', '', 2), ('combination', '', 2), ('therapy', '', 2), ('than', '', 2), ('with', '', 2), ('dabrafenib', '', 2), ('or', '', 2), ('vemurafenib', 'drug', 3), (',', '', 2), ('probably', '', 2), ('because', '', 2), ('of', '', 2), ('reduced', '', 2), ('paradoxical', '', 2), ('activation', '', 2), ('of', '', 2), ('the', '', 2), ('MAPK', '', 2), ('pathway', '', 2), ('.', '', 2)]\n"
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    main(model = \"en_core_web_lg\",output_dir= path_model_custom)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NER "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_doc = 'The BRAF inhibitor dabrafenib (Tafinlar(Â®)) and the MEK inhibitor trametinib (Mekinist(Â®)) are indicated, as monotherapy or in combination with each other, for the treatment of patients with unresectable or metastatic melanoma with a BRAF (V600) mutation. This article reviews the therapeutic efficacy and tolerability of combination treatment with dabrafenib and trametinib in this indication and summarizes relevant pharmacological data. Dabrafenib plus trametinib significantly prolonged progression-free survival (PFS) and overall survival (OS), improved objective response rates (ORRs) and preserved health-related quality of life (HR-QOL) to a greater extent than dabrafenib (in the double-blind COMBI-d study) and vemurafenib (in the open-label COMBI-v study) in two large, randomized, phase III studies in treatment-naÃ¯ve patients with unresectable or metastatic melanoma with BRAF (V600E/K) mutation. Limited treatment benefit with the combination was also seen in patients who had progressed on prior BRAF inhibitor therapy, as indicated by ORRs of â‰¤ 15Â\\xa0% and stable disease in â‰¤ 50Â\\xa0% of patients in small phase I and II studies. Combination therapy did not increase overall toxicity relative to dabrafenib or vemurafenib monotherapy, with most adverse events (AEs) mild or moderate in severity and generally manageable. Fewer skin-related AEs (e.g. cutaneous malignancies, hyperkeratinosis and hand-foot syndrome) were reported with combination therapy than with dabrafenib or vemurafenib, probably because of reduced paradoxical activation of the MAPK pathway. Thus, dabrafenib plus trametinib provides an important treatment option for patients with BRAF (V600) mutation-positive unresectable or metastatic melanoma.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BRAF --> gene\n",
      "dabrafenib --> drug\n",
      "MEK --> protien\n",
      "trametinib --> drug\n",
      "melanoma --> diesease\n",
      "BRAF --> gene\n",
      "V600 --> varient\n",
      "trametinib --> drug\n",
      "Dabrafenib --> drug\n",
      "trametinib --> drug\n",
      "melanoma --> diesease\n",
      "BRAF --> gene\n",
      "V600E/K --> varient\n",
      "vemurafenib --> drug\n",
      "vemurafenib --> drug\n",
      "trametinib --> drug\n",
      "BRAF --> gene\n",
      "V600 --> varient\n",
      "melanoma --> diesease\n"
     ]
    }
   ],
   "source": [
    "#nlp = spacy.load(\"en_core_web_lg\")\n",
    "#doc = nlp(u\"Apple is looking at buying U.K. startup for $1 billion\")\n",
    "\n",
    "nlp = spacy.load(\"/home/ubuntu/Grakn/Spacy Play/spacy_en_core_web_lg\")\n",
    "doc1 = nlp(test_doc)\n",
    "\n",
    "\n",
    "\n",
    "# for ent in doc.ents:\n",
    "#     print(ent.text, ent.start_char, ent.end_char, ent.label_)\n",
    "\n",
    "for ent in doc1.ents:\n",
    "    print(ent.text,\"-->\",ent.label_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Playing with Spacy Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc ='''The BRAF inhibitor dabrafenib (Tafinlar(Â®)) and the MEK inhibitor trametinib (Mekinist(Â®)) are indicated, as monotherapy or in combination with each other, for the treatment of patients with unresectable or metastatic melanoma with a BRAF (V600) mutation. \n",
    "This article reviews the therapeutic efficacy and tolerability of combination treatment with dabrafenib and trametinib in this indication and summarizes relevant pharmacological data. \n",
    "Dabrafenib plus trametinib significantly prolonged progression-free survival (PFS) and overall survival (OS), improved objective response rates (ORRs) and preserved health-related quality of life (HR-QOL) to a greater extent than dabrafenib (in the double-blind COMBI-d study) and vemurafenib (in the open-label COMBI-v study) in two large, randomized, phase III studies in treatment-naÃ¯ve patients with unresectable or metastatic melanoma with BRAF (V600E/K) mutation. Limited treatment benefit with the combination was also seen in patients who had progressed on prior BRAF inhibitor therapy, as indicated by ORRs of â‰¤ 15Â % and stable disease in â‰¤ 50Â % of patients in small phase I and II studies. Combination therapy did not increase overall toxicity relative to dabrafenib or vemurafenib monotherapy, with most adverse events (AEs) mild or moderate in severity and generally manageable. Fewer skin-related AEs (e.g. cutaneous malignancies, hyperkeratinosis and hand-foot syndrome) were reported with combination therapy than with dabrafenib or vemurafenib, probably because of reduced paradoxical activation of the MAPK pathway. Thus, dabrafenib plus trametinib provides an important treatment option for patients with BRAF (V600) mutation-positive unresectable or metastatic melanoma.\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc1 = '''Hi, my name is Tobias. I’m a trained mathematician that now works as a data scientist and machine learning engineer. In 2018 I achieved the status of a kaggle master and still spend quite some time there.'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading spacy model\n",
    "# In this case, trained model is being loaded\n",
    "nlp = spacy.load(\"/home/ubuntu/Grakn/Spacy Play/spacy_en_core_web_lg\")#(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parsing document to spacy loaded model\n",
    "nlp_doc = nlp(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#---- POS Tagging ----#\n",
      "\n",
      "The --> the --> DET --> DT --> det --> Xxx --> True --> True\n",
      "BRAF --> BRAF --> PROPN --> NNP --> compound --> XXXX --> True --> False\n",
      "inhibitor --> inhibitor --> NOUN --> NN --> compound --> xxxx --> True --> False\n",
      "dabrafenib --> dabrafenib --> NOUN --> NN --> nsubjpass --> xxxx --> True --> False\n",
      "( --> ( --> PUNCT --> -LRB- --> punct --> ( --> False --> False\n",
      "Tafinlar(Â --> Tafinlar(Â --> PROPN --> NNP --> compound --> Xxxxx(X --> False --> False\n",
      "® --> ® --> PROPN --> NNP --> appos --> ® --> False --> False\n",
      ") --> ) --> PUNCT --> -RRB- --> punct --> ) --> False --> False\n",
      ") --> ) --> PUNCT --> -RRB- --> punct --> ) --> False --> False\n",
      "and --> and --> CCONJ --> CC --> cc --> xxx --> True --> True\n",
      "the --> the --> DET --> DT --> det --> xxx --> True --> True\n",
      "MEK --> MEK --> PROPN --> NNP --> compound --> XXX --> True --> False\n",
      "inhibitor --> inhibitor --> NOUN --> NN --> compound --> xxxx --> True --> False\n",
      "trametinib --> trametinib --> NOUN --> NN --> conj --> xxxx --> True --> False\n",
      "( --> ( --> PUNCT --> -LRB- --> punct --> ( --> False --> False\n",
      "Mekinist(Â --> Mekinist(Â --> PROPN --> NNP --> compound --> Xxxxx(X --> False --> False\n",
      "® --> ® --> PROPN --> NNP --> appos --> ® --> False --> False\n",
      ") --> ) --> PUNCT --> -RRB- --> punct --> ) --> False --> False\n",
      ") --> ) --> PUNCT --> -RRB- --> punct --> ) --> False --> False\n",
      "are --> be --> VERB --> VBP --> auxpass --> xxx --> True --> True\n",
      "indicated --> indicate --> VERB --> VBN --> ROOT --> xxxx --> True --> False\n",
      ", --> , --> PUNCT --> , --> punct --> , --> False --> False\n",
      "as --> as --> ADP --> IN --> advmod --> xx --> True --> True\n",
      "monotherapy --> monotherapy --> ADV --> RB --> advmod --> xxxx --> True --> False\n",
      "or --> or --> CCONJ --> CC --> cc --> xx --> True --> True\n",
      "in --> in --> ADP --> IN --> advcl --> xx --> True --> True\n",
      "combination --> combination --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      "with --> with --> ADP --> IN --> prep --> xxxx --> True --> True\n",
      "each --> each --> DET --> DT --> det --> xxxx --> True --> True\n",
      "other --> other --> ADJ --> JJ --> pobj --> xxxx --> True --> True\n",
      ", --> , --> PUNCT --> , --> punct --> , --> False --> False\n",
      "for --> for --> ADP --> IN --> conj --> xxx --> True --> True\n",
      "the --> the --> DET --> DT --> det --> xxx --> True --> True\n",
      "treatment --> treatment --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      "of --> of --> ADP --> IN --> prep --> xx --> True --> True\n",
      "patients --> patient --> NOUN --> NNS --> pobj --> xxxx --> True --> False\n",
      "with --> with --> ADP --> IN --> prep --> xxxx --> True --> True\n",
      "unresectable --> unresectable --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "or --> or --> CCONJ --> CC --> cc --> xx --> True --> True\n",
      "metastatic --> metastatic --> ADJ --> JJ --> conj --> xxxx --> True --> False\n",
      "melanoma --> melanoma --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      "with --> with --> ADP --> IN --> prep --> xxxx --> True --> True\n",
      "a --> a --> DET --> DT --> det --> x --> True --> True\n",
      "BRAF --> BRAF --> PROPN --> NNP --> nmod --> XXXX --> True --> False\n",
      "( --> ( --> PUNCT --> -LRB- --> punct --> ( --> False --> False\n",
      "V600 --> V600 --> PROPN --> NNP --> subtok --> Xddd --> False --> False\n",
      ") --> ) --> PUNCT --> -RRB- --> punct --> ) --> False --> False\n",
      "mutation --> mutation --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      ". --> . --> PUNCT --> . --> punct --> . --> False --> False\n",
      "\n",
      " --> \n",
      " --> SPACE --> _SP -->  --> \n",
      " --> False --> False\n",
      "This --> this --> DET --> DT --> det --> Xxxx --> True --> True\n",
      "article --> article --> NOUN --> NN --> nsubj --> xxxx --> True --> False\n",
      "reviews --> review --> VERB --> VBZ --> ROOT --> xxxx --> True --> False\n",
      "the --> the --> DET --> DT --> det --> xxx --> True --> True\n",
      "therapeutic --> therapeutic --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "efficacy --> efficacy --> NOUN --> NN --> dobj --> xxxx --> True --> False\n",
      "and --> and --> CCONJ --> CC --> cc --> xxx --> True --> True\n",
      "tolerability --> tolerability --> NOUN --> NN --> conj --> xxxx --> True --> False\n",
      "of --> of --> ADP --> IN --> prep --> xx --> True --> True\n",
      "combination --> combination --> NOUN --> NN --> compound --> xxxx --> True --> False\n",
      "treatment --> treatment --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      "with --> with --> ADP --> IN --> prep --> xxxx --> True --> True\n",
      "dabrafenib --> dabrafenib --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      "and --> and --> CCONJ --> CC --> cc --> xxx --> True --> True\n",
      "trametinib --> trametinib --> NOUN --> NN --> conj --> xxxx --> True --> False\n",
      "in --> in --> ADP --> IN --> prep --> xx --> True --> True\n",
      "this --> this --> DET --> DT --> det --> xxxx --> True --> True\n",
      "indication --> indication --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      "and --> and --> CCONJ --> CC --> cc --> xxx --> True --> True\n",
      "summarizes --> summarize --> VERB --> VBZ --> conj --> xxxx --> True --> False\n",
      "relevant --> relevant --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "pharmacological --> pharmacological --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "data --> datum --> NOUN --> NNS --> dobj --> xxxx --> True --> False\n",
      ". --> . --> PUNCT --> . --> punct --> . --> False --> False\n",
      "\n",
      " --> \n",
      " --> SPACE --> _SP -->  --> \n",
      " --> False --> False\n",
      "Dabrafenib --> Dabrafenib --> PROPN --> NNP --> nsubj --> Xxxxx --> True --> False\n",
      "plus --> plus --> CCONJ --> CC --> cc --> xxxx --> True --> False\n",
      "trametinib --> trametinib --> NOUN --> NN --> conj --> xxxx --> True --> False\n",
      "significantly --> significantly --> ADV --> RB --> advmod --> xxxx --> True --> False\n",
      "prolonged --> prolong --> VERB --> VBD --> amod --> xxxx --> True --> False\n",
      "progression --> progression --> NOUN --> NN --> npadvmod --> xxxx --> True --> False\n",
      "- --> - --> PUNCT --> HYPH --> punct --> - --> False --> False\n",
      "free --> free --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "survival --> survival --> NOUN --> NN --> conj --> xxxx --> True --> False\n",
      "( --> ( --> PUNCT --> -LRB- --> punct --> ( --> False --> False\n",
      "PFS --> PFS --> PROPN --> NNP --> appos --> XXX --> True --> False\n",
      ") --> ) --> PUNCT --> -RRB- --> punct --> ) --> False --> False\n",
      "and --> and --> CCONJ --> CC --> cc --> xxx --> True --> True\n",
      "overall --> overall --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "survival --> survival --> NOUN --> NN --> conj --> xxxx --> True --> False\n",
      "( --> ( --> PUNCT --> -LRB- --> punct --> ( --> False --> False\n",
      "OS --> OS --> PROPN --> NNP --> appos --> XX --> True --> False\n",
      ") --> ) --> PUNCT --> -RRB- --> punct --> ) --> False --> False\n",
      ", --> , --> PUNCT --> , --> punct --> , --> False --> False\n",
      "improved --> improve --> VERB --> VBN --> ROOT --> xxxx --> True --> False\n",
      "objective --> objective --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "response --> response --> NOUN --> NN --> compound --> xxxx --> True --> False\n",
      "rates --> rate --> NOUN --> NNS --> dobj --> xxxx --> True --> False\n",
      "( --> ( --> PUNCT --> -LRB- --> punct --> ( --> False --> False\n",
      "ORRs --> orr --> NOUN --> NNS --> appos --> XXXx --> True --> False\n",
      ") --> ) --> PUNCT --> -RRB- --> punct --> ) --> False --> False\n",
      "and --> and --> CCONJ --> CC --> cc --> xxx --> True --> True\n",
      "preserved --> preserve --> VERB --> VBN --> amod --> xxxx --> True --> False\n",
      "health --> health --> NOUN --> NN --> npadvmod --> xxxx --> True --> False\n",
      "- --> - --> PUNCT --> HYPH --> punct --> - --> False --> False\n",
      "related --> relate --> VERB --> VBN --> amod --> xxxx --> True --> False\n",
      "quality --> quality --> NOUN --> NN --> conj --> xxxx --> True --> False\n",
      "of --> of --> ADP --> IN --> prep --> xx --> True --> True\n",
      "life --> life --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      "( --> ( --> PUNCT --> -LRB- --> punct --> ( --> False --> False\n",
      "HR --> HR --> PROPN --> NNP --> compound --> XX --> True --> False\n",
      "- --> - --> PUNCT --> HYPH --> punct --> - --> False --> False\n",
      "QOL --> QOL --> PROPN --> NNP --> appos --> XXX --> True --> False\n",
      ") --> ) --> PUNCT --> -RRB- --> punct --> ) --> False --> False\n",
      "to --> to --> ADP --> IN --> prep --> xx --> True --> True\n",
      "a --> a --> DET --> DT --> det --> x --> True --> True\n",
      "greater --> great --> ADJ --> JJR --> amod --> xxxx --> True --> False\n",
      "extent --> extent --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      "than --> than --> ADP --> IN --> prep --> xxxx --> True --> True\n",
      "dabrafenib --> dabrafenib --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      "( --> ( --> PUNCT --> -LRB- --> punct --> ( --> False --> False\n",
      "in --> in --> ADP --> IN --> prep --> xx --> True --> True\n",
      "the --> the --> DET --> DT --> det --> xxx --> True --> True\n",
      "double --> double --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "- --> - --> PUNCT --> HYPH --> punct --> - --> False --> False\n",
      "blind --> blind --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "COMBI --> combi --> NOUN --> NN --> compound --> XXXX --> True --> False\n",
      "- --> - --> PUNCT --> HYPH --> punct --> - --> False --> False\n",
      "d --> d --> ADJ --> JJ --> compound --> x --> True --> False\n",
      "study --> study --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      ") --> ) --> PUNCT --> -RRB- --> punct --> ) --> False --> False\n",
      "and --> and --> CCONJ --> CC --> cc --> xxx --> True --> True\n",
      "vemurafenib --> vemurafenib --> NOUN --> NN --> conj --> xxxx --> True --> False\n",
      "( --> ( --> PUNCT --> -LRB- --> punct --> ( --> False --> False\n",
      "in --> in --> ADP --> IN --> prep --> xx --> True --> True\n",
      "the --> the --> DET --> DT --> det --> xxx --> True --> True\n",
      "open --> open --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "- --> - --> PUNCT --> HYPH --> punct --> - --> False --> False\n",
      "label --> label --> NOUN --> NN --> compound --> xxxx --> True --> False\n",
      "COMBI --> COMBI --> PROPN --> NNP --> compound --> XXXX --> True --> False\n",
      "- --> - --> PUNCT --> HYPH --> punct --> - --> False --> False\n",
      "v --> v --> PROPN --> NNP --> compound --> x --> True --> False\n",
      "study --> study --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      ") --> ) --> PUNCT --> -RRB- --> punct --> ) --> False --> False\n",
      "in --> in --> ADP --> IN --> prep --> xx --> True --> True\n",
      "two --> two --> NUM --> CD --> nummod --> xxx --> True --> True\n",
      "large --> large --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      ", --> , --> PUNCT --> , --> punct --> , --> False --> False\n",
      "randomized --> randomize --> VERB --> VBN --> conj --> xxxx --> True --> False\n",
      ", --> , --> PUNCT --> , --> punct --> , --> False --> False\n",
      "phase --> phase --> NOUN --> NN --> compound --> xxxx --> True --> False\n",
      "III --> iii --> NUM --> CD --> compound --> XXX --> True --> False\n",
      "studies --> study --> NOUN --> NNS --> pobj --> xxxx --> True --> False\n",
      "in --> in --> ADP --> IN --> prep --> xx --> True --> True\n",
      "treatment --> treatment --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      "- --> - --> PUNCT --> HYPH --> punct --> - --> False --> False\n",
      "naÃ¯ve --> naã¯ve --> NUM --> CD --> punct --> xxX¯xx --> False --> False\n",
      "patients --> patient --> NOUN --> NNS --> dobj --> xxxx --> True --> False\n",
      "with --> with --> ADP --> IN --> prep --> xxxx --> True --> True\n",
      "unresectable --> unresectable --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "or --> or --> CCONJ --> CC --> cc --> xx --> True --> True\n",
      "metastatic --> metastatic --> ADJ --> JJ --> conj --> xxxx --> True --> False\n",
      "melanoma --> melanoma --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      "with --> with --> ADP --> IN --> prep --> xxxx --> True --> True\n",
      "BRAF --> BRAF --> PROPN --> NNP --> nmod --> XXXX --> True --> False\n",
      "( --> ( --> PUNCT --> -LRB- --> punct --> ( --> False --> False\n",
      "V600E --> V600E --> PROPN --> NNP --> nmod --> XdddX --> False --> False\n",
      "/ --> / --> SYM --> SYM --> punct --> / --> False --> False\n",
      "K --> K --> PROPN --> NNP --> nmod --> X --> True --> False\n",
      ") --> ) --> PUNCT --> -RRB- --> punct --> ) --> False --> False\n",
      "mutation --> mutation --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      ". --> . --> PUNCT --> . --> punct --> . --> False --> False\n",
      "Limited --> limited --> ADJ --> JJ --> amod --> Xxxxx --> True --> False\n",
      "treatment --> treatment --> NOUN --> NN --> compound --> xxxx --> True --> False\n",
      "benefit --> benefit --> NOUN --> NN --> nsubjpass --> xxxx --> True --> False\n",
      "with --> with --> ADP --> IN --> prep --> xxxx --> True --> True\n",
      "the --> the --> DET --> DT --> det --> xxx --> True --> True\n",
      "combination --> combination --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      "was --> be --> VERB --> VBD --> auxpass --> xxx --> True --> True\n",
      "also --> also --> ADV --> RB --> advmod --> xxxx --> True --> True\n",
      "seen --> see --> VERB --> VBN --> ROOT --> xxxx --> True --> False\n",
      "in --> in --> ADP --> IN --> prep --> xx --> True --> True\n",
      "patients --> patient --> NOUN --> NNS --> pobj --> xxxx --> True --> False\n",
      "who --> who --> PRON --> WP --> nsubj --> xxx --> True --> True\n",
      "had --> have --> VERB --> VBD --> aux --> xxx --> True --> True\n",
      "progressed --> progress --> VERB --> VBN --> relcl --> xxxx --> True --> False\n",
      "on --> on --> ADP --> IN --> prep --> xx --> True --> True\n",
      "prior --> prior --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "BRAF --> BRAF --> PROPN --> NNP --> compound --> XXXX --> True --> False\n",
      "inhibitor --> inhibitor --> NOUN --> NN --> compound --> xxxx --> True --> False\n",
      "therapy --> therapy --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      ", --> , --> PUNCT --> , --> punct --> , --> False --> False\n",
      "as --> as --> ADP --> IN --> mark --> xx --> True --> True\n",
      "indicated --> indicate --> VERB --> VBN --> advcl --> xxxx --> True --> False\n",
      "by --> by --> ADP --> IN --> agent --> xx --> True --> True\n",
      "ORRs --> orr --> NOUN --> NNS --> pobj --> XXXx --> True --> False\n",
      "of --> of --> ADP --> IN --> prep --> xx --> True --> True\n",
      "â‰¤ --> â‰¤ --> SYM --> $ --> nmod --> x‰¤ --> False --> False\n",
      "15Â --> 15â --> NUM --> CD --> nummod --> ddX --> False --> False\n",
      "% --> % --> NOUN --> NN --> pobj --> % --> False --> False\n",
      "and --> and --> CCONJ --> CC --> cc --> xxx --> True --> True\n",
      "stable --> stable --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "disease --> disease --> NOUN --> NN --> conj --> xxxx --> True --> False\n",
      "in --> in --> ADP --> IN --> prep --> xx --> True --> True\n",
      "â‰¤ --> â‰¤ --> SYM --> $ --> pobj --> x‰¤ --> False --> False\n",
      "50Â --> 50â --> NUM --> CD --> nummod --> ddX --> False --> False\n",
      "% --> % --> NOUN --> NN --> dobj --> % --> False --> False\n",
      "of --> of --> ADP --> IN --> prep --> xx --> True --> True\n",
      "patients --> patient --> NOUN --> NNS --> pobj --> xxxx --> True --> False\n",
      "in --> in --> ADP --> IN --> prep --> xx --> True --> True\n",
      "small --> small --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "phase --> phase --> NOUN --> NN --> nmod --> xxxx --> True --> False\n",
      "I --> -PRON- --> PRON --> PRP --> nmod --> X --> True --> True\n",
      "and --> and --> CCONJ --> CC --> cc --> xxx --> True --> True\n",
      "II --> II --> PROPN --> NNP --> conj --> XX --> True --> False\n",
      "studies --> study --> NOUN --> NNS --> pobj --> xxxx --> True --> False\n",
      ". --> . --> PUNCT --> . --> punct --> . --> False --> False\n",
      "Combination --> combination --> NOUN --> NN --> compound --> Xxxxx --> True --> False\n",
      "therapy --> therapy --> NOUN --> NN --> nsubj --> xxxx --> True --> False\n",
      "did --> do --> VERB --> VBD --> aux --> xxx --> True --> True\n",
      "not --> not --> ADV --> RB --> neg --> xxx --> True --> True\n",
      "increase --> increase --> VERB --> VB --> ROOT --> xxxx --> True --> False\n",
      "overall --> overall --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "toxicity --> toxicity --> NOUN --> NN --> dobj --> xxxx --> True --> False\n",
      "relative --> relative --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "to --> to --> ADP --> IN --> prep --> xx --> True --> True\n",
      "dabrafenib --> dabrafenib --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      "or --> or --> CCONJ --> CC --> cc --> xx --> True --> True\n",
      "vemurafenib --> vemurafenib --> NOUN --> NN --> conj --> xxxx --> True --> False\n",
      "monotherapy --> monotherapy --> NOUN --> NN --> advmod --> xxxx --> True --> False\n",
      ", --> , --> PUNCT --> , --> punct --> , --> False --> False\n",
      "with --> with --> ADP --> IN --> prep --> xxxx --> True --> True\n",
      "most --> most --> ADJ --> JJS --> amod --> xxxx --> True --> True\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "adverse --> adverse --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "events --> event --> NOUN --> NNS --> pobj --> xxxx --> True --> False\n",
      "( --> ( --> PUNCT --> -LRB- --> punct --> ( --> False --> False\n",
      "AEs --> AEs --> PROPN --> NNP --> appos --> XXx --> True --> False\n",
      ") --> ) --> PUNCT --> -RRB- --> punct --> ) --> False --> False\n",
      "mild --> mild --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "or --> or --> CCONJ --> CC --> cc --> xx --> True --> True\n",
      "moderate --> moderate --> ADJ --> JJ --> conj --> xxxx --> True --> False\n",
      "in --> in --> ADP --> IN --> prep --> xx --> True --> True\n",
      "severity --> severity --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      "and --> and --> CCONJ --> CC --> cc --> xxx --> True --> True\n",
      "generally --> generally --> ADV --> RB --> advmod --> xxxx --> True --> False\n",
      "manageable --> manageable --> ADJ --> JJ --> conj --> xxxx --> True --> False\n",
      ". --> . --> PUNCT --> . --> punct --> . --> False --> False\n",
      "Fewer --> few --> ADJ --> JJR --> amod --> Xxxxx --> True --> False\n",
      "skin --> skin --> NOUN --> NN --> npadvmod --> xxxx --> True --> False\n",
      "- --> - --> PUNCT --> HYPH --> punct --> - --> False --> False\n",
      "related --> relate --> VERB --> VBN --> amod --> xxxx --> True --> False\n",
      "AEs --> ae --> NOUN --> NNS --> nsubjpass --> XXx --> True --> False\n",
      "( --> ( --> PUNCT --> -LRB- --> punct --> ( --> False --> False\n",
      "e.g. --> e.g. --> X --> FW --> amod --> x.x. --> False --> False\n",
      "cutaneous --> cutaneous --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "malignancies --> malignancy --> NOUN --> NNS --> appos --> xxxx --> True --> False\n",
      ", --> , --> PUNCT --> , --> punct --> , --> False --> False\n",
      "hyperkeratinosis --> hyperkeratinosis --> NOUN --> NN --> conj --> xxxx --> True --> False\n",
      "and --> and --> CCONJ --> CC --> cc --> xxx --> True --> True\n",
      "hand --> hand --> NOUN --> NN --> compound --> xxxx --> True --> False\n",
      "- --> - --> PUNCT --> HYPH --> punct --> - --> False --> False\n",
      "foot --> foot --> NOUN --> NN --> compound --> xxxx --> True --> False\n",
      "syndrome --> syndrome --> NOUN --> NN --> conj --> xxxx --> True --> False\n",
      ") --> ) --> PUNCT --> -RRB- --> punct --> ) --> False --> False\n",
      "were --> be --> VERB --> VBD --> auxpass --> xxxx --> True --> True\n",
      "reported --> report --> VERB --> VBN --> ROOT --> xxxx --> True --> False\n",
      "with --> with --> ADP --> IN --> prep --> xxxx --> True --> True\n",
      "combination --> combination --> NOUN --> NN --> compound --> xxxx --> True --> False\n",
      "therapy --> therapy --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      "than --> than --> ADP --> IN --> prep --> xxxx --> True --> True\n",
      "with --> with --> ADP --> IN --> prep --> xxxx --> True --> True\n",
      "dabrafenib --> dabrafenib --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      "or --> or --> CCONJ --> CC --> cc --> xx --> True --> True\n",
      "vemurafenib --> vemurafenib --> NOUN --> NN --> conj --> xxxx --> True --> False\n",
      ", --> , --> PUNCT --> , --> punct --> , --> False --> False\n",
      "probably --> probably --> ADV --> RB --> advmod --> xxxx --> True --> False\n",
      "because --> because --> ADP --> IN --> prep --> xxxx --> True --> True\n",
      "of --> of --> ADP --> IN --> pcomp --> xx --> True --> True\n",
      "reduced --> reduced --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "paradoxical --> paradoxical --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "activation --> activation --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      "of --> of --> ADP --> IN --> prep --> xx --> True --> True\n",
      "the --> the --> DET --> DT --> det --> xxx --> True --> True\n",
      "MAPK --> MAPK --> PROPN --> NNP --> compound --> XXXX --> True --> False\n",
      "pathway --> pathway --> NOUN --> NN --> pobj --> xxxx --> True --> False\n",
      ". --> . --> PUNCT --> . --> punct --> . --> False --> False\n",
      "Thus --> thus --> ADV --> RB --> advmod --> Xxxx --> True --> True\n",
      ", --> , --> PUNCT --> , --> punct --> , --> False --> False\n",
      "dabrafenib --> dabrafenib --> NOUN --> NN --> nsubj --> xxxx --> True --> False\n",
      "plus --> plus --> CCONJ --> CC --> cc --> xxxx --> True --> False\n",
      "trametinib --> trametinib --> NOUN --> NN --> conj --> xxxx --> True --> False\n",
      "provides --> provide --> VERB --> VBZ --> ROOT --> xxxx --> True --> False\n",
      "an --> an --> DET --> DT --> det --> xx --> True --> True\n",
      "important --> important --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "treatment --> treatment --> NOUN --> NN --> compound --> xxxx --> True --> False\n",
      "option --> option --> NOUN --> NN --> dobj --> xxxx --> True --> False\n",
      "for --> for --> ADP --> IN --> prep --> xxx --> True --> True\n",
      "patients --> patient --> NOUN --> NNS --> pobj --> xxxx --> True --> False\n",
      "with --> with --> ADP --> IN --> prep --> xxxx --> True --> True\n",
      "BRAF --> BRAF --> PROPN --> NNP --> pobj --> XXXX --> True --> False\n",
      "( --> ( --> PUNCT --> -LRB- --> punct --> ( --> False --> False\n",
      "V600 --> V600 --> PROPN --> NNP --> subtok --> Xddd --> False --> False\n",
      ") --> ) --> PUNCT --> -RRB- --> punct --> ) --> False --> False\n",
      "mutation --> mutation --> NOUN --> NN --> npadvmod --> xxxx --> True --> False\n",
      "- --> - --> PUNCT --> HYPH --> punct --> - --> False --> False\n",
      "positive --> positive --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "unresectable --> unresectable --> ADJ --> JJ --> amod --> xxxx --> True --> False\n",
      "or --> or --> CCONJ --> CC --> cc --> xx --> True --> True\n",
      "metastatic --> metastatic --> ADJ --> JJ --> conj --> xxxx --> True --> False\n",
      "melanoma --> melanoma --> NOUN --> NN --> appos --> xxxx --> True --> False\n",
      ". --> . --> PUNCT --> . --> punct --> . --> False --> False\n",
      "\n",
      " --> \n",
      " --> SPACE --> _SP -->  --> \n",
      " --> False --> False\n",
      "\n",
      "#---- Dependency Parsing ----#\n",
      "\n",
      "The BRAF inhibitor dabrafenib --> dabrafenib --> nsubjpass --> indicated\n",
      "(Tafinlar(Â® --> ® --> appos --> dabrafenib\n",
      "the MEK inhibitor trametinib --> trametinib --> conj --> dabrafenib\n",
      "Mekinist(Â® --> ® --> appos --> trametinib\n",
      "combination --> combination --> pobj --> in\n",
      "the treatment --> treatment --> pobj --> for\n",
      "patients --> patients --> pobj --> of\n",
      "unresectable or metastatic melanoma --> melanoma --> pobj --> with\n",
      "a BRAF (V600) mutation --> mutation --> pobj --> with\n",
      "This article --> article --> nsubj --> reviews\n",
      "the therapeutic efficacy --> efficacy --> dobj --> reviews\n",
      "tolerability --> tolerability --> conj --> efficacy\n",
      "combination treatment --> treatment --> pobj --> of\n",
      "dabrafenib --> dabrafenib --> pobj --> with\n",
      "trametinib --> trametinib --> conj --> dabrafenib\n",
      "this indication --> indication --> pobj --> in\n",
      "relevant pharmacological data --> data --> dobj --> summarizes\n",
      "Dabrafenib --> Dabrafenib --> nsubj --> improved\n",
      "trametinib --> trametinib --> conj --> Dabrafenib\n",
      "significantly prolonged progression-free survival --> survival --> conj --> Dabrafenib\n",
      "PFS --> PFS --> appos --> survival\n",
      "overall survival --> survival --> conj --> survival\n",
      "OS --> OS --> appos --> survival\n",
      "objective response rates --> rates --> dobj --> improved\n",
      "ORRs --> ORRs --> appos --> rates\n",
      "preserved health-related quality --> quality --> conj --> rates\n",
      "life --> life --> pobj --> of\n",
      "HR-QOL --> QOL --> appos --> quality\n",
      "a greater extent --> extent --> pobj --> to\n",
      "dabrafenib --> dabrafenib --> pobj --> than\n",
      "the double-blind COMBI-d study --> study --> pobj --> in\n",
      "vemurafenib --> vemurafenib --> conj --> extent\n",
      "the open-label COMBI-v study --> study --> pobj --> in\n",
      "two large, randomized, phase III studies --> studies --> pobj --> in\n",
      "treatment --> treatment --> pobj --> in\n",
      "patients --> patients --> dobj --> improved\n",
      "unresectable or metastatic melanoma --> melanoma --> pobj --> with\n",
      "BRAF (V600E/K) mutation --> mutation --> pobj --> with\n",
      "Limited treatment benefit --> benefit --> nsubjpass --> seen\n",
      "the combination --> combination --> pobj --> with\n",
      "patients --> patients --> pobj --> in\n",
      "who --> who --> nsubj --> progressed\n",
      "prior BRAF inhibitor therapy --> therapy --> pobj --> on\n",
      "ORRs --> ORRs --> pobj --> by\n",
      "â‰¤ 15Â % --> % --> pobj --> of\n",
      "stable disease --> disease --> conj --> ORRs\n",
      "50Â % --> % --> dobj --> indicated\n",
      "patients --> patients --> pobj --> of\n",
      "small phase I and II studies --> studies --> pobj --> in\n",
      "Combination therapy --> therapy --> nsubj --> increase\n",
      "overall toxicity --> toxicity --> dobj --> increase\n",
      "dabrafenib --> dabrafenib --> pobj --> to\n",
      "vemurafenib --> vemurafenib --> conj --> dabrafenib\n",
      "most adverse events --> events --> pobj --> with\n",
      "AEs --> AEs --> appos --> events\n",
      "severity --> severity --> pobj --> in\n",
      "Fewer skin-related AEs --> AEs --> nsubjpass --> reported\n",
      "(e.g. cutaneous malignancies --> malignancies --> appos --> AEs\n",
      "hyperkeratinosis --> hyperkeratinosis --> conj --> malignancies\n",
      "hand-foot syndrome --> syndrome --> conj --> hyperkeratinosis\n",
      "combination therapy --> therapy --> pobj --> with\n",
      "dabrafenib --> dabrafenib --> pobj --> with\n",
      "vemurafenib --> vemurafenib --> conj --> dabrafenib\n",
      "reduced paradoxical activation --> activation --> pobj --> because\n",
      "the MAPK pathway --> pathway --> pobj --> of\n",
      "dabrafenib --> dabrafenib --> nsubj --> provides\n",
      "trametinib --> trametinib --> conj --> dabrafenib\n",
      "an important treatment option --> option --> dobj --> provides\n",
      "patients --> patients --> pobj --> for\n",
      "BRAF --> BRAF --> pobj --> with\n",
      "V600) mutation-positive unresectable or metastatic melanoma --> melanoma --> appos --> BRAF\n",
      "\n",
      "#---- Entity using Spacy ----#\n",
      "\n",
      "BRAF --> gene\n",
      "dabrafenib --> drug\n",
      "MEK --> protien\n",
      "trametinib --> drug\n",
      "melanoma --> diesease\n",
      "BRAF --> gene\n",
      "V600 --> varient\n",
      "trametinib --> drug\n",
      "Dabrafenib --> drug\n",
      "trametinib --> drug\n",
      "melanoma --> diesease\n",
      "BRAF --> gene\n",
      "V600E/K --> varient\n",
      "vemurafenib --> drug\n",
      "vemurafenib --> drug\n",
      "trametinib --> drug\n",
      "BRAF --> gene\n",
      "V600 --> varient\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "melanoma --> diesease\n"
     ]
    }
   ],
   "source": [
    "# POS tagging\n",
    "print('#---- POS Tagging ----#''\\n')\n",
    "for token in nlp_doc:\n",
    "    print(token.text,\"-->\", token.lemma_,\"-->\", token.pos_,\"-->\", token.tag_,\"-->\", token.dep_,\"-->\",\n",
    "            token.shape_,\"-->\", token.is_alpha,\"-->\", token.is_stop)\n",
    "\n",
    "# Dependency Parsing\n",
    "print('\\n'\"#---- Dependency Parsing ----#\"'\\n')\n",
    "for chunk in nlp_doc.noun_chunks:\n",
    "    print(chunk.text,\"-->\", chunk.root.text,\"-->\", chunk.root.dep_,\"-->\",\n",
    "            chunk.root.head.text)\n",
    "    \n",
    "# Entity Extration\n",
    "print('\\n'\"#---- Entity using Spacy ----#\"'\\n')\n",
    "for ent in nlp_doc.ents:\n",
    "    print(ent.text,\"-->\",ent.label_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tobias --> PERSON\n",
      "’m --> PERSON\n",
      "2018 --> DATE\n"
     ]
    }
   ],
   "source": [
    "nlp = spacy.load(\"en_core_web_lg\")\n",
    "for ent in nlp(doc1).ents:\n",
    "    print(ent.text, \"-->\", ent.label_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Playing with Displacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "from spacy import displacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"/home/ubuntu/Grakn/Spacy Play/spacy_en_core_web_lg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = nlp(TEXTS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Custome NER Model Output using Displacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">The \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    BRAF\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">gene</span>\n",
       "</mark>\n",
       " inhibitor \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    dabrafenib\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">drug</span>\n",
       "</mark>\n",
       " (Tafinlar(Â®)) and the \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    MEK\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">protien</span>\n",
       "</mark>\n",
       " inhibitor \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    trametinib\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">drug</span>\n",
       "</mark>\n",
       " (Mekinist(Â®)) are indicated, as monotherapy or in combination with each other, for the treatment of patients with unresectable or metastatic \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    melanoma\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">diesease</span>\n",
       "</mark>\n",
       " with a \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    BRAF\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">gene</span>\n",
       "</mark>\n",
       " (\n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    V600\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">varient</span>\n",
       "</mark>\n",
       ") mutation. </br>This article reviews the therapeutic efficacy and tolerability of combination treatment with dabrafenib and \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    trametinib\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">drug</span>\n",
       "</mark>\n",
       " in this indication and summarizes relevant pharmacological data. </br>\n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    Dabrafenib\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">drug</span>\n",
       "</mark>\n",
       " plus \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    trametinib\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">drug</span>\n",
       "</mark>\n",
       " significantly prolonged progression-free survival (PFS) and overall survival (OS), improved objective response rates (ORRs) and preserved health-related quality of life (HR-QOL) to a greater extent than dabrafenib (in the double-blind COMBI-d study) and vemurafenib (in the open-label COMBI-v study) in two large, randomized, phase III studies in treatment-naÃ¯ve patients with unresectable or metastatic \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    melanoma\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">diesease</span>\n",
       "</mark>\n",
       " with \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    BRAF\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">gene</span>\n",
       "</mark>\n",
       " (\n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    V600E/K\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">varient</span>\n",
       "</mark>\n",
       ") mutation. Limited treatment benefit with the combination was also seen in patients who had progressed on prior BRAF inhibitor therapy, as indicated by ORRs of â‰¤ 15Â % and stable disease in â‰¤ 50Â % of patients in small phase I and II studies. Combination therapy did not increase overall toxicity relative to dabrafenib or \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    vemurafenib\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">drug</span>\n",
       "</mark>\n",
       " monotherapy, with most adverse events (AEs) mild or moderate in severity and generally manageable. Fewer skin-related AEs (e.g. cutaneous malignancies, hyperkeratinosis and hand-foot syndrome) were reported with combination therapy than with dabrafenib or \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    vemurafenib\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">drug</span>\n",
       "</mark>\n",
       ", probably because of reduced paradoxical activation of the MAPK pathway. Thus, dabrafenib plus \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    trametinib\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">drug</span>\n",
       "</mark>\n",
       " provides an important treatment option for patients with \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    BRAF\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">gene</span>\n",
       "</mark>\n",
       " (\n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    V600\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">varient</span>\n",
       "</mark>\n",
       ") mutation-positive unresectable or metastatic \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    melanoma\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">diesease</span>\n",
       "</mark>\n",
       ".</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "displacy.render(doc, style=\"ent\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pretrained NER Output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_web_lg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = nlp(TEXTS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">The \n",
       "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    BRAF\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
       "</mark>\n",
       " inhibitor dabrafenib (Tafinlar(Â®)) and the \n",
       "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    MEK\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
       "</mark>\n",
       " inhibitor trametinib (Mekinist(Â®)) are indicated, as monotherapy or in combination with each other, for the treatment of patients with unresectable or metastatic melanoma with a \n",
       "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    BRAF\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
       "</mark>\n",
       " (V600) mutation. </br>This article reviews the therapeutic efficacy and tolerability of combination treatment with dabrafenib and trametinib in this indication and summarizes relevant pharmacological data. </br>Dabrafenib plus trametinib significantly prolonged progression-free survival (\n",
       "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    PFS\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
       "</mark>\n",
       ") and overall survival (\n",
       "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    OS\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
       "</mark>\n",
       "), improved objective response rates (ORRs) and preserved health-related quality of life (HR-QOL) to a greater extent than dabrafenib (in the double-blind COMBI-d study) and vemurafenib (in the open-label COMBI-v study) in \n",
       "<mark class=\"entity\" style=\"background: #e4e7d2; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    two\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">CARDINAL</span>\n",
       "</mark>\n",
       " large, randomized, phase III studies in treatment-naÃ¯ve patients with unresectable or metastatic melanoma with \n",
       "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    BRAF\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
       "</mark>\n",
       " (V600E/K) mutation. Limited treatment benefit with the combination was also seen in patients who had progressed on prior \n",
       "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    BRAF\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
       "</mark>\n",
       " inhibitor therapy, as indicated by ORRs of â‰¤ \n",
       "<mark class=\"entity\" style=\"background: #e4e7d2; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    15Â %\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">PERCENT</span>\n",
       "</mark>\n",
       " and stable disease in â‰¤ \n",
       "<mark class=\"entity\" style=\"background: #e4e7d2; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    50Â %\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">PERCENT</span>\n",
       "</mark>\n",
       " of patients in small phase I and II studies. Combination therapy did not increase overall toxicity relative to dabrafenib or vemurafenib monotherapy, with most adverse events (AEs) mild or moderate in severity and generally manageable. Fewer skin-related AEs (e.g. cutaneous malignancies, hyperkeratinosis and hand-foot syndrome) were reported with combination therapy than with dabrafenib or vemurafenib, probably because of reduced paradoxical activation of the \n",
       "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    MAPK\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
       "</mark>\n",
       " pathway. Thus, dabrafenib plus trametinib provides an important treatment option for patients with \n",
       "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em; box-decoration-break: clone; -webkit-box-decoration-break: clone\">\n",
       "    BRAF\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
       "</mark>\n",
       " (V600) mutation-positive unresectable or metastatic melanoma.</div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "displacy.render(doc, style=\"ent\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Playing with Spacy - Entity Relation Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf8\n",
    "\"\"\"A simple example of extracting relations between phrases and entities using\n",
    "spaCy's named entity recognizer and the dependency parse. Here, we extract\n",
    "money and currency values (entities labelled as MONEY) and then check the\n",
    "dependency tree to find the noun phrase they are referring to – for example:\n",
    "$9.4 million --> Net income.\n",
    "\n",
    "Compatible with: spaCy v2.0.0+\n",
    "Last tested with: v2.1.0\n",
    "\"\"\"\n",
    "\n",
    "from __future__ import unicode_literals, print_function\n",
    "\n",
    "import plac\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXTS = [\n",
    "    \"Net income was $9.4 million compared to the prior year of $2.7 million.\",\n",
    "    \"Revenue exceeded twelve billion dollars, with a loss of $1b.\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXTS = '''The BRAF inhibitor dabrafenib (Tafinlar(Â®)) and the MEK inhibitor trametinib (Mekinist(Â®)) are indicated, as monotherapy or in combination with each other, for the treatment of patients with unresectable or metastatic melanoma with a BRAF (V600) mutation. \\nThis article reviews the therapeutic efficacy and tolerability of combination treatment with dabrafenib and trametinib in this indication and summarizes relevant pharmacological data. \\nDabrafenib plus trametinib significantly prolonged progression-free survival (PFS) and overall survival (OS), improved objective response rates (ORRs) and preserved health-related quality of life (HR-QOL) to a greater extent than dabrafenib (in the double-blind COMBI-d study) and vemurafenib (in the open-label COMBI-v study) in two large, randomized, phase III studies in treatment-naÃ¯ve patients with unresectable or metastatic melanoma with BRAF (V600E/K) mutation. Limited treatment benefit with the combination was also seen in patients who had progressed on prior BRAF inhibitor therapy, as indicated by ORRs of â‰¤ 15Â % and stable disease in â‰¤ 50Â % of patients in small phase I and II studies. Combination therapy did not increase overall toxicity relative to dabrafenib or vemurafenib monotherapy, with most adverse events (AEs) mild or moderate in severity and generally manageable. Fewer skin-related AEs (e.g. cutaneous malignancies, hyperkeratinosis and hand-foot syndrome) were reported with combination therapy than with dabrafenib or vemurafenib, probably because of reduced paradoxical activation of the MAPK pathway. Thus, dabrafenib plus trametinib provides an important treatment option for patients with BRAF (V600) mutation-positive unresectable or metastatic melanoma.'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "@plac.annotations(model=(\"Model to load (needs parser and NER)\", \"positional\", None, str))\n",
    "\n",
    "def main(model=\"en_core_web_lg\"): ##  \"/home/ubuntu/Grakn/Spacy Play/spacy_en_core_web_lg\"\n",
    "    nlp = spacy.load(model)\n",
    "    print(\"Loaded model '%s'\" % model)\n",
    "    print(\"Processing %d texts\" % len(TEXTS))\n",
    "\n",
    "    for text in TEXTS:\n",
    "        doc = nlp(text)\n",
    "        relations = extract_currency_relations(doc)\n",
    "        for r1, r2 in relations:\n",
    "            print(\"{:<10}\\t{}\\t{}\".format(r1.text, r2.ent_type_, r2.text))\n",
    "\n",
    "\n",
    "def filter_spans(spans):\n",
    "    # Filter a sequence of spans so they don't contain overlaps\n",
    "    get_sort_key = lambda span: (span.end - span.start, span.start)\n",
    "    sorted_spans = sorted(spans, key=get_sort_key, reverse=True)\n",
    "    result = []\n",
    "    seen_tokens = set()\n",
    "    for span in sorted_spans:\n",
    "        if span.start not in seen_tokens and span.end - 1 not in seen_tokens:\n",
    "            result.append(span)\n",
    "            seen_tokens.update(range(span.start, span.end))\n",
    "    return result\n",
    "\n",
    "\n",
    "def extract_currency_relations(doc):\n",
    "    # Merge entities and noun chunks into one token\n",
    "    seen_tokens = set()\n",
    "    spans = list(doc.ents) + list(doc.noun_chunks)\n",
    "    spans = filter_spans(spans)\n",
    "    with doc.retokenize() as retokenizer:\n",
    "        for span in spans:\n",
    "            retokenizer.merge(span)\n",
    "\n",
    "    relations = []\n",
    "    for money in filter(lambda w: w.ent_type_ ==\"MONEY\" , doc): ##   \"gene\"                      \n",
    "        if money.dep_ in (\"attr\", \"dobj\"):\n",
    "            subject = [w for w in money.head.lefts if w.dep_ == \"nsubj\"]\n",
    "            if subject:\n",
    "                subject = subject[0]\n",
    "                relations.append((subject, money))\n",
    "        elif money.dep_ == \"pobj\" and money.head.dep_ == \"prep\":\n",
    "            relations.append((money.head.head, money))\n",
    "    return relations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    plac.call(main)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}