initial-notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: bert-tensorflow in /usr/local/lib/python3.6/dist-packages (1.0.1)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from bert-tensorflow) (1.14.0)\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (1.0.4)\n",
      "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas) (2.8.1)\n",
      "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas) (2020.1)\n",
      "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from pandas) (1.18.4)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.6.1->pandas) (1.14.0)\n",
      "Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (2.11.0)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.4)\n",
      "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.43)\n",
      "Requirement already satisfied: tokenizers==0.7.0 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7.0)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.46.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2020.6.8)\n",
      "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (from transformers) (0.1.91)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
      "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.15.1)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.14.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/lib/python3/dist-packages (from requests->transformers) (2.6)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.4.5.1)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.25.9)\n"
     ]
    }
   ],
   "source": [
    "!pip install bert-tensorflow\n",
    "!pip install pandas\n",
    "!pip install transformers # https://huggingface.co/transformers/installation.html#with-pip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensorflow version: 2.1.1\n",
      "Num GPUs Available: 1\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Dense, Input\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "\n",
    "import transformers\n",
    "from tqdm.notebook import tqdm\n",
    "from tokenizers import BertWordPieceTokenizer\n",
    "\n",
    "print(f\"Tensorflow version: {tf.version.VERSION}\")\n",
    "print(f\"Num GPUs Available: {len(tf.config.experimental.list_physical_devices('GPU'))}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH =  \"./toxicity-detection/jigsaw-multilingual-toxic-comment-classification\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data and balance toxicity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train1 = pd.read_csv(os.path.join(DATA_PATH, 'jigsaw-toxic-comment-train.csv'))\n",
    "train2 = pd.read_csv(os.path.join(DATA_PATH, 'jigsaw-unintended-bias-train.csv'))\n",
    "train2.toxic = train2.toxic.round().astype(int)\n",
    "\n",
    "valid = pd.read_csv(os.path.join(DATA_PATH, 'validation.csv'))\n",
    "test = pd.read_csv(os.path.join(DATA_PATH, 'test.csv'))\n",
    "sub = pd.read_csv(os.path.join(DATA_PATH, 'sample_submission.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train1 toxic comments 21384\n",
      "Train1 non-toxic comments 202165\n",
      "Proportion train1 toxic comments 9.565688059441108\n",
      "Proportion train2 non-toxic comments 90.4343119405589\n",
      "\n",
      "Train2 toxic comments 112226\n",
      "Train2 non-toxic comments 1789968\n",
      "Proportion train2 toxic comments 5.899818840770185\n",
      "Proportion train2 non-toxic comments 94.1001811592298\n"
     ]
    }
   ],
   "source": [
    "print(f\"Train1 toxic comments {len(train1[['comment_text', 'toxic']].query('toxic==1'))}\")\n",
    "print(f\"Train1 non-toxic comments {len(train1[['comment_text', 'toxic']].query('toxic==0'))}\")\n",
    "print(f\"Proportion train1 toxic comments {len(train1[['comment_text', 'toxic']].query('toxic==1')) / len(train1[['comment_text', 'toxic']]) * 100}\")\n",
    "print(f\"Proportion train2 non-toxic comments {len(train1[['comment_text', 'toxic']].query('toxic==0')) / len(train1[['comment_text', 'toxic']]) * 100}\\n\")\n",
    "\n",
    "print(f\"Train2 toxic comments {len(train2[['comment_text', 'toxic']].query('toxic==1'))}\")\n",
    "print(f\"Train2 non-toxic comments {len(train2[['comment_text', 'toxic']].query('toxic==0'))}\")\n",
    "print(f\"Proportion train2 toxic comments {len(train2[['comment_text', 'toxic']].query('toxic==1')) / len(train2[['comment_text', 'toxic']]) * 100}\")\n",
    "print(f\"Proportion train2 non-toxic comments {len(train2[['comment_text', 'toxic']].query('toxic==0')) / len(train2[['comment_text', 'toxic']]) * 100}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Downsample the train dataframe to 50% - 50%\n",
    "train = pd.concat([\n",
    "    train1[['comment_text', 'toxic']].query('toxic==1'),\n",
    "    train1[['comment_text', 'toxic']].query('toxic==0').sample(n = 25000, random_state = 0),\n",
    "    train2[['comment_text', 'toxic']].query('toxic==1'),\n",
    "    train2[['comment_text', 'toxic']].query('toxic==0').sample(n = 150000, random_state = 0)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "308610\n",
      "Final dataset toxic proportion 43.294125271378114\n",
      "Final dataset non-toxic proportion 56.705874728621886\n"
     ]
    }
   ],
   "source": [
    "print(len(train))\n",
    "print(f\"Final dataset toxic proportion {len(train[['comment_text', 'toxic']].query('toxic==1')) / len(train[['comment_text', 'toxic']]) * 100}\")\n",
    "print(f\"Final dataset non-toxic proportion {len(train[['comment_text', 'toxic']].query('toxic==0')) / len(train[['comment_text', 'toxic']]) * 100}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: numba in /usr/local/lib/python3.6/dist-packages (0.50.0)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from numba) (46.4.0)\n",
      "Requirement already satisfied: numpy>=1.15 in /usr/local/lib/python3.6/dist-packages (from numba) (1.18.4)\n",
      "Requirement already satisfied: llvmlite<0.34,>=0.33.0.dev0 in /usr/local/lib/python3.6/dist-packages (from numba) (0.33.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install numba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numba\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "@numba.jit()\n",
    "def fast_encode(texts, tokenizer, chunk_size = 256, maxlen = 512):\n",
    "    \n",
    "    # Maximum sequence size for BERT is 512, \n",
    "    # so we wll truncate any comment that is longer than this.\n",
    "    tokenizer.enable_truncation(max_length = maxlen)\n",
    "    \n",
    "    # Finally, we need to pad our input so it will have the \n",
    "    # same size of 512. It means that for any comment that is \n",
    "    # shorter than 512 tokens, we wll add zeros to reach 512 tokens.\n",
    "    tokenizer.enable_padding(max_length = maxlen)\n",
    "    \n",
    "    all_ids = []   \n",
    "    # tqdm progress bar: len(texts) // chunk_size\n",
    "    for i in tqdm(range(0, len(texts), chunk_size)):\n",
    "        text_chunk = texts[i:i + chunk_size].tolist()\n",
    "        # Tokenize current text chunk\n",
    "        encs = tokenizer.encode_batch(text_chunk)\n",
    "        # Extending the list is squeezing the list\n",
    "        all_ids.extend([enc.ids for enc in encs])\n",
    "    \n",
    "    return np.array(all_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tokenization for xlm (does not support truncation)\n",
    "@numba.jit()\n",
    "def encode(texts, tokenizer, maxlen = 512):\n",
    "    \n",
    "    enc_dict = tokenizer.batch_encode_plus(\n",
    "        texts, \n",
    "        return_attention_masks = False,\n",
    "        return_token_type_ids = False,\n",
    "        pad_to_max_length = True,\n",
    "        max_length = maxlen\n",
    "    )\n",
    "    \n",
    "    return np.array(enc_dict['input_ids'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "distilbert\n",
      "xlm\n",
      "xlm-roberta\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('./toxicity-detection/tokenizers/xlmroberta/vocab.txt',\n",
       " './toxicity-detection/tokenizers/xlmroberta/special_tokens_map.json',\n",
       " './toxicity-detection/tokenizers/xlmroberta/added_tokens.json')"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "models = []\n",
    "\n",
    "# mBERT\n",
    "distilbert_tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')\n",
    "models.append(distilbert_tokenizer)\n",
    "print('distilbert')\n",
    "# Save the loaded tokenizer locally\n",
    "! mkdir toxicity-detection/tokenizers/distilbert\n",
    "distilbert_model.save_pretrained('./toxicity-detection/tokenizers/distilbert')\n",
    "\n",
    "# XLM\n",
    "xlm_tokenizer = transformers.AutoTokenizer.from_pretrained('xlm-mlm-100-1280')\n",
    "models.append(xlm_tokenizer)\n",
    "print('xlm')\n",
    "# Save the loaded tokenizer locally\n",
    "! mkdir toxicity-detection/tokenizers/xlm\n",
    "distilbert_model.save_pretrained('./toxicity-detection/tokenizers/xlm')\n",
    "\n",
    "# XLM-RoBERTa\n",
    "xlm_roberta_tokenizer = transformers.AutoTokenizer.from_pretrained('xlm-roberta-large')\n",
    "models.append(xlm_roberta_tokenizer)\n",
    "print('xlm-roberta')\n",
    "# Save the loaded tokenizer locally\n",
    "! mkdir toxicity-detection/tokenizers/xlmroberta\n",
    "distilbert_model.save_pretrained('./toxicity-detection/tokenizers/xlmroberta')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('./test/vocab.json',\n",
       " './test/merges.txt',\n",
       " './test/special_tokens_map.json',\n",
       " './test/added_tokens.json')"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "! mkdir test\n",
    "transformers.AutoTokenizer.from_pretrained('xlm-mlm-100-1280')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4 µs, sys: 0 ns, total: 4 µs\n",
      "Wall time: 6.91 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# 17.3 s\n",
    "# x_train = fast_encode(train.comment_text.astype(str), fast_tokenizer, maxlen = MAX_LEN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 0 ns, sys: 3 µs, total: 3 µs\n",
      "Wall time: 6.44 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# 16.8 s\n",
    "# x_train = fast_encode(train.comment_text.astype(str), fast_tokenizer, maxlen = MAX_LEN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7686cff960a24c758b8052745d4778cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1206.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2dedb14d310a45f88f985aefd160be31",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=32.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13cd2e60b53e45a4baebc0de8e411738",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=250.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Configuration\n",
    "MAX_LEN = 256\n",
    "\n",
    "# Reload it with the huggingface tokenizers library\n",
    "tokenizer = BertWordPieceTokenizer('./toxicity-detection/tokenizers/distilbert/vocab.txt', lowercase = False)\n",
    "\n",
    "x_train = fast_encode(train.comment_text.astype(str), tokenizer, maxlen = MAX_LEN)\n",
    "x_valid = fast_encode(valid.comment_text.astype(str), tokenizer, maxlen = MAX_LEN)\n",
    "x_test = fast_encode(test.content.astype(str), tokenizer, maxlen = MAX_LEN)\n",
    "\n",
    "y_train = train.toxic.values\n",
    "y_valid = valid.toxic.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 16\n",
    "\n",
    "# Prepare train dataset\n",
    "train_dataset = (\n",
    "    tf.data.Dataset\n",
    "    .from_tensor_slices((x_train, y_train))\n",
    "    .repeat()\n",
    "    .shuffle(2048)\n",
    "    .batch(BATCH_SIZE)\n",
    ")\n",
    "\n",
    "# Prepare validation dataset\n",
    "valid_dataset = (\n",
    "    tf.data.Dataset\n",
    "    .from_tensor_slices((x_valid, y_valid))\n",
    "    .batch(BATCH_SIZE)\n",
    "    .cache()\n",
    ")\n",
    "\n",
    "# Prepare test dataset\n",
    "test_dataset = (\n",
    "    tf.data.Dataset\n",
    "    .from_tensor_slices(x_test)\n",
    "    .batch(BATCH_SIZE)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optimizer\n",
    "def set_optimizer(type = 'adam', lr = 1e-5):\n",
    "    if type == 'adam':\n",
    "        return Adam(lr = lr)\n",
    "    elif type == 'adadelta':\n",
    "        return Adadelta(lr = lr)\n",
    "    else: \n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build model\n",
    "def build_model(transformer, optimizer = set_optimizer('adam'), dropout_prop = 0.35, \n",
    "                loss = 'binary_crossentropy', max_len = 512):\n",
    "    # Instantiate tensor with input word IDs\n",
    "    input_word_ids = Input(shape = (max_len,), dtype = tf.int32, name ='input_word_ids')\n",
    "    \n",
    "    # Forward through transformer network\n",
    "    # Hugginface transformers have multiple outputs, embeddings are at the first one\n",
    "    sequence_output = transformer(input_word_ids)[0]\n",
    "    # We slice the first position, the paper says it is not worse than pooling\n",
    "    x = sequence_output[:, 0, :]\n",
    "    \n",
    "    x = Dense(512, activation = 'relu')(x)\n",
    "    \n",
    "    # Apply dropout to dense layer\n",
    "    x = tf.keras.layers.Dropout(dropout_prop)(x)\n",
    "    x = Dense(256, activation = 'relu')(x)\n",
    "    \n",
    "    # Apply dropout to dense layer\n",
    "    x = tf.keras.layers.Dropout(dropout_prop)(x)\n",
    "    x = Dense(128, activation = 'relu')(x)\n",
    "    \n",
    "    # Apply dropout to dense layer\n",
    "    x = tf.keras.layers.Dropout(dropout_prop)(x)\n",
    "    x = Dense(64, activation = 'relu')(x)\n",
    "    \n",
    "    # Apply dropout to dense layer\n",
    "    x = tf.keras.layers.Dropout(dropout_prop)(x)\n",
    "    x = Dense(32, activation = 'relu')(x)\n",
    "    \n",
    "    # Apply dropout to dense layer and final layer to ouput toxicity probability\n",
    "    x = tf.keras.layers.Dropout(dropout_prop)(x)\n",
    "    out = Dense(1, activation = 'sigmoid', name = 'toxicity_prob_head')(x)\n",
    "    \n",
    "    model = Model(inputs = input_word_ids, outputs = out)\n",
    "    model.compile(optimizer, loss = loss, metrics = ['accuracy', tf.keras.metrics.AUC()])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperparameters\n",
    "EPOCHS = 5\n",
    "BATCH_SIZE = BATCH_SIZE \n",
    "OPTIMIZER = 'adam'\n",
    "LR = 0.0001\n",
    "DROPOUT = 0.35"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a503fa620794b68bae5def44c363d7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51b510f6d7a748f7a1b042cd3e96a82d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=910749124.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_word_ids (InputLayer)  [(None, 256)]             0         \n",
      "_________________________________________________________________\n",
      "tf_distil_bert_model (TFDist ((None, 256, 768),)       134734080 \n",
      "_________________________________________________________________\n",
      "tf_op_layer_strided_slice (T [(None, 768)]             0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 512)               393728    \n",
      "_________________________________________________________________\n",
      "dropout_19 (Dropout)         (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "dropout_20 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dropout_21 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dropout_22 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 32)                2080      \n",
      "_________________________________________________________________\n",
      "dropout_23 (Dropout)         (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "toxicity_prob_head (Dense)   (None, 1)                 33        \n",
      "=================================================================\n",
      "Total params: 135,302,401\n",
      "Trainable params: 135,302,401\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "transformer_layer = (\n",
    "    transformers.TFDistilBertModel\n",
    "    .from_pretrained('distilbert-base-multilingual-cased')\n",
    ")\n",
    "\n",
    "model = build_model(transformer_layer, optimizer = set_optimizer(OPTIMIZER, lr = LR), dropout_prop = DROPOUT, max_len = MAX_LEN)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint_filepath = f'./toxicity-detection/models/checkpoints'\n",
    "model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n",
    "    filepath = checkpoint_filepath,\n",
    "    save_weights_only = True,\n",
    "    monitor = 'loss',\n",
    "    mode = 'max',\n",
    "    save_best_only = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.ModelCheckpoint at 0x7f0f3b40f320>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_checkpoint_callback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train for 19288 steps, validate for 500 steps\n",
      "Epoch 1/5\n",
      "18874/19288 [============================>.] - ETA: 3:50 - loss: 0.0423 - accuracy: 0.9837 - auc: 0.9986"
     ]
    }
   ],
   "source": [
    "# TRAIN!\n",
    "n_steps = x_train.shape[0] // BATCH_SIZE\n",
    "start = time.time()\n",
    "train_history = model.fit(\n",
    "    train_dataset,\n",
    "    steps_per_epoch = n_steps,\n",
    "    validation_data = valid_dataset,\n",
    "    epochs = EPOCHS,\n",
    "    callbacks = [model_checkpoint_callback],\n",
    "    verbose = 1\n",
    ")\n",
    "training_time = time.time() - start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'loss': [0.046408504734211396,\n",
       "  0.05378133220449028,\n",
       "  0.053325059956694114,\n",
       "  0.05243607540994543,\n",
       "  0.05120615940909028],\n",
       " 'accuracy': [0.98176324, 0.9805773, 0.98103094, 0.98106986, 0.98087543],\n",
       " 'auc': [0.99837035, 0.99799216, 0.9980141, 0.9980736, 0.99809664],\n",
       " 'val_loss': [0.8810894889831543,\n",
       "  0.8000532178878784,\n",
       "  0.7619827425479889,\n",
       "  0.7598212015628815,\n",
       "  0.7141085361242294],\n",
       " 'val_accuracy': [0.15375, 0.15375, 0.15375, 0.15375, 0.15375],\n",
       " 'val_auc': [0.5, 0.5, 0.5, 0.5, 0.5]}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_history.history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "! echo \"$training_time\" > ./toxicity-detection/models/distilbert/train.log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average train loss:  0.0514314263428863\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(14.959554066260656, None)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_time/3600, print(\"Average train loss: \", np.average(train_history.history['loss']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train for 500 steps\n",
      "Epoch 1/10\n",
      "500/500 [==============================] - 277s 553ms/step - loss: 0.4625 - accuracy: 0.8384 - auc: 0.5065\n",
      "Epoch 2/10\n",
      "500/500 [==============================] - 278s 556ms/step - loss: 0.4540 - accuracy: 0.8462 - auc: 0.4959\n",
      "Epoch 3/10\n",
      "500/500 [==============================] - 278s 556ms/step - loss: 0.4508 - accuracy: 0.8462 - auc: 0.5037\n",
      "Epoch 4/10\n",
      "500/500 [==============================] - 278s 557ms/step - loss: 0.4529 - accuracy: 0.8462 - auc: 0.4927\n",
      "Epoch 5/10\n",
      "500/500 [==============================] - 278s 556ms/step - loss: 0.4485 - accuracy: 0.8462 - auc: 0.5063\n",
      "Epoch 6/10\n",
      "500/500 [==============================] - 278s 556ms/step - loss: 0.4520 - accuracy: 0.8462 - auc: 0.4907\n",
      "Epoch 7/10\n",
      "500/500 [==============================] - 278s 556ms/step - loss: 0.4490 - accuracy: 0.8462 - auc: 0.4969\n",
      "Epoch 8/10\n",
      "500/500 [==============================] - 278s 556ms/step - loss: 0.4509 - accuracy: 0.8462 - auc: 0.4834\n",
      "Epoch 9/10\n",
      "500/500 [==============================] - 278s 557ms/step - loss: 0.4495 - accuracy: 0.8462 - auc: 0.4863\n",
      "Epoch 10/10\n",
      "500/500 [==============================] - 278s 557ms/step - loss: 0.4478 - accuracy: 0.8462 - auc: 0.4942\n"
     ]
    }
   ],
   "source": [
    "# VALIDATE!\n",
    "start = time.time()\n",
    "n_steps = x_valid.shape[0] // BATCH_SIZE\n",
    "train_history_2 = model.fit(\n",
    "    valid_dataset.repeat(),\n",
    "    steps_per_epoch = n_steps,\n",
    "    epochs = EPOCHS * 2\n",
    ")\n",
    "validation_time = time.time() - start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "! echo \"$validation_time\" > ./toxicity-detection/models/distilbert/validate.log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training time: 53854.39463853836\n",
      "Average train accuracy: 0.9810633659362793\n",
      "Validation time: 2780.5180530548096\n",
      "Average validation accuracy: 0.8454625010490417\n"
     ]
    }
   ],
   "source": [
    "print(f\"Training time: {training_time}\")\n",
    "print(f\"Average train accuracy: {np.average(train_history.history['accuracy'])}\")\n",
    "print(f\"Validation time: {validation_time}\")\n",
    "print(f\"Average validation accuracy: {np.average(train_history_2.history['accuracy'])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start saving\n",
      "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f0f3b4cfbe0>, because it is not built.\n",
      "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f0f3b4d2400>, because it is not built.\n",
      "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f0f3b4a0898>, because it is not built.\n",
      "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f0f3b4b3550>, because it is not built.\n",
      "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f0f3b56c278>, because it is not built.\n",
      "WARNING:tensorflow:Skipping full serialization of Keras layer <tensorflow.python.keras.layers.core.Dropout object at 0x7f0f3b4feb70>, because it is not built.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/ops/resource_variable_ops.py:1786: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "INFO:tensorflow:Assets written to: ./toxicity-detection/models/distilbert/deep_distilbert_batch16_epochs5_maxlen256/assets\n",
      "End saving\n"
     ]
    }
   ],
   "source": [
    "# !mkdir -p models\n",
    "print('Start saving')\n",
    "model.save(f'./toxicity-detection/models/distilbert/deep_distilbert_batch{BATCH_SIZE}_epochs{EPOCHS}_maxlen{MAX_LEN}')\n",
    "print('End saving')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub['toxic'] = model.predict(test_dataset, verbose = 1)\n",
    "sub.to_csv('submission.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "simple = f'./toxicity-detection/models/distilbert/distilbert_batch16_epochs3_maxlen192'\n",
    "deep = f'./toxicity-detection/models/distilbert/deep_distilbert_batch16_epochs5_maxlen256'\n",
    "simple_model = tf.keras.models.load_model(simple)\n",
    "deep_model = tf.keras.models.load_model(deep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_word_ids (InputLayer)  [(None, 192)]             0         \n",
      "_________________________________________________________________\n",
      "tf_distil_bert_model (TFDist multiple                  134734080 \n",
      "_________________________________________________________________\n",
      "tf_op_layer_strided_slice (T multiple                  0         \n",
      "_________________________________________________________________\n",
      "dropout_19 (Dropout)         multiple                  0         \n",
      "_________________________________________________________________\n",
      "toxicity_prob_head (Dense)   multiple                  769       \n",
      "=================================================================\n",
      "Total params: 134,734,849\n",
      "Trainable params: 134,734,849\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "simple_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_word_ids (InputLayer)  [(None, 256)]             0         \n",
      "_________________________________________________________________\n",
      "tf_distil_bert_model (TFDist multiple                  134734080 \n",
      "_________________________________________________________________\n",
      "tf_op_layer_strided_slice (T multiple                  0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                multiple                  393728    \n",
      "_________________________________________________________________\n",
      "dropout_19 (Dropout)         multiple                  0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              multiple                  131328    \n",
      "_________________________________________________________________\n",
      "dropout_20 (Dropout)         multiple                  0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              multiple                  32896     \n",
      "_________________________________________________________________\n",
      "dropout_21 (Dropout)         multiple                  0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              multiple                  8256      \n",
      "_________________________________________________________________\n",
      "dropout_22 (Dropout)         multiple                  0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              multiple                  2080      \n",
      "_________________________________________________________________\n",
      "dropout_23 (Dropout)         multiple                  0         \n",
      "_________________________________________________________________\n",
      "toxicity_prob_head (Dense)   multiple                  33        \n",
      "=================================================================\n",
      "Total params: 135,302,401\n",
      "Trainable params: 135,302,401\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "deep_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Tokenizer(vocabulary_size=119547, model=BertWordPiece, unk_token=[UNK], sep_token=[SEP], cls_token=[CLS], pad_token=[PAD], mask_token=[MASK], clean_text=True, handle_chinese_chars=True, strip_accents=True, lowercase=False, wordpieces_prefix=##)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def toxic(text, maxlen, model):\n",
    "    word = pd.DataFrame(data = {'content': [text]})\n",
    "    word_test = fast_encode(word.content.astype(str), tokenizer, maxlen = maxlen)\n",
    "    word_test_dataset = (\n",
    "        tf.data.Dataset\n",
    "        .from_tensor_slices(word_test)\n",
    "        .batch(BATCH_SIZE)\n",
    "    )\n",
    "    pred = model.predict(word_test_dataset, verbose = 1)\n",
    "    return f'Toxic {np.round(pred[0][0] * 100, 3)}%'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7300f347ccf4f6c8eba31199fb5c82a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1/1 [==============================] - 0s 46ms/step\n",
      "Toxic 12.497%\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e077cc54ea844e0c8669bd2a4c524ccb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "1/1 [==============================] - 0s 23ms/step\n",
      "Toxic 26.063%\n"
     ]
    }
   ],
   "source": [
    "text = 'hi'\n",
    "print(toxic(text, maxlen = 192, model = simple_model))\n",
    "print(toxic(text, maxlen = MAX_LEN, model = deep_model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>content</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Doctor Who adlı viki başlığına 12. doctor olar...</td>\n",
       "      <td>tr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Вполне возможно, но я пока не вижу необходимо...</td>\n",
       "      <td>ru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Quindi tu sei uno di quelli   conservativi  , ...</td>\n",
       "      <td>it</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Malesef gerçekleştirilmedi ancak şöyle bir şey...</td>\n",
       "      <td>tr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>:Resim:Seldabagcan.jpg resminde kaynak sorunu ...</td>\n",
       "      <td>tr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63807</th>\n",
       "      <td>63807</td>\n",
       "      <td>No, non risponderò, come preannunciato. Prefer...</td>\n",
       "      <td>it</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63808</th>\n",
       "      <td>63808</td>\n",
       "      <td>Ciao, I tecnici della Wikimedia Foundation sta...</td>\n",
       "      <td>it</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63809</th>\n",
       "      <td>63809</td>\n",
       "      <td>innnazitutto ti ringrazio per i ringraziamenti...</td>\n",
       "      <td>it</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63810</th>\n",
       "      <td>63810</td>\n",
       "      <td>Kaç olumlu oy gerekiyor? Şu an 7 oldu.  Hayır...</td>\n",
       "      <td>tr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63811</th>\n",
       "      <td>63811</td>\n",
       "      <td>Te pido disculpas. La verdad es que no me per...</td>\n",
       "      <td>es</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>63812 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                                            content lang\n",
       "0          0  Doctor Who adlı viki başlığına 12. doctor olar...   tr\n",
       "1          1   Вполне возможно, но я пока не вижу необходимо...   ru\n",
       "2          2  Quindi tu sei uno di quelli   conservativi  , ...   it\n",
       "3          3  Malesef gerçekleştirilmedi ancak şöyle bir şey...   tr\n",
       "4          4  :Resim:Seldabagcan.jpg resminde kaynak sorunu ...   tr\n",
       "...      ...                                                ...  ...\n",
       "63807  63807  No, non risponderò, come preannunciato. Prefer...   it\n",
       "63808  63808  Ciao, I tecnici della Wikimedia Foundation sta...   it\n",
       "63809  63809  innnazitutto ti ringrazio per i ringraziamenti...   it\n",
       "63810  63810   Kaç olumlu oy gerekiyor? Şu an 7 oldu.  Hayır...   tr\n",
       "63811  63811   Te pido disculpas. La verdad es que no me per...   es\n",
       "\n",
       "[63812 rows x 3 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3989/3989 [==============================] - 505s 127ms/step\n"
     ]
    }
   ],
   "source": [
    "model.predict(test_dataset, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Вот скажите мне дураку! Растолкуйте! ЧТО ЖЕ ВЫ ЗА МУЖИК ТО ТАКОЙ??? Что кроме Ваших модераторов никто неимеет права создавать статьи в Википедии??? Зачем была удалена статья о Алине Кизияровой??? Я хотел ещё раньше создать статью о сериале Дальнобойщики-4 у меня тоже удаляли Ваши коллеги! Наверняка Вам много платят с удовольствием бы поработал как Вы! И Легко и хорошооплачиваемо! Вот это мечта быть на Вашем месте!!! Но Вы мне растолкуете неужели нельзя никому кроме модераторов ничего создавать тут??? Денис-Кошкарёв '"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lang = 'ru'\n",
    "test_try = pd.concat((test[test.lang == lang].content, sub.toxic[test.lang == lang]), axis = 1)\n",
    "test_try[test_try.toxic > 0.99].content.values[6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19045</th>\n",
       "      <td>Marcus Qwertyus \\n\\n Your a freeking homosexua...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29398</th>\n",
       "      <td>==Please read unblock request carefully\\n\\nThi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40763</th>\n",
       "      <td>Sorry SITUSH, there is no contact us section o...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68580</th>\n",
       "      <td>Where do you live, you little shit? \\n\\nGive u...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86945</th>\n",
       "      <td>You seem to have something against me. If you ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89003</th>\n",
       "      <td>Thank you for being such a politically correct...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97667</th>\n",
       "      <td>\"\\n\\nSo I will start with a criticism of the q...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105685</th>\n",
       "      <td>\"\\n\\nElon refers to BFTS as Big FALCON Test St...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120380</th>\n",
       "      <td>\"\\n\\nDiscuss this topic\\nPlease type the follo...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121335</th>\n",
       "      <td>You're dead \\n\\nIt won't be hard finding your ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123186</th>\n",
       "      <td>I'm NOT threatening you\\n\\nI'm just joking aro...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155740</th>\n",
       "      <td>Interested to your address  \\n\\nAm I correct i...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181643</th>\n",
       "      <td>::::Give me your address and I'll forward it t...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196007</th>\n",
       "      <td>== Please give me your address == \\n\\n So I ca...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205910</th>\n",
       "      <td>\" \\n ::Excuse me for butting in here. I am wha...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140029</th>\n",
       "      <td>Well could you give me your address? I would b...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1234777</th>\n",
       "      <td>Give me your address and I'll come over and to...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288807</th>\n",
       "      <td>It's a pleasure to watch you post Roy, I can't...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1251387</th>\n",
       "      <td>You should have included your address in the p...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1377912</th>\n",
       "      <td>With approval of the city. Maybe you can provi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120271</th>\n",
       "      <td>whew! Forget the phone number. Send me your ad...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>827227</th>\n",
       "      <td>The Social Security Administration database ha...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398268</th>\n",
       "      <td>My bet was against Mr. Trump's election.  I've...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              comment_text  toxic\n",
       "19045    Marcus Qwertyus \\n\\n Your a freeking homosexua...      1\n",
       "29398    ==Please read unblock request carefully\\n\\nThi...      0\n",
       "40763    Sorry SITUSH, there is no contact us section o...      0\n",
       "68580    Where do you live, you little shit? \\n\\nGive u...      1\n",
       "86945    You seem to have something against me. If you ...      1\n",
       "89003    Thank you for being such a politically correct...      1\n",
       "97667    \"\\n\\nSo I will start with a criticism of the q...      0\n",
       "105685   \"\\n\\nElon refers to BFTS as Big FALCON Test St...      0\n",
       "120380   \"\\n\\nDiscuss this topic\\nPlease type the follo...      0\n",
       "121335   You're dead \\n\\nIt won't be hard finding your ...      1\n",
       "123186   I'm NOT threatening you\\n\\nI'm just joking aro...      0\n",
       "155740   Interested to your address  \\n\\nAm I correct i...      0\n",
       "181643   ::::Give me your address and I'll forward it t...      0\n",
       "196007   == Please give me your address == \\n\\n So I ca...      0\n",
       "205910   \" \\n ::Excuse me for butting in here. I am wha...      0\n",
       "1140029  Well could you give me your address? I would b...      1\n",
       "1234777  Give me your address and I'll come over and to...      1\n",
       "288807   It's a pleasure to watch you post Roy, I can't...      0\n",
       "1251387  You should have included your address in the p...      0\n",
       "1377912  With approval of the city. Maybe you can provi...      0\n",
       "120271   whew! Forget the phone number. Send me your ad...      0\n",
       "827227   The Social Security Administration database ha...      0\n",
       "398268   My bet was against Mr. Trump's election.  I've...      0"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "#train[re.search('not prepared',train.comment_text)]\n",
    "train[train.comment_text.str.contains(\"your address\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[\"I'm not prepared for you, I promise you 13 of January \\n\\n                         /´¯/) \\n                      ,/¯  // \\n                     /    / / \\n             /´¯/'   '/´¯¯`•¸ \\n          /'/   /    /       /¨¯\\\\ \\n        ('(   ´(  ´      ,~/'   ') \\n         \\\\                 \\\\/    / \\n             \\\\           _ •´ \\n            \\\\              ( \\n              \\\\             \\\\\\n\\nVersion 2\\n                         /´¯/) \\n                      ,/¯  // \\n                     /    / / \\n             /´¯/'   '/´¯¯`•¸ \\n          /'/   /    /       /¨¯\\\\ \\n        ('(   ´(  ´      ,~/'   ') \\n         \\\\                 \\\\/    / \\n             \\\\           _ •´ \\n            \\\\              ( \\n              \\\\             \\\\\",\n",
       "        0],\n",
       "       ['My plan A is to see this collection of fools and their bedfellows in the other party dumped and replaced by people who appreciate the needs of working Alaskans in the cities and villages and have a different priority than protecting the Alaskan tax haven and giveaways to the oil companies.',\n",
       "        1]], dtype=object)"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[27288].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(text, lang='en'):\n",
    "    text = str(text)\n",
    "    text = re.sub(r'[0-9\"]', '', text)\n",
    "    text = re.sub(r'#[\\S]+\\b', '', text)\n",
    "    text = re.sub(r'@[\\S]+\\b', '', text)\n",
    "    text = re.sub(r'https?\\S+', '', text)\n",
    "    text = re.sub(r'\\s+', ' ', text)\n",
    "    return text.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"[[I'm not prepared for you, I promise you of January \\\\n\\\\n /´¯/) \\\\n ,/¯ // \\\\n / / / \\\\n /´¯/' '/´¯¯`•¸ \\\\n /'/ / / /¨¯\\\\\\\\ \\\\n ('( ´( ´ ,~/' ') \\\\n \\\\\\\\ \\\\\\\\/ / \\\\n \\\\\\\\ _ •´ \\\\n \\\\\\\\ ( \\\\n \\\\\\\\ \\\\\\\\\\\\n\\\\nVersion \\\\n /´¯/) \\\\n ,/¯ // \\\\n / / / \\\\n /´¯/' '/´¯¯`•¸ \\\\n /'/ / / /¨¯\\\\\\\\ \\\\n ('( ´( ´ ,~/' ') \\\\n \\\\\\\\ \\\\\\\\/ / \\\\n \\\\\\\\ _ •´ \\\\n \\\\\\\\ ( \\\\n \\\\\\\\ \\\\\\\\ ] ['My plan A is to see this collection of fools and their bedfellows in the other party dumped and replaced by people who appreciate the needs of working Alaskans in the cities and villages and have a different priority than protecting the Alaskan tax haven and giveaways to the oil companies.' ]]\""
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_text(train.loc[27288].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}