NLP Practice.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NLTK Downloader\n",
      "---------------------------------------------------------------------------\n",
      "    d) Download   l) List    u) Update   c) Config   h) Help   q) Quit\n",
      "---------------------------------------------------------------------------\n",
      "Downloader> d\n",
      "\n",
      "Download which package (l=list; x=cancel)?\n",
      "  Identifier> l\n",
      "Packages:\n",
      "  [ ] abc................. Australian Broadcasting Commission 2006\n",
      "  [ ] alpino.............. Alpino Dutch Treebank\n",
      "  [ ] averaged_perceptron_tagger Averaged Perceptron Tagger\n",
      "  [ ] averaged_perceptron_tagger_ru Averaged Perceptron Tagger (Russian)\n",
      "  [ ] basque_grammars..... Grammars for Basque\n",
      "  [ ] biocreative_ppi..... BioCreAtIvE (Critical Assessment of Information\n",
      "                           Extraction Systems in Biology)\n",
      "  [ ] bllip_wsj_no_aux.... BLLIP Parser: WSJ Model\n",
      "  [ ] book_grammars....... Grammars from NLTK Book\n",
      "  [ ] brown............... Brown Corpus\n",
      "  [ ] brown_tei........... Brown Corpus (TEI XML Version)\n",
      "  [ ] cess_cat............ CESS-CAT Treebank\n",
      "  [ ] cess_esp............ CESS-ESP Treebank\n",
      "  [ ] chat80.............. Chat-80 Data Files\n",
      "  [ ] city_database....... City Database\n",
      "  [ ] cmudict............. The Carnegie Mellon Pronouncing Dictionary (0.6)\n",
      "  [ ] comparative_sentences Comparative Sentence Dataset\n",
      "  [ ] comtrans............ ComTrans Corpus Sample\n",
      "  [ ] conll2000........... CONLL 2000 Chunking Corpus\n",
      "  [ ] conll2002........... CONLL 2002 Named Entity Recognition Corpus\n",
      "Hit Enter to continue: \n",
      "  [ ] conll2007........... Dependency Treebanks from CoNLL 2007 (Catalan\n",
      "                           and Basque Subset)\n",
      "  [ ] crubadan............ Crubadan Corpus\n",
      "  [ ] dependency_treebank. Dependency Parsed Treebank\n",
      "  [ ] dolch............... Dolch Word List\n",
      "  [ ] europarl_raw........ Sample European Parliament Proceedings Parallel\n",
      "                           Corpus\n",
      "  [ ] floresta............ Portuguese Treebank\n",
      "  [ ] framenet_v15........ FrameNet 1.5\n",
      "  [ ] framenet_v17........ FrameNet 1.7\n",
      "  [ ] gazetteers.......... Gazeteer Lists\n",
      "  [ ] genesis............. Genesis Corpus\n",
      "  [ ] gutenberg........... Project Gutenberg Selections\n",
      "  [ ] ieer................ NIST IE-ER DATA SAMPLE\n",
      "  [ ] inaugural........... C-Span Inaugural Address Corpus\n",
      "  [ ] indian.............. Indian Language POS-Tagged Corpus\n",
      "  [ ] jeita............... JEITA Public Morphologically Tagged Corpus (in\n",
      "                           ChaSen format)\n",
      "  [ ] kimmo............... PC-KIMMO Data Files\n",
      "  [ ] knbc................ KNB Corpus (Annotated blog corpus)\n",
      "  [ ] large_grammars...... Large context-free and feature-based grammars\n",
      "                           for parser comparison\n",
      "Hit Enter to continue: \n",
      "  [ ] lin_thesaurus....... Lin's Dependency Thesaurus\n",
      "  [ ] mac_morpho.......... MAC-MORPHO: Brazilian Portuguese news text with\n",
      "                           part-of-speech tags\n",
      "  [ ] machado............. Machado de Assis -- Obra Completa\n",
      "  [ ] masc_tagged......... MASC Tagged Corpus\n",
      "  [ ] maxent_ne_chunker... ACE Named Entity Chunker (Maximum entropy)\n",
      "  [ ] maxent_treebank_pos_tagger Treebank Part of Speech Tagger (Maximum entropy)\n",
      "  [ ] moses_sample........ Moses Sample Models\n",
      "  [ ] movie_reviews....... Sentiment Polarity Dataset Version 2.0\n",
      "  [ ] mte_teip5........... MULTEXT-East 1984 annotated corpus 4.0\n",
      "  [ ] mwa_ppdb............ The monolingual word aligner (Sultan et al.\n",
      "                           2015) subset of the Paraphrase Database.\n",
      "  [ ] names............... Names Corpus, Version 1.3 (1994-03-29)\n",
      "  [ ] nombank.1.0......... NomBank Corpus 1.0\n",
      "  [ ] nonbreaking_prefixes Non-Breaking Prefixes (Moses Decoder)\n",
      "  [ ] nps_chat............ NPS Chat\n",
      "  [ ] omw................. Open Multilingual Wordnet\n",
      "  [ ] opinion_lexicon..... Opinion Lexicon\n",
      "  [ ] panlex_swadesh...... PanLex Swadesh Corpora\n",
      "  [ ] paradigms........... Paradigm Corpus\n",
      "  [ ] pe08................ Cross-Framework and Cross-Domain Parser\n",
      "                           Evaluation Shared Task\n",
      "Hit Enter to continue: stopwords\n",
      "  [ ] perluniprops........ perluniprops: Index of Unicode Version 7.0.0\n",
      "                           character properties in Perl\n",
      "  [ ] pil................. The Patient Information Leaflet (PIL) Corpus\n",
      "  [ ] pl196x.............. Polish language of the XX century sixties\n",
      "  [ ] porter_test......... Porter Stemmer Test Files\n",
      "  [ ] ppattach............ Prepositional Phrase Attachment Corpus\n",
      "  [ ] problem_reports..... Problem Report Corpus\n",
      "  [ ] product_reviews_1... Product Reviews (5 Products)\n",
      "  [ ] product_reviews_2... Product Reviews (9 Products)\n",
      "  [ ] propbank............ Proposition Bank Corpus 1.0\n",
      "  [ ] pros_cons........... Pros and Cons\n",
      "  [ ] ptb................. Penn Treebank\n",
      "  [ ] punkt............... Punkt Tokenizer Models\n",
      "  [ ] qc.................. Experimental Data for Question Classification\n",
      "  [ ] reuters............. The Reuters-21578 benchmark corpus, ApteMod\n",
      "                           version\n",
      "  [ ] rslp................ RSLP Stemmer (Removedor de Sufixos da Lingua\n",
      "                           Portuguesa)\n",
      "  [ ] rte................. PASCAL RTE Challenges 1, 2, and 3\n",
      "  [ ] sample_grammars..... Sample Grammars\n",
      "  [ ] semcor.............. SemCor 3.0\n",
      "Hit Enter to continue: \n",
      "  [ ] senseval............ SENSEVAL 2 Corpus: Sense Tagged Text\n",
      "  [ ] sentence_polarity... Sentence Polarity Dataset v1.0\n",
      "  [ ] sentiwordnet........ SentiWordNet\n",
      "  [ ] shakespeare......... Shakespeare XML Corpus Sample\n",
      "  [ ] sinica_treebank..... Sinica Treebank Corpus Sample\n",
      "  [ ] smultron............ SMULTRON Corpus Sample\n",
      "  [ ] snowball_data....... Snowball Data\n",
      "  [ ] spanish_grammars.... Grammars for Spanish\n",
      "  [ ] state_union......... C-Span State of the Union Address Corpus\n",
      "  [ ] subjectivity........ Subjectivity Dataset v1.0\n",
      "  [ ] swadesh............. Swadesh Wordlists\n",
      "  [ ] switchboard......... Switchboard Corpus Sample\n",
      "  [ ] tagsets............. Help on Tagsets\n",
      "  [ ] timit............... TIMIT Corpus Sample\n",
      "  [ ] toolbox............. Toolbox Sample Files\n",
      "  [ ] treebank............ Penn Treebank Sample\n",
      "  [ ] twitter_samples..... Twitter Samples\n",
      "  [ ] udhr2............... Universal Declaration of Human Rights Corpus\n",
      "                           (Unicode Version)\n",
      "  [ ] udhr................ Universal Declaration of Human Rights Corpus\n",
      "  [ ] unicode_samples..... Unicode Samples\n",
      "Hit Enter to continue: d\n",
      "  [ ] universal_tagset.... Mappings to the Universal Part-of-Speech Tagset\n",
      "  [ ] universal_treebanks_v20 Universal Treebanks Version 2.0\n",
      "  [ ] vader_lexicon....... VADER Sentiment Lexicon\n",
      "  [ ] verbnet3............ VerbNet Lexicon, Version 3.3\n",
      "  [ ] verbnet............. VerbNet Lexicon, Version 2.1\n",
      "  [ ] webtext............. Web Text Corpus\n",
      "  [ ] wmt15_eval.......... Evaluation data from WMT15\n",
      "  [ ] word2vec_sample..... Word2Vec Sample\n",
      "  [ ] wordnet............. WordNet\n",
      "  [ ] wordnet_ic.......... WordNet-InfoContent\n",
      "  [ ] words............... Word Lists\n",
      "  [ ] ycoe................ York-Toronto-Helsinki Parsed Corpus of Old\n",
      "                           English Prose\n",
      "\n",
      "Collections:\n",
      "  [P] all-corpora......... All the corpora\n",
      "  [P] all-nltk............ All packages available on nltk_data gh-pages\n",
      "                           branch\n",
      "  [P] all................. All packages\n",
      "  [P] book................ Everything used in the NLTK Book\n",
      "  [P] popular............. Popular packages\n",
      "  [ ] tests............... Packages for running tests\n",
      "Hit Enter to continue: \n",
      "  [ ] third-party......... Third-party data packages\n",
      "\n",
      "([*] marks installed packages; [P] marks partially installed collections)\n",
      "\n",
      "Download which package (l=list; x=cancel)?\n",
      "  Identifier> \n",
      "\n",
      "---------------------------------------------------------------------------\n",
      "    d) Download   l) List    u) Update   c) Config   h) Help   q) Quit\n",
      "---------------------------------------------------------------------------\n",
      "Downloader> q\n"
     ]
    }
   ],
   "source": [
    "nltk.download_shell()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "messages = [line.rstrip() for line in open('smsspamcollection/SMSSpamCollection')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5574\n"
     ]
    }
   ],
   "source": [
    "print(len(messages))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ham\\tWhat you thinked about me. First time you saw me in class.'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages[50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 ham\tGo until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\n",
      "\n",
      "\n",
      "1 ham\tOk lar... Joking wif u oni...\n",
      "\n",
      "\n",
      "2 spam\tFree entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005. Text FA to 87121 to receive entry question(std txt rate)T&C's apply 08452810075over18's\n",
      "\n",
      "\n",
      "3 ham\tU dun say so early hor... U c already then say...\n",
      "\n",
      "\n",
      "4 ham\tNah I don't think he goes to usf, he lives around here though\n",
      "\n",
      "\n",
      "5 spam\tFreeMsg Hey there darling it's been 3 week's now and no word back! I'd like some fun you up for it still? Tb ok! XxX std chgs to send, Â£1.50 to rcv\n",
      "\n",
      "\n",
      "6 ham\tEven my brother is not like to speak with me. They treat me like aids patent.\n",
      "\n",
      "\n",
      "7 ham\tAs per your request 'Melle Melle (Oru Minnaminunginte Nurungu Vettam)' has been set as your callertune for all Callers. Press *9 to copy your friends Callertune\n",
      "\n",
      "\n",
      "8 spam\tWINNER!! As a valued network customer you have been selected to receivea Â£900 prize reward! To claim call 09061701461. Claim code KL341. Valid 12 hours only.\n",
      "\n",
      "\n",
      "9 spam\tHad your mobile 11 months or more? U R entitled to Update to the latest colour mobiles with camera for Free! Call The Mobile Update Co FREE on 08002986030\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for mess_no, message in enumerate(messages[:10]):\n",
    "    print(mess_no,message)\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ham\\tGo until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "messages = pd.read_csv('smsspamcollection/SMSSpamCollection',sep='\\t',names=['label','message'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label                                            message\n",
       "0   ham  Go until jurong point, crazy.. Available only ...\n",
       "1   ham                      Ok lar... Joking wif u oni...\n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...\n",
       "3   ham  U dun say so early hor... U c already then say...\n",
       "4   ham  Nah I don't think he goes to usf, he lives aro..."
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5572</td>\n",
       "      <td>5572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>2</td>\n",
       "      <td>5169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>ham</td>\n",
       "      <td>Sorry, I'll call later</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>4825</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       label                 message\n",
       "count   5572                    5572\n",
       "unique     2                    5169\n",
       "top      ham  Sorry, I'll call later\n",
       "freq    4825                      30"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"4\" halign=\"left\">message</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>unique</th>\n",
       "      <th>top</th>\n",
       "      <th>freq</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ham</th>\n",
       "      <td>4825</td>\n",
       "      <td>4516</td>\n",
       "      <td>Sorry, I'll call later</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spam</th>\n",
       "      <td>747</td>\n",
       "      <td>653</td>\n",
       "      <td>Please call our customer service representativ...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      message                                                               \n",
       "        count unique                                                top freq\n",
       "label                                                                       \n",
       "ham      4825   4516                             Sorry, I'll call later   30\n",
       "spam      747    653  Please call our customer service representativ...    4"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages.groupby('label').describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "messages['length'] = messages['message'].apply(len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>message</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label                                            message  length\n",
       "0   ham  Go until jurong point, crazy.. Available only ...     111\n",
       "1   ham                      Ok lar... Joking wif u oni...      29\n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...     155\n",
       "3   ham  U dun say so early hor... U c already then say...      49\n",
       "4   ham  Nah I don't think he goes to usf, he lives aro...      61"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt \n",
    "import seaborn as sns "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAD8CAYAAABthzNFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAErtJREFUeJzt3X+QXWV9x/H31wRB0BKBgGl+uFAzCOPIj640FjtV0CqgBDtgYRyJTDSdKa1andHFOlVn2hmYsQKOHWoU20BVfimSEqrFADr9QyAIBRQoEVNYQ0mU8KOiIvLtH/dZclmeZM8me/be3ft+zdy553nOc+/93pPDfjg/b2QmkiSN96JeFyBJ6k8GhCSpyoCQJFUZEJKkKgNCklRlQEiSqgwISVKVASFJqjIgJElVc3tdwO444IADcmhoqNdlSNKMctttt/0sM+dPNG5GB8TQ0BAbNmzodRmSNKNExP80GecuJklSlQEhSaoyICRJVQaEJKnKgJAkVRkQkqSqVgMiIjZFxF0RcUdEbCh9+0XE9RFxf3l+eemPiPhcRGyMiDsj4ug2a5Mk7dx0bEG8KTOPzMzh0h4B1mfmUmB9aQOcACwtj1XARdNQmyRpB3qxi2k5sKZMrwFO6eq/JDu+D8yLiAU9qE+SRPsBkcB/RMRtEbGq9B2UmQ8DlOcDS/9C4KGu146Wvmk1NLKOoZF10/2xktR32r7VxrGZuTkiDgSuj4h7dzI2Kn35gkGdoFkFsGTJkqmpUpL0Aq1uQWTm5vK8BbgaOAZ4ZGzXUXneUoaPAou7Xr4I2Fx5z9WZOZyZw/PnT3ivqV3mloSkQddaQETEPhHxsrFp4E+Au4G1wIoybAVwTZleC5xZzmZaBjw+titKkjT92tzFdBBwdUSMfc5XM/NbEXErcEVErAQeBE4r468DTgQ2Ak8BZ7VYmyRpAq0FRGY+ABxR6f85cHylP4Gz26pHkjQ5XkktSaoyICRJVQaEJKnKgJAkVRkQkqQqA0KSVGVASJKqDAhJUpUBIUmqMiAkSVUGhCSpyoCQJFUZEJKkKgNiAv5wkKRBZUBIkqoMCElSlQEhSaoyICRJVQaEJKnKgJAkVRkQkqQqA0KSVGVASJKqDAhJUpUBIUmqMiAkSVUGhCSpyoCQJFUZEJKkKgNCklRlQEiSqgwISVJV6wEREXMi4vaIuLa0D46ImyPi/oi4PCJeXPr3LO2NZf5Q27VJknZsOrYgPgjc09U+Dzg/M5cC24CVpX8lsC0zXwWcX8ZJknqk1YCIiEXAScCXSjuA44CrypA1wCllenlpU+YfX8ZLknqg7S2IC4CPAs+W9v7AY5n5TGmPAgvL9ELgIYAy//EyXpLUA60FRES8HdiSmbd1d1eGZoN53e+7KiI2RMSGrVu3TkGlkqSaNrcgjgVOjohNwGV0di1dAMyLiLllzCJgc5keBRYDlPn7Ao+Of9PMXJ2Zw5k5PH/+/BbLl6TB1lpAZOY5mbkoM4eA04EbMvPdwI3AqWXYCuCaMr22tCnzb8jMF2xBSJKmRy+ug/gY8OGI2EjnGMPFpf9iYP/S/2FgpAe1SZKKuRMP2X2ZeRNwU5l+ADimMuZXwGnTUY8kaWJeSS1JqjIgJElVBoQkqcqAkCRVGRCSpCoDQpJUZUBIkqoMCElSlQEhSaoyIBoaGlnH0Mi6XpchSdPGgJAkVRkQkqQqA0KSVGVASJKqDAhJUpUBIUmqMiAkSVXT8otys0n3tRCbzj2ph5VIUrvcgpAkVRkQs4BXeUtqgwEhSaoyICRJVQaEJKnKgJAkVRkQkqQqA0KSVGVASJKqGgVERLym7UIkSf2l6RbEP0XELRHxFxExr9WKJEl9oVFAZOYbgHcDi4ENEfHViHhLq5VJknqq8TGIzLwf+ATwMeCPgc9FxL0R8adtFSdJ6p2mxyBeGxHnA/cAxwHvyMzDyvT5LdYnSeqRprf7/jzwReDjmfnLsc7M3BwRn2ilMklSTzXdxXQi8NWxcIiIF0XE3gCZeWntBRGxVzmw/V8R8cOI+HTpPzgibo6I+yPi8oh4cenfs7Q3lvlDu/vlJEm7rmlAfAd4SVd779K3M78GjsvMI4AjgbdFxDLgPOD8zFwKbANWlvErgW2Z+So6u63Oa1ibJKkFTQNir8z8v7FGmd57Zy/IjrHX7FEeSee4xVWlfw1wSpleXtqU+cdHRDSsT5I0xZoGxC8i4uixRkT8PvDLnYwfGzcnIu4AtgDXAz8GHsvMZ8qQUWBhmV4IPARQ5j8O7F95z1URsSEiNmzdurVh+ZKkyWp6kPpDwJURsbm0FwB/NtGLMvO3wJHl4rqrgcNqw8pzbWshX9CRuRpYDTA8PPyC+ZKkqdEoIDLz1oh4NXAonT/k92bmb5p+SGY+FhE3AcuAeRExt2wlLALGQmeUzoV4oxExF9gXeLTxN5EkTanJ3KzvdcBrgaOAMyLizJ0Njoj5Y7fliIiXAG+mcx3FjcCpZdgK4Joyvba0KfNvyEy3ECSpRxptQUTEpcDvAXcAvy3dCVyyk5ctANZExBw6QXRFZl4bET8CLouIvwNuBy4u4y8GLo2IjXS2HE6f7JeRJE2dpscghoHDJ/N/9Jl5J52tjfH9DwDHVPp/BZzW9P0lSe1quovpbuAVbRYyEw2NrGNoZF2vy5CkVjTdgjgA+FFE3ELnAjgAMvPkVqqSJPVc04D4VJtFSJL6T9PTXL8bEa8Elmbmd8p9mOa0W5okqZea3u77/XRuf/GF0rUQ+GZbRUmSeq/pQeqzgWOBJ+C5Hw86sK2iJEm91zQgfp2ZT481ypXOXsQmSbNY04D4bkR8HHhJ+S3qK4F/a68sSVKvNQ2IEWArcBfw58B1dH6fWpI0SzU9i+lZOj85+sV2y5Ek9Yum92L6CfVbbx8y5RVJkvrCZO7FNGYvOvdM2m/qy5Ek9YtGxyAy8+ddj59m5gV0fjpUkjRLNd3FdHRX80V0tihe1kpFkqS+0HQX0z90TT8DbALeNeXVSJL6RtOzmN7UdiGSpP7SdBfTh3c2PzM/OzXlSJL6xWTOYnodnd+NBngH8D3goTaKkiT13mR+MOjozHwSICI+BVyZme9rqzBJUm81vdXGEuDprvbTwNCUVyNJ6htNtyAuBW6JiKvpXFH9TuCS1qqSJPVc07OY/j4i/h34o9J1Vmbe3l5ZkqRea7qLCWBv4InMvBAYjYiDW6pJktQHmv7k6CeBjwHnlK49gH9tqyhJUu813YJ4J3Ay8AuAzNyMt9qQpFmtaUA8nZlJueV3ROzTXkmSpH7QNCCuiIgvAPMi4v3Ad/DHgyRpVmt6FtNnym9RPwEcCvxtZl7famWSpJ6aMCAiYg7w7cx8M2AoSNKAmHAXU2b+FngqIvadhnokSX2i6ZXUvwLuiojrKWcyAWTmB1qpSpLUc00DYl15SJIGxE4DIiKWZOaDmblmsm8cEYvp3K/pFcCzwOrMvDAi9gMup3Ozv03AuzJzW0QEcCFwIvAU8N7M/MFkP7cfDI10snTTuSf1uBJJ2nUTHYP45thERHx9ku/9DPCRzDwMWAacHRGHAyPA+sxcCqwvbYATgKXlsQq4aJKfJ0maQhPtYoqu6UMm88aZ+TDwcJl+MiLuARYCy4E3lmFrgJvo3MZjOXBJuSDv+xExLyIWlPeZEca2HCRpNpgoIHIH05MSEUPAUcDNwEFjf/Qz8+GIOLAMW8jzf6FutPQ9LyAiYhWdLQyWLFmyqyVNKYNB0mw00S6mIyLiiYh4EnhtmX4iIp6MiCeafEBEvBT4OvChzNzZa6LS94JQyszVmTmcmcPz589vUoIkaRfsdAsiM+fszptHxB50wuErmfmN0v3I2K6jiFgAbCn9o8DirpcvAjbvzufPdm65SGrTZH4PYlLKWUkXA/dk5me7Zq0FVpTpFcA1Xf1nRscy4PGZdPxBkmabptdB7IpjgffQucDujtL3ceBcOjf/Wwk8CJxW5l1H5xTXjXROcz2rxdokSRNoLSAy8z+pH1cAOL4yPoGz26pHkjQ5re1ikiTNbAaEJKnKgJAkVRkQkqQqA0KSVGVASJKqDAhJUpUBIUmqMiAkSVUGhCSpyoCYBkMj67zzqqQZx4CQJFW1eTdXtcStEUnTwYBokX/IJc1k7mKSJFUZEJKkKgNCklRlQEiSqgwISVKVASFJqjIgJElVBoQkqcqAkCRVGRCSpCoDYhp5V1dJM4n3YppBDBdJ08mAmAEMBkm94C4mSVKVATGLeIxD0lQyICRJVQaEJKnKgJAkVbV2FlNEfBl4O7AlM19T+vYDLgeGgE3AuzJzW0QEcCFwIvAU8N7M/EFbtfWr8ccPNp17Uo8qkaR2tyD+BXjbuL4RYH1mLgXWlzbACcDS8lgFXNRiXZKkBloLiMz8HvDouO7lwJoyvQY4pav/kuz4PjAvIha0VZskaWLTfQzioMx8GKA8H1j6FwIPdY0bLX2zkqejSpoJ+uVK6qj0ZXVgxCo6u6FYsmRJmzW1bqKQMEQk9dJ0b0E8MrbrqDxvKf2jwOKucYuAzbU3yMzVmTmcmcPz589vtVhJGmTTHRBrgRVlegVwTVf/mdGxDHh8bFeUJKk32jzN9WvAG4EDImIU+CRwLnBFRKwEHgROK8Ovo3OK60Y6p7me1VZdkqRmWguIzDxjB7OOr4xN4Oy2apEkTZ5XUkuSqgwISVKVASFJqjIgJElVBoQkqcqAkCRV9cutNnrO21pI0vO5BSFJqjIgJElVBoQkqcqAkCRVGRCSpCoDQpJUZUBIkqoMCElSlQEhSaoa2IAYGlnn1dOStBMDf6sNQ0KS6gZ2C0KStHMGhCSpyoCQJFUZEJKkKgNCklRlQMxCnsIraSoYEJKkKgNCklRlQEiSqgyIAeAxCUm7YuBvtTGbGQqSdodbEJKkKgNCklRlQEiSqvoqICLibRFxX0RsjIiRXtczW40/aO1BbEk1fXOQOiLmAP8IvAUYBW6NiLWZ+aPeVjZ7jA+BiUJhR/M3nXvSlNUkqX/1TUAAxwAbM/MBgIi4DFgOGBDTZCwQJgqA8cHRJDAmCqM2PnM6NV120kzSTwGxEHioqz0K/EGPahlok93dNBW7p3b1M8f+IO+oPZEd/UHfUSDtSohO9Bm9DpWJ6tjR/H6pvw39/N2ms7bIzNY/pImIOA14a2a+r7TfAxyTmX81btwqYFVpHgrct4sfeQDws1187WzjstjOZbGdy2K72bYsXpmZ8yca1E9bEKPA4q72ImDz+EGZuRpYvbsfFhEbMnN4d99nNnBZbOey2M5lsd2gLot+OovpVmBpRBwcES8GTgfW9rgmSRpYfbMFkZnPRMRfAt8G5gBfzswf9rgsSRpYfRMQAJl5HXDdNH3cbu+mmkVcFtu5LLZzWWw3kMuibw5SS5L6Sz8dg5Ak9ZGBC4hBu51HRCyOiBsj4p6I+GFEfLD07xcR10fE/eX55aU/IuJzZfncGRFH9/YbTL2ImBMRt0fEtaV9cETcXJbF5eUkCSJiz9LeWOYP9bLuqRYR8yLiqoi4t6wfrx/U9SIi/rr893F3RHwtIvYa1PWi20AFRNftPE4ADgfOiIjDe1tV654BPpKZhwHLgLPLdx4B1mfmUmB9aUNn2Swtj1XARdNfcus+CNzT1T4POL8si23AytK/EtiWma8Czi/jZpMLgW9l5quBI+gsk4FbLyJiIfABYDgzX0PnJJnTGdz1YrvMHJgH8Hrg213tc4Bzel3XNC+Da+jc7+o+YEHpWwDcV6a/AJzRNf65cbPhQef6mvXAccC1QNC5AGru+HWEzhl1ry/Tc8u46PV3mKLl8DvAT8Z/n0FcL9h+F4f9yr/ztcBbB3G9GP8YqC0I6rfzWNijWqZd2RQ+CrgZOCgzHwYozweWYbN9GV0AfBR4trT3Bx7LzGdKu/v7PrcsyvzHy/jZ4BBgK/DPZXfblyJiHwZwvcjMnwKfAR4EHqbz73wbg7lePM+gBURU+gbiNK6IeCnwdeBDmfnEzoZW+mbFMoqItwNbMvO27u7K0Gwwb6abCxwNXJSZRwG/YPvupJpZuyzKcZblwMHA7wL70NmlNt4grBfPM2gB0eh2HrNNROxBJxy+kpnfKN2PRMSCMn8BsKX0z+ZldCxwckRsAi6js5vpAmBeRIxdE9T9fZ9bFmX+vsCj01lwi0aB0cy8ubSvohMYg7hevBn4SWZuzczfAN8A/pDBXC+eZ9ACYuBu5xERAVwM3JOZn+2atRZYUaZX0Dk2MdZ/ZjlrZRnw+Nguh5kuM8/JzEWZOUTn3/6GzHw3cCNwahk2flmMLaNTy/hZ8X+Kmfm/wEMRcWjpOp7OrfUHbr2gs2tpWUTsXf57GVsWA7devECvD4JM9wM4Efhv4MfA3/S6nmn4vm+gs/l7J3BHeZxIZ5/peuD+8rxfGR90zvT6MXAXnTM7ev49WlgubwSuLdOHALcAG4ErgT1L/16lvbHMP6TXdU/xMjgS2FDWjW8CLx/U9QL4NHAvcDdwKbDnoK4X3Q+vpJYkVQ3aLiZJUkMGhCSpyoCQJFUZEJKkKgNCklRlQEiSqgwISVKVASFJqvp/T/sYnhlygB0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "messages['length'].plot.hist(bins=150)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    5572.000000\n",
       "mean       80.489950\n",
       "std        59.942907\n",
       "min         2.000000\n",
       "25%        36.000000\n",
       "50%        62.000000\n",
       "75%       122.000000\n",
       "max       910.000000\n",
       "Name: length, dtype: float64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages['length'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"For me the love should start with attraction.i should feel that I need her every time around me.she should be the first thing which comes in my thoughts.I would start the day and end it with her.she should be there every time I dream.love will be then when my every breath has her name.my life should happen around her.my life will be named to her.I would cry for her.will give all my happiness and take all her sorrows.I will be ready to fight with anyone for her.I will be in love when I will be doing the craziest things for her.love will be when I don't have to proove anyone that my girl is the most beautiful lady on the whole planet.I will always be singing praises for her.love will be when I start up making chicken curry and end up makiing sambar.life will be the most beautiful then.will get every morning and thank god for the day because she is with me.I would like to say a lot..will tell later..\""
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages[messages['length']==910]['message'].iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuUAAAEQCAYAAAAXjQrJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAHhxJREFUeJzt3X+05XVd7/HnS0ZQMPl5IJgBB2PCylLphKS3IscUxBXkksJbMnLxTneF/bJ1A6u1yG51x26FuLp6m/jheP0BiBVTksbFzNUP0AEJ+WEyIsLw8xg/sigVed8/9vfE5syZmTPnnL0/s/d+Ptaadb778/18937vvc+c72t/9uf7/aaqkCRJktTOM1oXIEmSJE06Q7kkSZLUmKFckiRJasxQLkmSJDVmKJckSZIaM5RLkiRJjRnKNfKS3JXkla3rkCRJWixDuSRJktSYoVySJElqzFCucfHiJDcneSzJ5UmeleTAJH+eZCbJI93yqtkNknwyyW8m+bsk/5Lkz5IcnOQDSf45yWeSrG73lCRJuyPJuUnuTfLVJP+YZG2SX09yZbdv+GqSG5O8qG+b85J8sVt3W5If61v3piR/m+SCJI8muTPJy7r2e5I8lGRdm2ercWMo17j4ceAk4Gjge4A30fv9vhR4HnAU8G/AH8zZ7gzgjcBK4NuAv++2OQi4HTh/8KVLkpYqybHAW4Dvq6pvAV4N3NWtPhX4ML2/7R8E/jTJM7t1XwR+ANgfeDvw/iSH9931S4GbgYO7bS8Dvg84Bvgp4A+SPGdwz0yTwlCucfGuqrqvqh4G/gx4cVX9U1V9pKoer6qvAr8F/NCc7S6tqi9W1WPAXwBfrKr/V1VP0PsD/pKhPgtJ0mJ9E9gH+M4kz6yqu6rqi926G6rqyqr6BvD7wLOAEwCq6sPd/uPJqrocuAM4vu9+v1RVl1bVN4HLgSOB36iqr1XVXwJfpxfQpSUxlGtcPNC3/DjwnCT7JvnDJF9O8s/Ap4ADkuzV1/fBvuV/m+e2ox+SNAKqaivwC8CvAw8luSzJEd3qe/r6PQlsA44ASHJmkpu66SmPAi8EDum767n7BarKfYWWnaFc4+yXgGOBl1bVc4Ef7NrTriRJ0qBU1Qer6j/Rm7ZYwDu6VUfO9knyDGAVcF+S5wF/RG/ay8FVdQBwC+4n1IChXOPsW+iNYDya5CCcHy5JYyvJsUlekWQf4N/p/f3/Zrf6e5O8LskKeqPpXwOuA/ajF95nuvs4i95IuTR0hnKNs3cCzwa+Qu+P78faliNJGqB9gA30/uY/ABwK/Eq37irgJ4BH6B3c/7qq+kZV3Qb8Hr2D/B8Evhv42yHXLQGQqmpdgyRJ0kAk+XXgmKr6qda1SDvjSLkkSZLUmKFckiRJaszpK5IkSVJjjpRLkpZNkku6S4/f0tf2v5J8PsnNSf4kyQF9696WZGt3SfRXt6laktozlEuSltN7gZPmtF0DvLCqvgf4AvA2gCTfCZwBfFe3zbvnXNxLkibGitYF7MwhhxxSq1evbl2GJC3IDTfc8JWqmmpdR0tV9akkq+e0/WXfzeuA13fLpwKXVdXXgC8l2Urv8uZ/v7PHcN8gaZQsdN+wR4fy1atXs2XLltZlSNKCJPly6xpGwH8BLu+WV9IL6bO2dW3bSbIeWA9w1FFHuW+QNDIWum9w+ookaSiS/CrwBPCB2aZ5us179oGq2lhV01U1PTU10V9GSBpTe/RIuSRpPCRZB7wWWFtPnfZrG3BkX7dVwH3Drk2S9gSOlEuSBirJScC5wI9W1eN9qzYDZyTZJ8nRwBrg0y1qlKTWHCmXJC2bJB8CTgQOSbINOJ/e2Vb2Aa5JAnBdVf23qro1yRXAbfSmtZxTVd9sU7kktWUolyQtm6p6wzzNF++k/28BvzW4iiRpNDh9RZIkSWrMUC5JkiQ1ZiiXJEmSGpuYOeWrz/vo027fteGURpVIkiQtD/PN+HCkXJIkSWrMUC5JkiQ1ZiiXJEmSGttlKE9ySZKHktzS13ZQkmuS3NH9PLBrT5J3Jdma5OYkx/Vts67rf0d3uWVJkiRJLGyk/L3ASXPazgOurao1wLXdbYCT6V0meQ2wHngP9EI8vau6vRQ4Hjh/NshLkiRJk26XobyqPgU8PKf5VGBTt7wJOK2v/X3Vcx1wQJLDgVcD11TVw1X1CHAN2wd9SZIkaSIt9pSIh1XV/QBVdX+SQ7v2lcA9ff22dW07am9m7imEwNMISZIkqY3lPtAz87TVTtq3v4NkfZItSbbMzMwsa3GSJEnSnmixofzBbloK3c+HuvZtwJF9/VYB9+2kfTtVtbGqpqtqempqapHlSZIkSaNjsaF8MzB7BpV1wFV97Wd2Z2E5AXism+byceBVSQ7sDvB8VdcmSZIkTbxdzilP8iHgROCQJNvonUVlA3BFkrOBu4HTu+5XA68BtgKPA2cBVNXDSf4H8Jmu329U1dyDRyVJkqSJtMtQXlVv2MGqtfP0LeCcHdzPJcAlu1WdJEmSNAG8oqckSZLU2GJPiShJkqQ9jKd8Hl2OlEuSJEmNGcolSZKkxgzlkiRJUmOGckmSJKkxQ7kkSZLUmKFckiRJasxQLkmSJDVmKJckSZIaM5RLkpZNkkuSPJTklr62g5Jck+SO7ueBXXuSvCvJ1iQ3JzmuXeWS1JahXJK0nN4LnDSn7Tzg2qpaA1zb3QY4GVjT/VsPvGdINUrSHsdQLklaNlX1KeDhOc2nApu65U3AaX3t76ue64ADkhw+nEolac9iKJckDdphVXU/QPfz0K59JXBPX79tXZskTRxDuSSplczTVvN2TNYn2ZJky8zMzIDLkqThM5RLkgbtwdlpKd3Ph7r2bcCRff1WAffNdwdVtbGqpqtqempqaqDFSlILhnJJ0qBtBtZ1y+uAq/raz+zOwnIC8NjsNBdJmjQrWhcgSRofST4EnAgckmQbcD6wAbgiydnA3cDpXfergdcAW4HHgbOGXrAk7SEM5ZKkZVNVb9jBqrXz9C3gnMFWJEmjwekrkiRJUmOGckmSJKkxQ7kkSZLUmKFckiRJasxQLkmSJDVmKJckSZIaM5RLkiRJjRnKJUmSpMYM5ZIkSVJjhnJJkiSpMUO5JEmS1JihXJIkSWpsSaE8yS8muTXJLUk+lORZSY5Ocn2SO5JcnmTvru8+3e2t3frVy/EEJEmSpFG36FCeZCXwc8B0Vb0Q2As4A3gHcEFVrQEeAc7uNjkbeKSqjgEu6PpJkiRJE2+p01dWAM9OsgLYF7gfeAVwZbd+E3Bat3xqd5tu/dokWeLjS5IkSSNv0aG8qu4Ffhe4m14Yfwy4AXi0qp7oum0DVnbLK4F7um2f6PofPPd+k6xPsiXJlpmZmcWWJ0mSJI2MpUxfOZDe6PfRwBHAfsDJ83St2U12su6phqqNVTVdVdNTU1OLLU+SJEkaGUuZvvJK4EtVNVNV3wD+GHgZcEA3nQVgFXBft7wNOBKgW78/8PASHl+SJEkaC0sJ5XcDJyTZt5sbvha4Dfgr4PVdn3XAVd3y5u423fpPVNV2I+WSJEnSpFnKnPLr6R2weSPwue6+NgLnAm9NspXenPGLu00uBg7u2t8KnLeEuiVJkqSxsWLXXXasqs4Hzp/TfCdw/Dx9/x04fSmPJ0mSJI0jr+gpSZIkNWYolyRJkhozlEuSJEmNGcolSZKkxgzlkiRJUmOGckmSJKkxQ7kkaSiS/GKSW5PckuRDSZ6V5Ogk1ye5I8nlSfZuXacktWAolyQNXJKVwM8B01X1QmAv4AzgHcAFVbUGeAQ4u12VktSOoVySNCwrgGcnWQHsC9wPvILe1aEBNgGnNapNkpoylEuSBq6q7gV+F7ibXhh/DLgBeLSqnui6bQNWtqlQktoylEuSBi7JgcCpwNHAEcB+wMnzdK0dbL8+yZYkW2ZmZgZXqCQ1YiiXJA3DK4EvVdVMVX0D+GPgZcAB3XQWgFXAffNtXFUbq2q6qqanpqaGU7EkDZGhXJI0DHcDJyTZN0mAtcBtwF8Br+/6rAOualSfJDVlKJckDVxVXU/vgM4bgc/R2/9sBM4F3ppkK3AwcHGzIiWpoRW77iJJ0tJV1fnA+XOa7wSOb1COJO1RHCmXJEmSGjOUS5IkSY0ZyiVJkqTGDOWSJElSY4ZySZIkqTFDuSRJktSYoVySJElqzFAuSZIkNWYolyRJkhozlEuSJEmNGcolSZKkxla0LkCSJEmDs/q8jz7t9l0bTmlUiXbGkXJJkiSpMUfK+/hJUpIkSS04Ui5JkiQ1tqRQnuSAJFcm+XyS25N8f5KDklyT5I7u54Fd3yR5V5KtSW5OctzyPAVJkiRptC11pPxC4GNV9QLgRcDtwHnAtVW1Bri2uw1wMrCm+7ceeM8SH1uSJEkaC4sO5UmeC/wgcDFAVX29qh4FTgU2dd02Aad1y6cC76ue64ADkhy+6MolSZKkMbGUkfLnAzPApUk+m+SiJPsBh1XV/QDdz0O7/iuBe/q239a1SZIkSRNtKaF8BXAc8J6qegnwrzw1VWU+maettuuUrE+yJcmWmZmZJZQnSZIkjYalhPJtwLaqur67fSW9kP7g7LSU7udDff2P7Nt+FXDf3Dutqo1VNV1V01NTU0soT5IkSRoNiw7lVfUAcE+SY7umtcBtwGZgXde2DriqW94MnNmdheUE4LHZaS6SJEnSJFvqxYN+FvhAkr2BO4Gz6AX9K5KcDdwNnN71vRp4DbAVeLzrK0mSJE28JYXyqroJmJ5n1dp5+hZwzlIeT5IkaVJ4pfHJ4hU9JUmSpMYM5ZIkSVJjhnJJkiSpMUO5JGkokhyQ5Mokn09ye5LvT3JQkmuS3NH9PLB1nZLUgqFckjQsFwIfq6oXAC8Cbqd30blrq2oNcC07vwidJI0tQ7kkaeCSPBf4QeBigKr6elU9CpwKbOq6bQJOa1OhJLVlKJckDcPzgRng0iSfTXJRkv2Aw2YvJNf9PHS+jZOsT7IlyZaZmZnhVS1JQ2IolyQNwwrgOOA9VfUS4F/ZjakqVbWxqqaranpqampQNUpSM4ZySdIwbAO2VdX13e0r6YX0B5McDtD9fKhRfZLUlKFckjRwVfUAcE+SY7umtcBtwGZgXde2DriqQXmS1NyK1gVIkibGzwIfSLI3cCdwFr3BoSuSnA3cDZzesD5JasZQLkkaiqq6CZieZ9XaYdciSXsap69IkiRJjRnKJUmSpMYM5ZIkSVJjhnJJkiSpMUO5JEmS1JihXJIkSWrMUC5JkiQ1ZiiXJEmSGvPiQTux+ryPbtd214ZTGlQiSZKkceZIuSRJktSYoVySJElqzFAuSZIkNWYolyRJkhozlEuSJEmNGcolSZKkxgzlkiRJUmOGckmSJKkxQ7kkSZLUmFf0lCRJGgHzXWlc42PJI+VJ9kry2SR/3t0+Osn1Se5IcnmSvbv2fbrbW7v1q5f62JIkSdI4WI7pKz8P3N53+x3ABVW1BngEOLtrPxt4pKqOAS7o+kmSJEkTb0mhPMkq4BTgou52gFcAV3ZdNgGndcundrfp1q/t+kuSJEkTbakj5e8Efhl4srt9MPBoVT3R3d4GrOyWVwL3AHTrH+v6S5IkSRNt0Qd6Jnkt8FBV3ZDkxNnmebrWAtb13+96YD3AUUcdtdjyJEmSRoYHcWopI+UvB340yV3AZfSmrbwTOCDJbNhfBdzXLW8DjgTo1u8PPDz3TqtqY1VNV9X01NTUEsqTJEmSRsOiQ3lVva2qVlXVauAM4BNV9ZPAXwGv77qtA67qljd3t+nWf6KqthsplyRJkibNIC4edC7w1iRb6c0Zv7hrvxg4uGt/K3DeAB5bkiRJGjnLcvGgqvok8Mlu+U7g+Hn6/Dtw+nI8niRpNCXZC9gC3FtVr01yNL0pkAcBNwJvrKqvt6xRkloYxEi5JEk7stBrW0jSRDGUS5KGYjevbSFJE8VQLkkalt25tsXTJFmfZEuSLTMzM4OvVJKGzFAuSRq4/mtb9DfP03Xes3J5ulxJ425ZDvSUJGkXZq9t8RrgWcBz6bu2RTda3n9tC0maKI6US5IGbhHXtpCkieJI+W6aexncuzac0qgSSRoL5wKXJflN4LM8dW0LSZoohnJJ0lAt5NoWkjRpDOWSJEnLZO436uC36loY55RLkiRJjRnKJUmSpMYM5ZIkSVJjhnJJkiSpMUO5JEmS1JihXJIkSWrMUC5JkiQ1ZiiXJEmSGvPiQZIkSQPkBYW0EIZySZKkIZsvqGuyOX1FkiRJasxQLkmSJDVmKJckSZIaM5RLkiRJjRnKJUmSpMYM5ZIkSVJjhnJJkiSpMUO5JEmS1JihXJIkSWrMUC5JkiQ1ZiiXJEmSGlvRuoBRt/q8j27XdteGUxpUIkmSpFG16FCe5EjgfcC3Ak8CG6vqwiQHAZcDq4G7gB+vqkeSBLgQeA3wOPCmqrpxaeXvmeYGdUO6JEmSdmYp01eeAH6pqr4DOAE4J8l3AucB11bVGuDa7jbAycCa7t964D1LeGxJkiRpbCx6pLyq7gfu75a/muR2YCVwKnBi120T8Eng3K79fVVVwHVJDkhyeHc/E8dpL5IkSZq1LHPKk6wGXgJcDxw2G7Sr6v4kh3bdVgL39G22rWt7WihPsp7eSDpHHXXUcpQ3Mpz2Imlc7e6Ux1Z1SlIrSz77SpLnAB8BfqGq/nlnXedpq+0aqjZW1XRVTU9NTS21PEnSnmF3pzxK0kRZ0kh5kmfSC+QfqKo/7pofnJ2WkuRw4KGufRtwZN/mq4D7lvL4o2K+qSqSNEkWMeVRkibKokfKu7OpXAzcXlW/37dqM7CuW14HXNXXfmZ6TgAem9T55JI0yXY25RE4dMdbStL4WspI+cuBNwKfS3JT1/YrwAbgiiRnA3cDp3frrqZ3OsSt9E6JeNYSHluSNILmTnnsje8saLuJPd5I0mRYytlX/ob554kDrJ2nfwHnLPbxJEmjbTenPD5NVW0ENgJMT09vdzySJI26JR/oKUnSrixiyqMkTZRlOSWiJEm7sLtTHiVpohjKJUkDt7tTHiVp0jh9RZIkSWrMUC5JkiQ15vQVSZI08ea70N9dG05pUIkmlSPlkiRJUmOGckmSJKkxQ7kkSZLU2FjOKZ9vXpgkSZK0p3KkXJIkSWpsLEfKJUnSZFjIWVOW68wqfhOvQXKkXJIkSWrMUC5JkiQ1ZiiXJEmSGjOUS5IkSY15oKckSQK2P5Bx0i8z74GdGiZHyiVJkqTGHCmXJEkjYaEj145waxQZyiVJkibIcp23XcvL6SuSJElSY46US5K0h3JEU5ocjpRLkiRJjTlSLkmS5rWQkfpBjuZ7wKYmiSPlkiRJUmOOlEuSpGW1mIsQOSquSWco34N5gI8kSdJkMJRLkjTm9sRBntYj460ffxQs5hsPLZ6hXJKkJVhs4DUUak+ykN/H1gf+jjsP9JQkSZIaG/pIeZKTgAuBvYCLqmrDsGsYZQv5JOsnUkmjxP1CG4sdqV/Mdn4rIO3aUEN5kr2A/w38CLAN+EySzVV12zDrmDTL9dWqYV/ScnO/IEk9wx4pPx7YWlV3AiS5DDgV8I/vMlrMvLDFzn+cbzvDvKTdMLT9wmK+aRzmaPJi73shf4elUdN6bnqLxx92KF8J3NN3exvw0iHXoHkMcsez2ANDdrXNQo3ClJ/Wf3ykhtwvSBKQqhregyWnA6+uqjd3t98IHF9VP9vXZz2wvrt5LPCPu/kwhwBfWYZyR8mkPWef73gb5ef7vKqaal3EKFnIfqFrX+q+YRyM8v+N5eJr4GsAo/caLGjfMOyR8m3AkX23VwH39Xeoqo3AxsU+QJItVTW92O1H0aQ9Z5/veJu056td7xdg6fuGceD/DV8D8DWA8X0Nhn1KxM8Aa5IcnWRv4Axg85BrkCTtOdwvSBJDHimvqieSvAX4OL1TX11SVbcOswZJ0p7D/YIk9Qz9POVVdTVw9QAfYhK/3py05+zzHW+T9nwn3hD2C+PC/xu+BuBrAGP6Ggz1QE9JkiRJ2xv2nHJJkiRJcxjKJUmSpMaGPqd8uSV5Ab2rv60Eit6ptDZX1e1NC5MkSZIWaKTnlCc5F3gDcBm9c91C7xy3ZwCXVdWGVrUNUpLD6PsQUlUPNi5p4JIcBFRVPdK6lmHwPZYk6SmTsF8c9VD+BeC7quobc9r3Bm6tqjVtKhuMJC8G/g+wP3Bv17wKeBT4maq6sVVtg5DkKOB3gLX0nmOA5wKfAM6rqrvaVTcYvsfj/x5LC5Fkf+BtwGnA7JUAHwKuAjZU1aOtahu2SQhjO5MkwPE8fUbAp2uUA9xumKT94qhPX3kSOAL48pz2w7t14+a9wE9X1fX9jUlOAC4FXtSiqAG6HHgn8JNV9U2AJHsBp9P7duSEhrUNynvxPR7391haiCvofTg9saoeAEjyrcA64MPAjzSsbSh2FMaSjF0Y25EkrwLeDdzB0wPpMUl+pqr+sllxw/NeJmS/OOoj5ScBf0Dvl/Wervko4BjgLVX1sVa1DUKSO3Y0+p9ka1UdM+yaBmkXz3eH60aZ7/HC1knjLsk/VtWxu7tunCS5iR2HsT+sqrEJYzuS5Hbg5LnfGiY5Gri6qr6jSWFDNEn7xZEeKa+qjyX5dp76Wif05pZ/ZnbUbcz8RZKPAu/jqQ8hRwJnAmP1AaRzQ5J3A5t4+vNdB3y2WVWD5Xs8/u+xtBBfTvLLwKbZ6RrdNI438dT/lXG339xADlBV1yXZr0VBDazgqWPm+t0LPHPItbQyMfvFkR4pn0RJTuaps83MfgjZ3F0Rb6x0xwaczTzPF7i4qr7WsLyB8T0e//dY2pUkBwLn0fu/cRi9ucQP0vu/8Y6qerhheUOR5F3AtzF/GPtSVb2lVW3DkuRtwI/Tm87X/xqcAVxRVf+zVW3DNCn7RUO5JEl7uCQ/QO9b4c9NyDxiYHLC2M4k+Q7mfw1ua1qYlp2hfIT0HY1/KnBo1zy2R+MnWUFvFPU0nn7U+VX0RlG/sZPNR5Lv8fi/x9JCJPl0VR3fLb8ZOAf4U+BVwJ+N6yl/pbkmab/oFT1HyxXAI8APV9XBVXUw8MP0Tgv04aaVDcb/BV4MvB14DXBKt/wi4P0N6xok3+Pxf4+lheifL/zTwKuq6u30QvlPtilpuJLsn2RDktuT/FP37/au7YDW9Q1Dd0KL2eX9k1yU5OYkH+yOMZgEE7NfdKR8hEza0fi7eL5fqKpvH3ZNg+Z7/LR1Y/keSwuR5B+AE+kNnn28qqb71n22ql7SqrZhSfJxeqeF3DTntJBvAtZW1SScFvLGqjquW74IeAD4I+B1wA9V1Wkt6xuGSdovOlI+Wr6c5Jf7Px0nOay7suk4Ho3/SJLTk/zH72mSZyT5CXqfmseR7/H4v8fSQuwP3ABsAQ7qwihJnkNvXvEkWF1V75gN5ABV9UA3deeohnW1Ml1Vv1ZVX66qC4DVrQsakonZLxrKR8tPAAcDf53kkSQPA58EDqJ3dPa4OQN4PfBgki8kuYPeKMHrunXjaFLf4we69/gLjP97LO1SVa2uqudX1dHdz9lg+iTwYy1rG6KJCWM7cWiStyb5JeC5Sfo/kE1KhpuY/aLTV0ZMkhfQu5rXdVX1L33tJ43bxZL6JTmY3ujQO6vqp1rXMyhJXgp8vqoeS7IvvVOiHQfcCvx2VT3WtMBl1p0S8Q30Du68ETgZeBm957vRAz2lyTXntJCzB/jNnhZyQ1WN/bdpSc6f0/Tuqprpvjn5nao6s0VdwzYp2cdQPkKS/By9I/Bvp3dw3M9X1VXduv+YdzYukmyep/kV9OYYUlU/OtyKBi/JrcCLquqJJBuBfwU+Aqzt2l/XtMBlluQD9C6O8WzgMWA/4E/oPd9U1bqG5UnaQyU5q6oubV1HS5PyGkxS9hnpK3pOoP8KfG9V/UuS1cCVSVZX1YWM5xzDVcBtwEX0TpUX4PuA32tZ1IA9o6qe6Jan+/7Y/E16l5weN99dVd/TnRrxXuCIqvpmkvcD/9C4Nkl7rrcDYx9Id2FSXoOJyT6G8tGy1+zXNlV1V5IT6f1yPo8x+8XsTAM/D/wq8N+r6qYk/1ZVf924rkG6pW/04x+STFfVliTfDozjVI5ndFNY9gP2pXdw28PAPkzOJaQlzSPJzTtaRe8qp2PP1wCYoOxjKB8tDyR5cVXdBNB9anwtcAnw3W1LW35V9SRwQZIPdz8fZPx/Z98MXJjk14CvAH+f5B56BzW9uWllg3Ex8HlgL3ofvj6c5E7gBHqXlZY0uQ4DXs32Z2IK8HfDL6cJX4MJyj7OKR8hSVYBT/SfHqpv3cur6m8blDU0SU4BXl5Vv9K6lkFL8i3A8+l9CNlWVQ82LmlgkhwBUFX3dRcEeSVwd1V9um1lklpKcjFwaVX9zTzrPlhV/7lBWUPlazBZ2cdQLkmSJDU2Kee4lCRJkvZYhnJJkiSpMUO5JEmS1JihXJIkSWrMUC5JkiQ19v8BRLorFNdDbMoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Important Plot\n",
    "messages.hist(column='length',by='label',bins=60,figsize=(12,4)) #X-axis has the length, ham messages are centered \n",
    "plt.show()                                                       #around 100 where as spam messages are centred around \n",
    "                                                                #150"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "mess = 'Sample message! Notice: it has punctuation.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string.punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "nopunc = [c for c in mess if c not in string.punctuation]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['S',\n",
       " 'a',\n",
       " 'm',\n",
       " 'p',\n",
       " 'l',\n",
       " 'e',\n",
       " ' ',\n",
       " 'm',\n",
       " 'e',\n",
       " 's',\n",
       " 's',\n",
       " 'a',\n",
       " 'g',\n",
       " 'e',\n",
       " ' ',\n",
       " 'N',\n",
       " 'o',\n",
       " 't',\n",
       " 'i',\n",
       " 'c',\n",
       " 'e',\n",
       " ' ',\n",
       " 'i',\n",
       " 't',\n",
       " ' ',\n",
       " 'h',\n",
       " 'a',\n",
       " 's',\n",
       " ' ',\n",
       " 'p',\n",
       " 'u',\n",
       " 'n',\n",
       " 'c',\n",
       " 't',\n",
       " 'u',\n",
       " 'a',\n",
       " 't',\n",
       " 'i',\n",
       " 'o',\n",
       " 'n']"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nopunc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i',\n",
       " 'me',\n",
       " 'my',\n",
       " 'myself',\n",
       " 'we',\n",
       " 'our',\n",
       " 'ours',\n",
       " 'ourselves',\n",
       " 'you',\n",
       " \"you're\",\n",
       " \"you've\",\n",
       " \"you'll\",\n",
       " \"you'd\",\n",
       " 'your',\n",
       " 'yours',\n",
       " 'yourself',\n",
       " 'yourselves',\n",
       " 'he',\n",
       " 'him',\n",
       " 'his',\n",
       " 'himself',\n",
       " 'she',\n",
       " \"she's\",\n",
       " 'her',\n",
       " 'hers',\n",
       " 'herself',\n",
       " 'it',\n",
       " \"it's\",\n",
       " 'its',\n",
       " 'itself',\n",
       " 'they',\n",
       " 'them',\n",
       " 'their',\n",
       " 'theirs',\n",
       " 'themselves',\n",
       " 'what',\n",
       " 'which',\n",
       " 'who',\n",
       " 'whom',\n",
       " 'this',\n",
       " 'that',\n",
       " \"that'll\",\n",
       " 'these',\n",
       " 'those',\n",
       " 'am',\n",
       " 'is',\n",
       " 'are',\n",
       " 'was',\n",
       " 'were',\n",
       " 'be',\n",
       " 'been',\n",
       " 'being',\n",
       " 'have',\n",
       " 'has',\n",
       " 'had',\n",
       " 'having',\n",
       " 'do',\n",
       " 'does',\n",
       " 'did',\n",
       " 'doing',\n",
       " 'a',\n",
       " 'an',\n",
       " 'the',\n",
       " 'and',\n",
       " 'but',\n",
       " 'if',\n",
       " 'or',\n",
       " 'because',\n",
       " 'as',\n",
       " 'until',\n",
       " 'while',\n",
       " 'of',\n",
       " 'at',\n",
       " 'by',\n",
       " 'for',\n",
       " 'with',\n",
       " 'about',\n",
       " 'against',\n",
       " 'between',\n",
       " 'into',\n",
       " 'through',\n",
       " 'during',\n",
       " 'before',\n",
       " 'after',\n",
       " 'above',\n",
       " 'below',\n",
       " 'to',\n",
       " 'from',\n",
       " 'up',\n",
       " 'down',\n",
       " 'in',\n",
       " 'out',\n",
       " 'on',\n",
       " 'off',\n",
       " 'over',\n",
       " 'under',\n",
       " 'again',\n",
       " 'further',\n",
       " 'then',\n",
       " 'once',\n",
       " 'here',\n",
       " 'there',\n",
       " 'when',\n",
       " 'where',\n",
       " 'why',\n",
       " 'how',\n",
       " 'all',\n",
       " 'any',\n",
       " 'both',\n",
       " 'each',\n",
       " 'few',\n",
       " 'more',\n",
       " 'most',\n",
       " 'other',\n",
       " 'some',\n",
       " 'such',\n",
       " 'no',\n",
       " 'nor',\n",
       " 'not',\n",
       " 'only',\n",
       " 'own',\n",
       " 'same',\n",
       " 'so',\n",
       " 'than',\n",
       " 'too',\n",
       " 'very',\n",
       " 's',\n",
       " 't',\n",
       " 'can',\n",
       " 'will',\n",
       " 'just',\n",
       " 'don',\n",
       " \"don't\",\n",
       " 'should',\n",
       " \"should've\",\n",
       " 'now',\n",
       " 'd',\n",
       " 'll',\n",
       " 'm',\n",
       " 'o',\n",
       " 're',\n",
       " 've',\n",
       " 'y',\n",
       " 'ain',\n",
       " 'aren',\n",
       " \"aren't\",\n",
       " 'couldn',\n",
       " \"couldn't\",\n",
       " 'didn',\n",
       " \"didn't\",\n",
       " 'doesn',\n",
       " \"doesn't\",\n",
       " 'hadn',\n",
       " \"hadn't\",\n",
       " 'hasn',\n",
       " \"hasn't\",\n",
       " 'haven',\n",
       " \"haven't\",\n",
       " 'isn',\n",
       " \"isn't\",\n",
       " 'ma',\n",
       " 'mightn',\n",
       " \"mightn't\",\n",
       " 'mustn',\n",
       " \"mustn't\",\n",
       " 'needn',\n",
       " \"needn't\",\n",
       " 'shan',\n",
       " \"shan't\",\n",
       " 'shouldn',\n",
       " \"shouldn't\",\n",
       " 'wasn',\n",
       " \"wasn't\",\n",
       " 'weren',\n",
       " \"weren't\",\n",
       " 'won',\n",
       " \"won't\",\n",
       " 'wouldn',\n",
       " \"wouldn't\"]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stopwords.words('english') #These are the words that do not tell you any distinguishing feature "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "nopunc = ''.join(nopunc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Sample message Notice it has punctuation'"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nopunc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Sample', 'message', 'Notice', 'it', 'has', 'punctuation']"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nopunc.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_mess = [word for word in nopunc.split() if word.lower() not in stopwords.words('english')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Sample', 'message', 'Notice', 'punctuation']"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_mess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_process(mess):\n",
    "    \"\"\"\n",
    "    1. remove punc\n",
    "    2. remove stop words \n",
    "    3. return text of clean text words\n",
    "    \"\"\"\n",
    "    nopunc = [char for char in mess if char not in string.punctuation]\n",
    "    nopunc = ''.join(nopunc)\n",
    "    return [word for word in nopunc.split() if word.lower() not in stopwords.words('english')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>message</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label                                            message  length\n",
       "0   ham  Go until jurong point, crazy.. Available only ...     111\n",
       "1   ham                      Ok lar... Joking wif u oni...      29\n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...     155\n",
       "3   ham  U dun say so early hor... U c already then say...      49\n",
       "4   ham  Nah I don't think he goes to usf, he lives aro...      61"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    [Go, jurong, point, crazy, Available, bugis, n...\n",
       "1                       [Ok, lar, Joking, wif, u, oni]\n",
       "2    [Free, entry, 2, wkly, comp, win, FA, Cup, fin...\n",
       "3        [U, dun, say, early, hor, U, c, already, say]\n",
       "4    [Nah, dont, think, goes, usf, lives, around, t...\n",
       "Name: message, dtype: object"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages['message'].head(5).apply(text_process)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "bow_transformer = CountVectorizer(analyzer=text_process).fit(messages['message'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11425\n"
     ]
    }
   ],
   "source": [
    "print(len(bow_transformer.vocabulary_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "mess4 = messages['message'][3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "U dun say so early hor... U c already then say...\n"
     ]
    }
   ],
   "source": [
    "print(mess4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "bow4 = bow_transformer.transform([mess4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 4068)\t2\n",
      "  (0, 4629)\t1\n",
      "  (0, 5261)\t1\n",
      "  (0, 6204)\t1\n",
      "  (0, 6222)\t1\n",
      "  (0, 7186)\t1\n",
      "  (0, 9554)\t2\n"
     ]
    }
   ],
   "source": [
    "print(bow4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 11425)\n"
     ]
    }
   ],
   "source": [
    "print(bow4.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'U'"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bow_transformer.get_feature_names()[4068]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'say'"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bow_transformer.get_feature_names()[9554]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "messages_bow = bow_transformer.transform(messages['message'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of the Sparse Matrix:  (5572, 11425)\n"
     ]
    }
   ],
   "source": [
    "print('Shape of the Sparse Matrix: ', messages_bow.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50548"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages_bow.nnz #Amount of non-zero occurences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sparsity: 0.07940295412668218\n"
     ]
    }
   ],
   "source": [
    "sparsity = (100.0 * messages_bow.nnz / (messages_bow.shape[0] * messages_bow.shape[1]))\n",
    "print('sparsity: {}'.format((sparsity)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_transformer = TfidfTransformer().fit(messages_bow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf4 = tfidf_transformer.transform(bow4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 9554)\t0.5385626262927564\n",
      "  (0, 7186)\t0.4389365653379857\n",
      "  (0, 6222)\t0.3187216892949149\n",
      "  (0, 6204)\t0.29953799723697416\n",
      "  (0, 5261)\t0.29729957405868723\n",
      "  (0, 4629)\t0.26619801906087187\n",
      "  (0, 4068)\t0.40832589933384067\n"
     ]
    }
   ],
   "source": [
    "print(tfidf4) #Term Frequency & Inverse term frequency Transform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8.527076498901426"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_transformer.idf_[bow_transformer.vocabulary_['university']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "messages_tfidf = tfidf_transformer.transform(messages_bow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "Spam_detect_model = MultinomialNB().fit(messages_tfidf,messages['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ham'"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Spam_detect_model.predict(tfidf4)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ham'"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages['label'][3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_pred = Spam_detect_model.predict(messages_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['ham', 'ham', 'spam', ..., 'ham', 'ham', 'ham'], dtype='<U4')"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "msg_train, msg_test, label_train, label_test = train_test_split(messages['message'],messages['label'],test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline = Pipeline([\n",
    "                    ('bow',CountVectorizer(analyzer=text_process)),\n",
    "                    ('tfidf',TfidfTransformer()),\n",
    "                    ('classifier',MultinomialNB())\n",
    "                    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('bow', CountVectorizer(analyzer=<function text_process at 0x000002AC82D48048>,\n",
       "        binary=False, decode_error='strict', dtype=<class 'numpy.int64'>,\n",
       "        encoding='utf-8', input='content', lowercase=True, max_df=1.0,\n",
       "        max_features=None, min_df=1, ngram_range=(1, 1), preprocesso...f=False, use_idf=True)), ('classifier', MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True))])"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.fit(msg_train,label_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = pipeline.predict(msg_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ham       0.96      1.00      0.98      1440\n",
      "        spam       1.00      0.72      0.83       232\n",
      "\n",
      "   micro avg       0.96      0.96      0.96      1672\n",
      "   macro avg       0.98      0.86      0.91      1672\n",
      "weighted avg       0.96      0.96      0.96      1672\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(label_test, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline = Pipeline([\n",
    "                    ('bow',CountVectorizer(analyzer=text_process)),\n",
    "                    ('tfidf',TfidfTransformer()),\n",
    "                    ('classifier',RandomForestClassifier())\n",
    "                    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\aujas\\AppData\\Local\\Programs\\Python\\Python37\\lib\\site-packages\\sklearn\\ensemble\\forest.py:246: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('bow', CountVectorizer(analyzer=<function text_process at 0x000002AC82D48048>,\n",
       "        binary=False, decode_error='strict', dtype=<class 'numpy.int64'>,\n",
       "        encoding='utf-8', input='content', lowercase=True, max_df=1.0,\n",
       "        max_features=None, min_df=1, ngram_range=(1, 1), preprocesso...obs=None,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False))])"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.fit(msg_train,label_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = pipeline.predict(msg_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         ham       0.97      1.00      0.98      1440\n",
      "        spam       1.00      0.78      0.87       232\n",
      "\n",
      "   micro avg       0.97      0.97      0.97      1672\n",
      "   macro avg       0.98      0.89      0.93      1672\n",
      "weighted avg       0.97      0.97      0.97      1672\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(label_test, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}