models_testing-trained_Word2Vec.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model testing using embeddings computed using our own Word2Vec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import RidgeClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "from gensim.models import Word2Vec\n",
    "from helpers import *\n",
    "from preprocessing import *\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Loading model that was trained on all the training data. See file `word2vec_training.py`._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Word2Vec.load('word2vec.model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Let's see how many distinct words we have in the model._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "94930"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(model.wv.vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_The vectors are of size 250._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dim = 250"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def toVector(tokens, model, dim):\n",
    "    \"\"\"\n",
    "    This method convert an array of token into a vector of size dim given that word vector in model are of size dim.\n",
    "    \n",
    "    Attributes:\n",
    "    \n",
    "    tokens: arr\n",
    "            tokens to consider for the outputed embedding\n",
    "    \n",
    "    model: gensin model\n",
    "            used to convert a token to a vector of size din\n",
    "            \n",
    "    dim: int \n",
    "            size of the vector    \n",
    "    \"\"\"\n",
    "    \n",
    "    vector = np.zeros(dim)\n",
    "    word_count = 0\n",
    "    \n",
    "    for token in tokens:\n",
    "        if token in model.wv:\n",
    "            vector += model.wv[token]\n",
    "            word_count += 1\n",
    "            \n",
    "    if word_count == 0:\n",
    "        return vector\n",
    "    else:\n",
    "        return vector / float(word_count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_For testing purpose let's build model on the subset of tweets provided._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = load_tweets(full = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. tweets with just the tags <user\\> and <url\\> removed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Let's first remove tags and tokenize the tweet._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>polarity</th>\n",
       "      <th>tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i dunno justin read my mention or not . only j...</td>\n",
       "      <td>1</td>\n",
       "      <td>[i, dunno, justin, read, my, mention, or, not,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>because your logic is so dumb , i won't even c...</td>\n",
       "      <td>1</td>\n",
       "      <td>[because, your, logic, is, so, dumb, i, wo, nt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>just put casper in a box !  looved the battle ...</td>\n",
       "      <td>1</td>\n",
       "      <td>[just, put, casper, in, a, box, looved, the, b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>thanks sir &gt; &gt; don't trip lil mama ... just ke...</td>\n",
       "      <td>1</td>\n",
       "      <td>[thanks, sir, do, nt, trip, lil, mama, just, k...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>visiting my brother tmr is the bestest birthda...</td>\n",
       "      <td>1</td>\n",
       "      <td>[visiting, my, brother, tmr, is, the, bestest,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               tweet  polarity  \\\n",
       "0  i dunno justin read my mention or not . only j...         1   \n",
       "1  because your logic is so dumb , i won't even c...         1   \n",
       "2  just put casper in a box !  looved the battle ...         1   \n",
       "3  thanks sir > > don't trip lil mama ... just ke...         1   \n",
       "4  visiting my brother tmr is the bestest birthda...         1   \n",
       "\n",
       "                                              tokens  \n",
       "0  [i, dunno, justin, read, my, mention, or, not,...  \n",
       "1  [because, your, logic, is, so, dumb, i, wo, nt...  \n",
       "2  [just, put, casper, in, a, box, looved, the, b...  \n",
       "3  [thanks, sir, do, nt, trip, lil, mama, just, k...  \n",
       "4  [visiting, my, brother, tmr, is, the, bestest,...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_1 = remove_tags(tweets)\n",
    "tweets_1 = tokenize_tweets(tweets_1)\n",
    "tweets_1.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Using the tokens we can now compute vector embedding._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_1['vector'] = tweets_1['tokens'].apply(lambda tokens: toVector(tokens, model, dim))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vector</th>\n",
       "      <th>polarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[-0.10623393052568038, 0.27006867496917647, 0....</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[0.3716979566961527, 0.009992905064589448, -0....</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[0.4348416645079851, 0.5073618728201836, 0.431...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[-0.14007517136633396, -0.730455274383227, 0.3...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[0.4355273447930813, 0.40050916727632285, 0.68...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              vector  polarity\n",
       "0  [-0.10623393052568038, 0.27006867496917647, 0....         1\n",
       "1  [0.3716979566961527, 0.009992905064589448, -0....         1\n",
       "2  [0.4348416645079851, 0.5073618728201836, 0.431...         1\n",
       "3  [-0.14007517136633396, -0.730455274383227, 0.3...         1\n",
       "4  [0.4355273447930813, 0.40050916727632285, 0.68...         1"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_1[['vector', 'polarity']].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Let's obtain training and testing data._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_1, test_1 = train_test_split(tweets_1[['vector', 'polarity']], test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "157576\n"
     ]
    }
   ],
   "source": [
    "print(len(train_1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_1 = np.concatenate(train_1['vector'].values).reshape((len(train_1), dim))\n",
    "y_train_1 = train_1['polarity'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test_1 = np.concatenate(test_1['vector'].values).reshape((len(test_1), dim))\n",
    "y_test_1 = test_1['polarity'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will now consider several models for classification."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Without regularization._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=500,\n",
       "                   multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic_reg_1 = LogisticRegression(solver='lbfgs', max_iter = 500)\n",
    "logistic_reg_1.fit(x_train_1, y_train_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7754226531959182\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = logistic_reg_1.score(x_test_1, y_test_1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Tuning regularization parameter._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# C is the inverse of the regularization strength\n",
    "C = np.logspace(0, 4, 10)\n",
    "hyperparameters = dict(C=C)\n",
    "\n",
    "# We first define a new logistic regression model\n",
    "logistic_reg_1 = LogisticRegression(solver='lbfgs', max_iter = 500)\n",
    "# we now perform grid search to find optimal hyperparameter\n",
    "# we use cross-validation with 5 folds\n",
    "clf = GridSearchCV(logistic_reg_1, hyperparameters, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_logistic_1 = clf.fit(x_train_1, y_train_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best C: 21.544346900318832\n"
     ]
    }
   ],
   "source": [
    "print('Best C:', best_logistic_1.best_estimator_.get_params()['C'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.779636492866934\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = best_logistic_1.score(x_test_1, y_test_1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ridge regression\n",
    "\n",
    "_Without regularization._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7754480377722496\n"
     ]
    }
   ],
   "source": [
    "ridge_clf_1 = RidgeClassifier().fit(x_train_1, y_train_1)\n",
    "print('Accuracy = {a}'.format(a = ridge_clf_1.score(x_test_1, y_test_1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Tuning regularization parameter._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "# alpha is a constant that multiplies the regularization term\n",
    "alpha = np.logspace(0, 4, 10)\n",
    "hyperparameters = dict(alpha=alpha)\n",
    "\n",
    "ridge_clf_1 = RidgeClassifier()\n",
    "# we now perform grid search to find optimal hyperparameter\n",
    "# we use cross-validation with 5 folds\n",
    "clf = GridSearchCV(ridge_clf_1, hyperparameters, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_ridge_clf_1 = clf.fit(x_train_1, y_train_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best alpha: 1.0\n"
     ]
    }
   ],
   "source": [
    "print('Best alpha:', best_ridge_clf_1.best_estimator_.get_params()['alpha'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7754480377722496\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = best_ridge_clf_1.score(x_test_1, y_test_1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Support Vector Machine\n",
    "\n",
    "_Without hyperparameters tuning._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "sgd_clf_1 = SGDClassifier(loss = 'hinge').fit(x_train_1, y_train_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7726049652231304\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = sgd_clf_1.score(x_test_1, y_test_1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_With hyperparameters tuning._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# alpha is a constant that multiplies the regularization term\n",
    "alpha = np.logspace(0, 4, 10)\n",
    "hyperparameters = dict(alpha=alpha)\n",
    "\n",
    "svm_clf_1 = SGDClassifier(loss = 'hinge')\n",
    "# we now perform grid search to find optimal hyperparameter\n",
    "# we use cross-validation with 5 folds\n",
    "svm_clf_1 = GridSearchCV(svm_clf_1, hyperparameters, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_svm_clf_1 = svm_clf_1.fit(x_train_1, y_train_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best alpha: 1.0\n"
     ]
    }
   ],
   "source": [
    "print('Best alpha:', best_svm_clf_1.best_estimator_.get_params()['alpha'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.6432197796618775\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = best_svm_clf_1.score(x_test_1, y_test_1)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. tweets with the tags <user\\> and <url\\> as well as english stop words removed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Let's first remove tags and tokenize the tweets with stop words removal._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>polarity</th>\n",
       "      <th>tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i dunno justin read my mention or not . only j...</td>\n",
       "      <td>1</td>\n",
       "      <td>[dunno, justin, read, mention, justin, god, kn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>because your logic is so dumb , i won't even c...</td>\n",
       "      <td>1</td>\n",
       "      <td>[logic, dumb, wo, nt, even, crop, name, photo,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>just put casper in a box !  looved the battle ...</td>\n",
       "      <td>1</td>\n",
       "      <td>[put, casper, box, looved, battle, crakkbitch]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>thanks sir &gt; &gt; don't trip lil mama ... just ke...</td>\n",
       "      <td>1</td>\n",
       "      <td>[thanks, sir, nt, trip, lil, mama, keep, doin,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>visiting my brother tmr is the bestest birthda...</td>\n",
       "      <td>1</td>\n",
       "      <td>[visiting, brother, tmr, bestest, birthday, gi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               tweet  polarity  \\\n",
       "0  i dunno justin read my mention or not . only j...         1   \n",
       "1  because your logic is so dumb , i won't even c...         1   \n",
       "2  just put casper in a box !  looved the battle ...         1   \n",
       "3  thanks sir > > don't trip lil mama ... just ke...         1   \n",
       "4  visiting my brother tmr is the bestest birthda...         1   \n",
       "\n",
       "                                              tokens  \n",
       "0  [dunno, justin, read, mention, justin, god, kn...  \n",
       "1  [logic, dumb, wo, nt, even, crop, name, photo,...  \n",
       "2     [put, casper, box, looved, battle, crakkbitch]  \n",
       "3  [thanks, sir, nt, trip, lil, mama, keep, doin,...  \n",
       "4  [visiting, brother, tmr, bestest, birthday, gi...  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_2 = remove_tags(tweets)\n",
    "tweets_2 = tokenize_tweets(tweets_2, stop_words = True)\n",
    "tweets_2.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_We then obtain tweets embeddings._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_2['vector'] = tweets_2['tokens'].apply(lambda tokens: toVector(tokens, model, dim))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vector</th>\n",
       "      <th>polarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[-0.2076540985568003, 0.03211568295955658, 0.8...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[-0.1722315645052327, 0.040821442380547523, -0...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[-0.010814150795340538, 0.20931822061538696, 1...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[-0.2327376276254654, -0.8353355824947357, 0.4...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[0.1854836568236351, 0.673973988209452, 1.2295...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              vector  polarity\n",
       "0  [-0.2076540985568003, 0.03211568295955658, 0.8...         1\n",
       "1  [-0.1722315645052327, 0.040821442380547523, -0...         1\n",
       "2  [-0.010814150795340538, 0.20931822061538696, 1...         1\n",
       "3  [-0.2327376276254654, -0.8353355824947357, 0.4...         1\n",
       "4  [0.1854836568236351, 0.673973988209452, 1.2295...         1"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_2[['vector', 'polarity']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_2, test_2 = train_test_split(tweets_2[['vector', 'polarity']], test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "157576\n"
     ]
    }
   ],
   "source": [
    "print(len(train_2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_2 = np.concatenate(train_2['vector'].values).reshape((len(train_2), dim))\n",
    "y_train_2 = train_2['polarity'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test_2 = np.concatenate(test_2['vector'].values).reshape((len(test_2), dim))\n",
    "y_test_2 = test_2['polarity'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression\n",
    "\n",
    "_Without regularization._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=1000,\n",
       "                   multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic_reg_2 = LogisticRegression(solver='lbfgs', max_iter = 1000)\n",
    "logistic_reg_2.fit(x_train_2, y_train_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7669188201248921\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = logistic_reg_2.score(x_test_2, y_test_2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Tuning regularization parameter._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# C is the inverse of the regularization strenght\n",
    "C = np.logspace(0, 4, 10)\n",
    "hyperparameters = dict(C=C)\n",
    "\n",
    "logistic_reg_2 = LogisticRegression(solver='lbfgs', max_iter = 500)\n",
    "# we now perform grid search to find optimal hyperparameter\n",
    "# we use cross-validation with 5 folds\n",
    "clf = GridSearchCV(logistic_reg_2, hyperparameters, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_logistic_2 = clf.fit(x_train_2, y_train_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best C: 1291.5496650148827\n"
     ]
    }
   ],
   "source": [
    "print('Best C:', best_logistic_2.best_estimator_.get_params()['C'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.766969589277555\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = best_logistic_2.score(x_test_2, y_test_2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ridge regression\n",
    "\n",
    "_Without regularization._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7657257450373153\n"
     ]
    }
   ],
   "source": [
    "ridge_clf_2 = RidgeClassifier().fit(x_train_2, y_train_2)\n",
    "print('Accuracy = {a}'.format(a = ridge_clf_2.score(x_test_2, y_test_2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Tuning regularization parameter._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# alpha is a constant that multiplies the regularization term\n",
    "alpha = np.logspace(0, 4, 10)\n",
    "hyperparameters = dict(alpha=alpha)\n",
    "\n",
    "ridge_clf_2 = RidgeClassifier()\n",
    "# we now perform grid search to find optimal hyperparameter\n",
    "# we use cross-validation with 5 folds\n",
    "clf = GridSearchCV(ridge_clf_2, hyperparameters, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_ridge_clf_2 = clf.fit(x_train_2, y_train_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best alpha: 59.94842503189409\n"
     ]
    }
   ],
   "source": [
    "print('Best alpha:', best_ridge_clf_2.best_estimator_.get_params()['alpha'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7659288216479667\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = best_ridge_clf_2.score(x_test_2, y_test_2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Support Vector Machine\n",
    "\n",
    "_Without hyperparameters tuning._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "sgd_clf_2 = SGDClassifier(loss = 'hinge').fit(x_train_2, y_train_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7657511296136468\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = sgd_clf_2.score(x_test_2, y_test_2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_With hyperparameters tuning._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# alpha is a constant that multiplies the regularization term\n",
    "alpha = np.logspace(0, 4, 10)\n",
    "hyperparameters = dict(alpha=alpha)\n",
    "\n",
    "svm_clf_2 = SGDClassifier(loss = 'hinge')\n",
    "# we now perform grid search to find optimal hyperparameter\n",
    "# we use cross-validation with 5 folds\n",
    "svm_clf = GridSearchCV(svm_clf_2, hyperparameters, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_svm_clf_2 = clf.fit(x_train_2, y_train_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best alpha: 59.94842503189409\n"
     ]
    }
   ],
   "source": [
    "print('Best alpha:', best_svm_clf_2.best_estimator_.get_params()['alpha'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7659288216479667\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = best_svm_clf_2.score(x_test_2, y_test_2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***Conclusion: Removing english stopwords decreases the performance.***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. tweets with tags <user\\> and <url\\> removed, and also with word stemming"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_We now remove stop words and add stemming too. We trained another Word2Vec model for this purpose._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_stemming = Word2Vec.load('word2vec_stemming.model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_The total number of entries in our Word2Vec model is now less because we used stemming._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "77889"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(model_stemming.wv.vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_We remove tags, stop words and perform stemming._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>polarity</th>\n",
       "      <th>tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i dunno justin read my mention or not . only j...</td>\n",
       "      <td>1</td>\n",
       "      <td>[i, dunno, justin, read, my, mention, or, not,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>because your logic is so dumb , i won't even c...</td>\n",
       "      <td>1</td>\n",
       "      <td>[becaus, your, logic, is, so, dumb, i, wo, nt,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>just put casper in a box !  looved the battle ...</td>\n",
       "      <td>1</td>\n",
       "      <td>[just, put, casper, in, a, box, loov, the, bat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>thanks sir &gt; &gt; don't trip lil mama ... just ke...</td>\n",
       "      <td>1</td>\n",
       "      <td>[thank, sir, do, nt, trip, lil, mama, just, ke...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>visiting my brother tmr is the bestest birthda...</td>\n",
       "      <td>1</td>\n",
       "      <td>[visit, my, brother, tmr, is, the, bestest, bi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               tweet  polarity  \\\n",
       "0  i dunno justin read my mention or not . only j...         1   \n",
       "1  because your logic is so dumb , i won't even c...         1   \n",
       "2  just put casper in a box !  looved the battle ...         1   \n",
       "3  thanks sir > > don't trip lil mama ... just ke...         1   \n",
       "4  visiting my brother tmr is the bestest birthda...         1   \n",
       "\n",
       "                                              tokens  \n",
       "0  [i, dunno, justin, read, my, mention, or, not,...  \n",
       "1  [becaus, your, logic, is, so, dumb, i, wo, nt,...  \n",
       "2  [just, put, casper, in, a, box, loov, the, bat...  \n",
       "3  [thank, sir, do, nt, trip, lil, mama, just, ke...  \n",
       "4  [visit, my, brother, tmr, is, the, bestest, bi...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_3 = remove_tags(tweets)\n",
    "tweets_3 = tokenize_tweets(tweets_3, stop_words = False, stemming = True)\n",
    "tweets_3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_3['vector'] = tweets_3['tokens'].apply(lambda tokens: toVector(tokens, model_stemming, dim))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vector</th>\n",
       "      <th>polarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[0.12418689959061642, -0.15489598460650691, 0....</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[0.8505662182966868, -0.33197376662347877, -0....</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[0.20243584281868404, -0.6361863786975542, -0....</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[0.6778068460213641, -0.27396117507790524, 0.1...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[0.343004809319973, -0.5250801883637906, 0.209...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              vector  polarity\n",
       "0  [0.12418689959061642, -0.15489598460650691, 0....         1\n",
       "1  [0.8505662182966868, -0.33197376662347877, -0....         1\n",
       "2  [0.20243584281868404, -0.6361863786975542, -0....         1\n",
       "3  [0.6778068460213641, -0.27396117507790524, 0.1...         1\n",
       "4  [0.343004809319973, -0.5250801883637906, 0.209...         1"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_3[['vector', 'polarity']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_3, test_3 = train_test_split(tweets_3[['vector', 'polarity']], test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "157576\n"
     ]
    }
   ],
   "source": [
    "print(len(train_3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_3 = np.concatenate(train_3['vector'].values).reshape((len(train_3), dim))\n",
    "y_train_3 = train_3['polarity'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test_3 = np.concatenate(test_3['vector'].values).reshape((len(test_3), dim))\n",
    "y_test_3 = test_3['polarity'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression\n",
    "\n",
    "_Without regularization._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=1000,\n",
       "                   multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic_reg_3 = LogisticRegression(solver='lbfgs', max_iter = 1000)\n",
    "logistic_reg_3.fit(x_train_3, y_train_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7776564959130832\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = logistic_reg_3.score(x_test_3, y_test_3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Tuning regularization parameter._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# C is the inverse of the regularization strenght\n",
    "C = np.logspace(0, 4, 10)\n",
    "hyperparameters = dict(C=C)\n",
    "\n",
    "logistic_reg_3 = LogisticRegression(solver='lbfgs', max_iter = 500)\n",
    "# we now perform grid search to find optimal hyperparameter\n",
    "# we use cross-validation with 5 folds\n",
    "clf = GridSearchCV(logistic_reg_3, hyperparameters, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_logistic_3 = clf.fit(x_train_3, y_train_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best C: 2.7825594022071245\n"
     ]
    }
   ],
   "source": [
    "print('Best C:', best_logistic_3.best_estimator_.get_params()['C'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7774534193024318\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = best_logistic_3.score(x_test_3, y_test_3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ridge regression\n",
    "\n",
    "_Without regularization._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7719195816621821\n"
     ]
    }
   ],
   "source": [
    "ridge_clf_3 = RidgeClassifier().fit(x_train_3, y_train_3)\n",
    "print('Accuracy = {a}'.format(a = ridge_clf_3.score(x_test_3, y_test_3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_Tuning regularization parameter._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# alpha is a constant that multiplies the regularization term\n",
    "alpha = np.logspace(0, 4, 10)\n",
    "hyperparameters = dict(alpha=alpha)\n",
    "\n",
    "ridge_clf_3 = RidgeClassifier()\n",
    "# we now perform grid search to find optimal hyperparameter\n",
    "# we use cross-validation with 5 folds\n",
    "clf = GridSearchCV(ridge_clf_3, hyperparameters, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_ridge_clf_3 = clf.fit(x_train_3, y_train_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best alpha: 59.94842503189409\n"
     ]
    }
   ],
   "source": [
    "print('Best alpha:', best_ridge_clf_3.best_estimator_.get_params()['alpha'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7713357364065594\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = best_ridge_clf_3.score(x_test_3, y_test_3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Support Vector Machine\n",
    "\n",
    "_Without hyperparameters tuning._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "sgd_clf_3 = SGDClassifier(loss = 'hinge').fit(x_train_3, y_train_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7748134233639641\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = sgd_clf_3.score(x_test_3, y_test_3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_With hyperparameters tuning._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# alpha is a constant that multiplies the regularization term\n",
    "alpha = np.logspace(0, 4, 10)\n",
    "hyperparameters = dict(alpha=alpha)\n",
    "\n",
    "svm_clf_3 = SGDClassifier(loss = 'hinge')\n",
    "# we now perform grid search to find optimal hyperparameter\n",
    "# we use cross-validation with 5 folds\n",
    "svm_clf = GridSearchCV(svm_clf_3, hyperparameters, cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_svm_clf_3 = clf.fit(x_train_3, y_train_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best alpha: 59.94842503189409\n"
     ]
    }
   ],
   "source": [
    "print('Best alpha:', best_svm_clf_3.best_estimator_.get_params()['alpha'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy = 0.7713357364065594\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy = {a}'.format(a = best_svm_clf_3.score(x_test_3, y_test_3)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}