classifier_language_recognition-tensor-flow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As usual we start loading the packages that we will use in our notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import model_selection\n",
    "from sklearn.preprocessing import LabelEncoder \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#PRINT VERSION!!\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>language</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Jean Beauverie (Fontaines-sur-Saône, 18 febbra...</td>\n",
       "      <td>italian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Il pinguino saltarocce (Eudyptes chrysocome (F...</td>\n",
       "      <td>italian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Maison Ikkoku - Cara dolce Kyoko (めぞん一刻 Mezon ...</td>\n",
       "      <td>italian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>La mia città è un singolo della cantante itali...</td>\n",
       "      <td>italian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>L'Armata Rossa dei Lavoratori e dei Contadini ...</td>\n",
       "      <td>italian</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            sentence language\n",
       "0  Jean Beauverie (Fontaines-sur-Saône, 18 febbra...  italian\n",
       "1  Il pinguino saltarocce (Eudyptes chrysocome (F...  italian\n",
       "2  Maison Ikkoku - Cara dolce Kyoko (めぞん一刻 Mezon ...  italian\n",
       "3  La mia città è un singolo della cantante itali...  italian\n",
       "4  L'Armata Rossa dei Lavoratori e dei Contadini ...  italian"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.read_csv(\"train_languages.csv\")#here we have the dataset we extracted\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3633"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_df) #we print the length, not a big one but sufficient"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A key step is to label encode the target variable from text to number\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y = train_df['language']\n",
    "encoder = LabelEncoder()\n",
    "encoder.fit(Y)\n",
    "Y = encoder.transform(Y)\n",
    "Y = tf.keras.utils.to_categorical(\n",
    "    Y,\n",
    "    num_classes=4 #equals to the number of languages\n",
    "    \n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we mentioned in the slides we will perform the previous text processing steps except for stopword removal."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df['sentence_lower'] = train_df[\"sentence\"].str.lower()\n",
    "train_df['sentence_no_punctuation'] = train_df['sentence_lower'].str.replace('[^\\w\\s]','')\n",
    "train_df['sentence_no_punctuation'] = train_df[\"sentence_no_punctuation\"].fillna(\"fillna\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_features=5000 #we set maximum number of words to 5000\n",
    "maxlen=400 #we set maximum sequence length to 400"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok = tf.keras.preprocessing.text.Tokenizer(num_words=max_features) #again tokenizer step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok.fit_on_texts(list(train_df['sentence_no_punctuation'])) #fit to cleaned text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "51630\n"
     ]
    }
   ],
   "source": [
    "print(len(tok.word_index))\n",
    "vocab_size = len(tok.word_index) + 1 \n",
    "#this represents the number of words that we tokenize different from max_features but necessary for\n",
    "#the definition of the dimension of the embedding space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = tok.texts_to_sequences(list(train_df['sentence_no_punctuation'])) #this is how we create sequences\n",
    "train_df = tf.keras.preprocessing.sequence.pad_sequences(train_df, maxlen=maxlen) #let's execute pad step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from sklearn.model_selection import train_test_split #divide into train and test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(train_df, Y, test_size=0.1, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_dim = 50 #this is the final dimension of the embedding space.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's write down the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.models.Sequential([\n",
    "  tf.keras.layers.Embedding(input_dim=vocab_size, #embedding input\n",
    "                           output_dim=embedding_dim,#embedding output\n",
    "                           input_length=maxlen), #maximum length of an input sequence\n",
    "  tf.keras.layers.Flatten(), #flatten layer\n",
    "\n",
    "  tf.keras.layers.Dense(4, activation=tf.nn.softmax) #ouput layer a Dense layer with 4 probabilities\n",
    "  #we also define our final activation function which is the softmax function typical for multiclass\n",
    "  #classifiction problems\n",
    "\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='adam',\n",
    "              loss='categorical_crossentropy', #we recommend this loss function you\n",
    "              metrics=['accuracy'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 400, 50)           2581550   \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 20000)             0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 4)                 80004     \n",
      "=================================================================\n",
      "Total params: 2,661,554\n",
      "Trainable params: 2,661,554\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary() #here we show the architecture "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 3269 samples\n",
      "Epoch 1/3\n",
      "3269/3269 [==============================] - 4s 1ms/sample - loss: 0.8173 - accuracy: 0.7372\n",
      "Epoch 2/3\n",
      "3269/3269 [==============================] - 3s 980us/sample - loss: 0.0766 - accuracy: 0.9951\n",
      "Epoch 3/3\n",
      "3269/3269 [==============================] - 3s 996us/sample - loss: 0.0255 - accuracy: 0.9979\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1dadaa4c2e8>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(np.array(X_train), np.array(y_train), epochs=3) #let's fit the model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remember the train_test_split? now we use the test to evaluate our model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "364/1 [========================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================] - 0s 372us/sample - loss: 0.0088 - accuracy: 1.0000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.012563308880019646, 1.0]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(np.array(X_test), np.array(y_test)) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LOOKS LIKE WE HAVE A PERFECT MODEL!! \n",
    "LET'S TAKE A LOOK AT THE CONFUSION MATRIX OF OUR EVALUATION SET!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix #we import this package from sklearn and output it\n",
    "predictions = model.predict(X_test) #here we make predictions\n",
    "cm = confusion_matrix(predictions.argmax(axis=1), y_test.argmax(axis=1))#we generate the confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 91,   0,   0,   0],\n",
       "       [  0,  88,   0,   0],\n",
       "       [  0,   0, 102,   0],\n",
       "       [  0,   0,   0,  83]], dtype=int64)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm #well this is really perfect!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's try brand new text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "english [0]\n",
      "french [1]\n",
      "italian [2]\n",
      "spanish [3]\n"
     ]
    }
   ],
   "source": [
    "#these are the codes for each language in order to evaluate properly\n",
    "print('english', encoder.transform(['english']))\n",
    "print('french', encoder.transform(['french']))\n",
    "print('italian', encoder.transform(['italian']))\n",
    "print('spanish', encoder.transform(['spanish']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this experiment we will predict the language of the same sentence in the different languages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#new_text = [\"tensorflow is a great tool you can find a lot of tutorials from packt\"]\n",
    "#new_text = [\"tensorflow est un excellent outil vous pouvez trouver beaucoup de tutoriels de packt\"]\n",
    "#new_text = [\"tensorflow è un ottimo strumento puoi trovare molti tutorial di packt\"]\n",
    "new_text = [\"tensorflow es una gran herramienta puedes encontrar muchos tutoriales de packt\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_text = tok.texts_to_sequences(new_text) #this is how we create sequences\n",
    "test_text = tf.keras.preprocessing.sequence.pad_sequences(test_text, maxlen=maxlen) #let's execute pad step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n",
      "[[0.06981006 0.11323404 0.11446167 0.7024943 ]]\n"
     ]
    }
   ],
   "source": [
    "np.set_printoptions(suppress=True)\n",
    "predictions = model.predict(test_text)\n",
    "print(predictions.argmax())\n",
    "print(predictions) #spanish you can get confused with italian which makes sense since they are more similar languages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import wikipedia"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's build a brand new data set with only spanish and let's see if we recognize it ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n"
     ]
    }
   ],
   "source": [
    "#language codes\n",
    "#english: en\n",
    "#italian: it\n",
    "#french: fr\n",
    "#spanish: es\n",
    "new_wiki_text = []\n",
    "wikipedia.set_lang('es')\n",
    "for i in range(0, 5):\n",
    "    print(i)\n",
    "    random = wikipedia.random(1)\n",
    "       \n",
    "    try:\n",
    "        new_wiki_text.append([wikipedia.page(random).summary])\n",
    "    except wikipedia.exceptions.DisambiguationError as e:\n",
    "        random = wikipedia.random(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Chlöe Howl (4 de marzo de 1995) es una cantant...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>La Misión de Kolobeng (en inglés: Kolobeng Mis...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Microscapha baloghi es una especie de coleópte...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Valentine es un lugar designado por el censo u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Jonathan Niven Cryer (Nueva York, 16 de abril ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            sentence\n",
       "0  Chlöe Howl (4 de marzo de 1995) es una cantant...\n",
       "1  La Misión de Kolobeng (en inglés: Kolobeng Mis...\n",
       "2  Microscapha baloghi es una especie de coleópte...\n",
       "3  Valentine es un lugar designado por el censo u...\n",
       "4  Jonathan Niven Cryer (Nueva York, 16 de abril ..."
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_wiki_text = pd.DataFrame(new_wiki_text)\n",
    "new_wiki_text.columns = ['sentence']\n",
    "new_wiki_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_wiki_text['sentence_lower'] = new_wiki_text[\"sentence\"].str.lower()\n",
    "new_wiki_text['sentence_no_punctuation'] = new_wiki_text['sentence_lower'].str.replace('[^\\w\\s]','')\n",
    "new_wiki_text['sentence_no_punctuation'] = new_wiki_text[\"sentence_no_punctuation\"].fillna(\"fillna\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.set_printoptions(suppress=True)\n",
    "test_wiki_text = tok.texts_to_sequences(list(new_wiki_text['sentence_no_punctuation'] )) #this is how we create sequences\n",
    "test_wiki_text = tf.keras.preprocessing.sequence.pad_sequences(test_wiki_text, maxlen=maxlen) #let's execute pad step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.00046239 0.00053851 0.00004136 0.99895775]\n",
      " [0.00000421 0.00001063 0.00000142 0.9999838 ]\n",
      " [0.00399885 0.01329104 0.0046148  0.9780953 ]\n",
      " [0.00003278 0.00010141 0.00000824 0.99985754]\n",
      " [0.00002722 0.00032142 0.00000575 0.9996456 ]]\n"
     ]
    }
   ],
   "source": [
    "predictions = model.predict(test_wiki_text)\n",
    "print(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "english [0]\n",
      "french [1]\n",
      "italian [2]\n",
      "spanish [3]\n"
     ]
    }
   ],
   "source": [
    "print('english', encoder.transform(['english']))\n",
    "print('french', encoder.transform(['french']))\n",
    "print('italian', encoder.transform(['italian']))\n",
    "print('spanish', encoder.transform(['spanish']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "WE DID A GOOD JOB!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}