Amazon_Reviews_Web_Scraping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-ijjJJA2KArD"
   },
   "source": [
    "## Web scrapping for reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:23:51.112956Z",
     "start_time": "2023-03-24T06:23:49.751855Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 72
    },
    "colab_type": "code",
    "id": "JM2cURE5wIz0",
    "outputId": "d4bd6064-3398-4096-fe71-33069cfc3765"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "import re\n",
    "import time\n",
    "from datetime import datetime\n",
    "import matplotlib.dates as mdates\n",
    "import matplotlib.ticker as ticker\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup as bs\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:23:53.794989Z",
     "start_time": "2023-03-24T06:23:51.121039Z"
    }
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup as bs\n",
    "\n",
    "url = \"https://www.amazon.in/Bajaj-Torque-New-Honeycomb-Technology/dp/B09R3QNGW5/ref=sr_1_1_sspa?crid=1DXZ4HMCXU7FH&keywords=bajaj+px+97+torque+room+36l+air+cooler&qid=1679637791&sprefix=%2Caps%2C200&sr=8-1-spons&sp_csd=d2lkZ2V0TmFtZT1zcF9hdGY&psc=1\"\n",
    "response = requests.get(url)\n",
    "\n",
    "soup = bs(response.content,\"html.parser\") # creating soup object to iterate over the extracted content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:55.678335Z",
     "start_time": "2023-03-24T06:23:53.798353Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "RcRQlGcZJmyb"
   },
   "outputs": [],
   "source": [
    "wm_title=[]  \n",
    "wm_date = []\n",
    "wm_content = []\n",
    "wm_rating = []\n",
    "\n",
    "for i in range(1,150):\n",
    "\n",
    "  link = \"https://www.amazon.in/Bajaj-Torque-New-Honeycomb-Technology/dp/B09R3QNGW5/ref=sr_1_1_sspa?crid=1DXZ4HMCXU7FH&keywords=bajaj+px+97+torque+room+36l+air+cooler&qid=1679637791&sprefix=%2Caps%2C200&sr=8-1-spons&sp_csd=d2lkZ2V0TmFtZT1zcF9hdGY&psc=1\"\n",
    "  soup = bs(response.content,\"html.parser\")# creating soup object to iterate over the extracted content \n",
    "\n",
    "  # extracting Review Title\n",
    "  title = soup.find_all('a',class_='review-title-content')\n",
    "  review_title = []\n",
    "  for i in range(0,len(title)):\n",
    "      review_title.append(title[i].get_text())\n",
    "  review_title[:] = [titles.lstrip('\\n') for titles in review_title]\n",
    "  review_title[:] = [titles.rstrip('\\n') for titles in review_title]\n",
    "  wm_title = wm_title + review_title\n",
    "\n",
    "  ## Extracting Ratings\n",
    "  rating = soup.find_all('i',class_='review-rating')\n",
    "  review_rating = []\n",
    "  for i in range(2,len(rating)):\n",
    "      review_rating.append(rating[i].get_text())\n",
    "  #review_rating.pop(0)\n",
    "  #review_rating.pop(0)\n",
    "  review_rating[:] = [reviews.rstrip(' out of 5 stars') for reviews in review_rating]\n",
    "  wm_rating = wm_rating + review_rating\n",
    "\n",
    "  #Extracting Content of review\n",
    "  review = soup.find_all(\"span\",{\"data-hook\":\"review-body\"})\n",
    "  review_content = []\n",
    "  for i in range(0,len(review)):\n",
    "      review_content.append(review[i].get_text())\n",
    "  review_content[:] = [reviews.lstrip('\\n') for reviews in review_content]\n",
    "  review_content[:] = [reviews.rstrip('\\n') for reviews in review_content]\n",
    "  wm_content = wm_content + review_content\n",
    "\n",
    "  #Extracting dates of reviews\n",
    "  dates = soup.find_all('span',class_='review-date')\n",
    "  review_dates = []\n",
    "  for i in range(2,len(rating)):\n",
    "      review_dates.append(dates[i].get_text())\n",
    "  review_dates[:] = [reviews.lstrip('Reviewed in India on') for reviews in review_dates]\n",
    "  #review_dates.pop(0)\n",
    "  #review_dates.pop(0)\n",
    "  wm_date  = wm_date + review_dates\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:55.710279Z",
     "start_time": "2023-03-24T06:24:55.694566Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 87
    },
    "colab_type": "code",
    "id": "444XG0UiWCLn",
    "outputId": "b5844bc6-f3b4-4f4a-e7e4-2c60cb5945c8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1192\n",
      "894\n",
      "1192\n",
      "894\n"
     ]
    }
   ],
   "source": [
    "print(len(wm_title))\n",
    "print(len(wm_rating))\n",
    "print(len(wm_content))\n",
    "print(len(wm_date))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:55.789305Z",
     "start_time": "2023-03-24T06:24:55.719390Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1192\n",
      "894\n",
      "1192\n",
      "894\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# check the lengths of the lists\n",
    "print(len(wm_title))\n",
    "print(len(wm_rating))\n",
    "print(len(wm_content))\n",
    "print(len(wm_date))\n",
    "\n",
    "# assign NaN to the missing ratings\n",
    "if len(wm_rating) < len(wm_title):\n",
    "    diff = len(wm_title) - len(wm_rating)\n",
    "    wm_rating += [float('nan')] * diff\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:26:45.719214Z",
     "start_time": "2023-03-24T06:26:45.672099Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 197
    },
    "colab_type": "code",
    "id": "Px_811J8YJLJ",
    "outputId": "ec025a44-9c5c-4c3f-905b-58526ca28929"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1192 1192 1192 894\n",
      "                                               Title Ratings  \\\n",
      "0  Platina on, Garmi gone! Efficient cooler for n...     4.0   \n",
      "1                                        Noise level     4.0   \n",
      "2                                   Worthy for value     4.0   \n",
      "3          Im okey with this, never loose your Money     4.0   \n",
      "4                           Damaged product received     4.0   \n",
      "\n",
      "                                            Comments                 Date  \n",
      "0  Summer and air cooler goes synonymously. As te...  🇮🇳 on 17 March 2023  \n",
      "1  Overall product is great , at this price you c...  🇮🇳 on 19 March 2023  \n",
      "2                                          Read more  🇮🇳 on 19 March 2023  \n",
      "3                      The media could not be loa...  🇮🇳 on 15 March 2023  \n",
      "4                      The media could not be loa...  🇮🇳 on 17 March 2023  \n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# create DataFrame\n",
    "df = pd.DataFrame()\n",
    "\n",
    "# check lengths and adjust if needed\n",
    "print(len(wm_title), len(wm_rating), len(wm_content), len(wm_date))\n",
    "if len(wm_title) != len(wm_rating) or len(wm_rating) != len(wm_content):\n",
    "    raise ValueError('Lengths of wm_title, wm_rating, and wm_content do not match')\n",
    "elif len(wm_date) < len(wm_title):\n",
    "    wm_date += [None] * (len(wm_title) - len(wm_date))\n",
    "\n",
    "# add columns to DataFrame\n",
    "df['Title'] = wm_title\n",
    "df['Ratings'] = wm_rating\n",
    "df['Comments'] = wm_content\n",
    "df['Date'] = wm_date\n",
    "\n",
    "# print the first 5 rows of the DataFrame\n",
    "print(df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:28:23.180348Z",
     "start_time": "2023-03-24T06:28:23.149378Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 107
    },
    "colab_type": "code",
    "id": "EAMZ8Ga0eXdV",
    "outputId": "5447c128-300a-4a48-f513-dd4c70765740"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Ratings</th>\n",
       "      <th>Comments</th>\n",
       "      <th>Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Platina on, Garmi gone! Efficient cooler for n...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Summer and air cooler goes synonymously. As te...</td>\n",
       "      <td>🇮🇳 on 17 March 2023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Noise level</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Overall product is great , at this price you c...</td>\n",
       "      <td>🇮🇳 on 19 March 2023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Worthy for value</td>\n",
       "      <td>4.0</td>\n",
       "      <td>Read more</td>\n",
       "      <td>🇮🇳 on 19 March 2023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Im okey with this, never loose your Money</td>\n",
       "      <td>4.0</td>\n",
       "      <td>The media could not be loa...</td>\n",
       "      <td>🇮🇳 on 15 March 2023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Damaged product received</td>\n",
       "      <td>4.0</td>\n",
       "      <td>The media could not be loa...</td>\n",
       "      <td>🇮🇳 on 17 March 2023</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Title Ratings  \\\n",
       "0  Platina on, Garmi gone! Efficient cooler for n...     4.0   \n",
       "1                                        Noise level     4.0   \n",
       "2                                   Worthy for value     4.0   \n",
       "3          Im okey with this, never loose your Money     4.0   \n",
       "4                           Damaged product received     4.0   \n",
       "\n",
       "                                            Comments                 Date  \n",
       "0  Summer and air cooler goes synonymously. As te...  🇮🇳 on 17 March 2023  \n",
       "1  Overall product is great , at this price you c...  🇮🇳 on 19 March 2023  \n",
       "2                                          Read more  🇮🇳 on 19 March 2023  \n",
       "3                      The media could not be loa...  🇮🇳 on 15 March 2023  \n",
       "4                      The media could not be loa...  🇮🇳 on 17 March 2023  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:39:22.918171Z",
     "start_time": "2023-03-24T06:39:22.746953Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 107
    },
    "colab_type": "code",
    "id": "3ATDYJFuuFJU",
    "outputId": "9e8b6232-5486-4183-c747-25f25015b4ca",
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'Date'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py:3621\u001b[0m, in \u001b[0;36mIndex.get_loc\u001b[1;34m(self, key, method, tolerance)\u001b[0m\n\u001b[0;32m   3620\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m-> 3621\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_loc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcasted_key\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   3622\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\_libs\\index.pyx:136\u001b[0m, in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\_libs\\index.pyx:163\u001b[0m, in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32mpandas\\_libs\\hashtable_class_helper.pxi:5198\u001b[0m, in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32mpandas\\_libs\\hashtable_class_helper.pxi:5206\u001b[0m, in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'Date'",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Input \u001b[1;32mIn [19]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mDate\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mto_datetime(\u001b[43mdf\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mDate\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m)\n\u001b[0;32m      2\u001b[0m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mRatings\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m df[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mRatings\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mastype(\u001b[38;5;28mfloat\u001b[39m)\n\u001b[0;32m      3\u001b[0m df\u001b[38;5;241m.\u001b[39mhead(\u001b[38;5;241m2\u001b[39m)\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\core\\frame.py:3505\u001b[0m, in \u001b[0;36mDataFrame.__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   3503\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcolumns\u001b[38;5;241m.\u001b[39mnlevels \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[0;32m   3504\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_getitem_multilevel(key)\n\u001b[1;32m-> 3505\u001b[0m indexer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcolumns\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_loc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m   3506\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m is_integer(indexer):\n\u001b[0;32m   3507\u001b[0m     indexer \u001b[38;5;241m=\u001b[39m [indexer]\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py:3623\u001b[0m, in \u001b[0;36mIndex.get_loc\u001b[1;34m(self, key, method, tolerance)\u001b[0m\n\u001b[0;32m   3621\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_engine\u001b[38;5;241m.\u001b[39mget_loc(casted_key)\n\u001b[0;32m   3622\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[1;32m-> 3623\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mKeyError\u001b[39;00m(key) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01merr\u001b[39;00m\n\u001b[0;32m   3624\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m:\n\u001b[0;32m   3625\u001b[0m     \u001b[38;5;66;03m# If we have a listlike key, _check_indexing_error will raise\u001b[39;00m\n\u001b[0;32m   3626\u001b[0m     \u001b[38;5;66;03m#  InvalidIndexError. Otherwise we fall through and re-raise\u001b[39;00m\n\u001b[0;32m   3627\u001b[0m     \u001b[38;5;66;03m#  the TypeError.\u001b[39;00m\n\u001b[0;32m   3628\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_check_indexing_error(key)\n",
      "\u001b[1;31mKeyError\u001b[0m: 'Date'"
     ]
    }
   ],
   "source": [
    "df['Date'] = pd.to_datetime(df['Date'])\n",
    "df['Ratings'] = df['Ratings'].astype(float)\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "eUJww4_LUn2-"
   },
   "source": [
    "## Text Cleaning\n",
    "\n",
    "1. lower the text\n",
    "2. tokenize the text (split the text into words) and remove the punctuation\n",
    "3. remove useless words that contain numbers\n",
    "4. remove useless stop words like ‘the’, ‘a’ ,’this’ etc.\n",
    "5. Part-Of-Speech (POS) tagging: assign a tag to every word to define 6. if it corresponds to a noun, a verb etc. using the WordNet lexical database\n",
    "7. lemmatize the text: transform every word into their root form (e.g. rooms -> room, slept -> sleep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:58.228372Z",
     "start_time": "2023-03-24T06:24:58.228372Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 107
    },
    "colab_type": "code",
    "id": "bTARqziecwPd",
    "outputId": "3b06e8bd-bdbb-4955-9764-5349f9c9e1b2"
   },
   "outputs": [],
   "source": [
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:58.228372Z",
     "start_time": "2023-03-24T06:24:58.228372Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 176
    },
    "colab_type": "code",
    "id": "6IfyN5S7uFiA",
    "outputId": "8217795d-df69-4850-b2d5-7f89a6d71f17"
   },
   "outputs": [],
   "source": [
    "from nltk.corpus import wordnet\n",
    "\n",
    "def get_wordnet_pos(pos_tag):\n",
    "    if pos_tag.startswith('J'):\n",
    "        return wordnet.ADJ\n",
    "    elif pos_tag.startswith('V'):\n",
    "        return wordnet.VERB\n",
    "    elif pos_tag.startswith('N'):\n",
    "        return wordnet.NOUN\n",
    "    elif pos_tag.startswith('R'):\n",
    "        return wordnet.ADV\n",
    "    else:\n",
    "        return wordnet.NOUN\n",
    "\n",
    "import string\n",
    "import nltk\n",
    "from nltk import pos_tag\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import WhitespaceTokenizer\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "nltk.download('stopwords')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('vader_lexicon')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:58.228372Z",
     "start_time": "2023-03-24T06:24:58.228372Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "XkxYXLymLNR5"
   },
   "outputs": [],
   "source": [
    "def clean_text(text):\n",
    "    # lower text\n",
    "    text = text.lower()\n",
    "    # tokenize text and remove puncutation\n",
    "    text = [word.strip(string.punctuation) for word in text.split(\" \")]\n",
    "    # remove words that contain numbers\n",
    "    text = [word for word in text if not any(c.isdigit() for c in word)]\n",
    "    # remove stop words\n",
    "    stop = stopwords.words('english')\n",
    "    text = [x for x in text if x not in stop]\n",
    "    # remove empty tokens\n",
    "    text = [t for t in text if len(t) > 0]\n",
    "    # pos tag text\n",
    "    pos_tags = pos_tag(text)\n",
    "    # lemmatize text\n",
    "    text = [WordNetLemmatizer().lemmatize(t[0], get_wordnet_pos(t[1])) for t in pos_tags]\n",
    "    # remove words with only one letter\n",
    "    text = [t for t in text if len(t) > 1]\n",
    "    # join all\n",
    "    text = \" \".join(text)\n",
    "    return(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:58.244018Z",
     "start_time": "2023-03-24T06:24:58.244018Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "NJl5EJmUv8Bd"
   },
   "outputs": [],
   "source": [
    "# clean text data\n",
    "df[\"Comments\"] = df[\"Comments\"].apply(lambda x: clean_text(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:58.244018Z",
     "start_time": "2023-03-24T06:24:58.244018Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "fglUhK31OcXe"
   },
   "outputs": [],
   "source": [
    "df['Title'] = df['Title'].astype(str)\n",
    "df['Title'] = df['Title'].apply(lambda x: clean_text(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:58.244018Z",
     "start_time": "2023-03-24T06:24:58.244018Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 197
    },
    "colab_type": "code",
    "id": "py9plvt7R63B",
    "outputId": "eebf36e1-aeb7-46b5-b63a-46360263fa28"
   },
   "outputs": [],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "WLR5wxHFVE3E"
   },
   "source": [
    "## Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:58.244018Z",
     "start_time": "2023-03-24T06:24:58.244018Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 108
    },
    "colab_type": "code",
    "id": "GrutSs82UcEo",
    "outputId": "f521365e-ad01-46f4-90a4-9712c9c5cf58"
   },
   "outputs": [],
   "source": [
    "#  add sentiment anaylsis columns\n",
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
    "\n",
    "sid = SentimentIntensityAnalyzer()\n",
    "df[\"sentiments\"] = df[\"Comments\"].apply(lambda x: sid.polarity_scores(x))\n",
    "df = pd.concat([df.drop(['sentiments'], axis=1), df['sentiments'].apply(pd.Series)], axis=1)\n",
    "'''\n",
    "sid = SentimentIntensityAnalyzer()\n",
    "df[\"sentiments_title\"] = df[\"Title\"].apply(lambda x: sid.polarity_scores(x))\n",
    "df = pd.concat([df.drop(['sentiments_title'], axis=1), df['sentiments_title'].apply(pd.Series)], axis=1)\n",
    "'''\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:58.244018Z",
     "start_time": "2023-03-24T06:24:58.244018Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 53
    },
    "colab_type": "code",
    "id": "zu8HmO0xUcJb",
    "outputId": "ca8f51da-ed6b-424e-96f4-83fa7e7b5ef7"
   },
   "outputs": [],
   "source": [
    "# add number of characters column\n",
    "df[\"nb_chars\"] = df[\"Comments\"].apply(lambda x: len(x))\n",
    "\n",
    "# add number of words column\n",
    "df[\"nb_words\"] = df[\"Comments\"].apply(lambda x: len(x.split(\" \")))\n",
    "\n",
    "''''\n",
    "# add number of characters column\n",
    "df[\"nb_chars_title\"] = df[\"Title\"].apply(lambda x: len(x))\n",
    "\n",
    "# add number of words column\n",
    "df[\"nb_words_title\"] = df[\"Title\"].apply(lambda x: len(x.split(\" \")))\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:58.244018Z",
     "start_time": "2023-03-24T06:24:58.244018Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 53
    },
    "colab_type": "code",
    "id": "8lo7DglXUcNd",
    "outputId": "2f4662f2-7f9a-448e-9374-1ed48f5a603e"
   },
   "outputs": [],
   "source": [
    "# create doc2vec vector columns\n",
    "from gensim.test.utils import common_texts\n",
    "from gensim.models.doc2vec import Doc2Vec, TaggedDocument\n",
    "\n",
    "documents = [TaggedDocument(doc, [i]) for i, doc in enumerate(df[\"Comments\"].apply(lambda x: x.split(\" \")))]\n",
    "\n",
    "# train a Doc2Vec model with our text data\n",
    "model = Doc2Vec(documents, vector_size=5, window=2, min_count=1, workers=4)\n",
    "\n",
    "# transform each Comment into a vector data\n",
    "doc2vec_df = df[\"Comments\"].apply(lambda x: model.infer_vector(x.split(\" \"))).apply(pd.Series)\n",
    "doc2vec_df.columns = [\"doc2vec_vector_\" + str(x) for x in doc2vec_df.columns]\n",
    "df = pd.concat([df, doc2vec_df], axis=1)\n",
    "\n",
    "'''\n",
    "# transform each Title into a vector data\n",
    "doc2vec_df_title = df[\"Title\"].apply(lambda x: model.infer_vector(x.split(\" \"))).apply(pd.Series)\n",
    "doc2vec_df_title.columns = [\"doc2vec_vector_\" + str(x) for x in doc2vec_df.columns]\n",
    "df = pd.concat([df, doc2vec_df_title], axis=1)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:58.244018Z",
     "start_time": "2023-03-24T06:24:58.244018Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 53
    },
    "colab_type": "code",
    "id": "hmtlc9X3YMko",
    "outputId": "d2e5fce1-a1c4-40ea-cbdf-2e72cf81b1e8"
   },
   "outputs": [],
   "source": [
    "# add tf-idfs columns\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "tfidf = TfidfVectorizer(min_df = 10)\n",
    "tfidf_result = tfidf.fit_transform(df[\"Comments\"]).toarray()\n",
    "tfidf_df = pd.DataFrame(tfidf_result, columns = tfidf.get_feature_names())\n",
    "tfidf_df.columns = [\"word_\" + str(x) for x in tfidf_df.columns]\n",
    "tfidf_df.index = df.index\n",
    "df = pd.concat([df, tfidf_df], axis=1)\n",
    "\n",
    "'''\n",
    "##TF-IDF for Titles\n",
    "tfidf = TfidfVectorizer(min_df = 10)\n",
    "tfidf_result = tfidf.fit_transform(df[\"Title\"]).toarray()\n",
    "tfidf_df_title = pd.DataFrame(tfidf_result, columns = tfidf.get_feature_names())\n",
    "tfidf_df_title.columns = [\"word_\" + str(x) for x in tfidf_df_title.columns]\n",
    "tfidf_df_title.index = df.index\n",
    "df = pd.concat([df, tfidf_df_title], axis=1)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:58.244018Z",
     "start_time": "2023-03-24T06:24:58.244018Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 588
    },
    "colab_type": "code",
    "id": "6PYdEdMVYkxk",
    "outputId": "54e0ec41-37d9-4b37-a5d8-e1e2c9a22113"
   },
   "outputs": [],
   "source": [
    "from wordcloud import WordCloud\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def show_wordcloud(data, title = None):\n",
    "    wordcloud = WordCloud(\n",
    "        background_color = 'white',\n",
    "        max_words = 200,\n",
    "        max_font_size = 40, \n",
    "        scale = 3,\n",
    "        random_state = 42\n",
    "    ).generate(str(data))\n",
    "\n",
    "    fig = plt.figure(1, figsize = (20, 20))\n",
    "    plt.axis('off')\n",
    "    if title: \n",
    "        fig.suptitle(title, fontsize = 20)\n",
    "        fig.subplots_adjust(top = 2.3)\n",
    "\n",
    "    plt.imshow(wordcloud)\n",
    "    plt.show()\n",
    "    \n",
    "# print wordcloud\n",
    "show_wordcloud(df[\"Comments\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:58.244018Z",
     "start_time": "2023-03-24T06:24:58.244018Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 588
    },
    "colab_type": "code",
    "id": "yjexZHO2Yk6e",
    "outputId": "6d86e8e4-d568-48dc-a0e7-a74c6b8d428b"
   },
   "outputs": [],
   "source": [
    "# print wordcloud\n",
    "show_wordcloud(df[\"Title\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:58.244018Z",
     "start_time": "2023-03-24T06:24:58.244018Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 306
    },
    "colab_type": "code",
    "id": "B8IpuSE_N6sa",
    "outputId": "cd7990ff-c4b2-4381-df18-4cd691a5c741"
   },
   "outputs": [],
   "source": [
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:58.263831Z",
     "start_time": "2023-03-24T06:24:58.263831Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 347
    },
    "colab_type": "code",
    "id": "-9FczFNwYlBw",
    "outputId": "6b5e0145-ec21-4ce6-83d8-1d4c1e14b995"
   },
   "outputs": [],
   "source": [
    "# highest positive sentiment reviews (with more than 5 words)\n",
    "df[df[\"nb_words\"] >= 5].sort_values(\"pos\", ascending = False)[[\"Comments\", \"pos\"]].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:58.265903Z",
     "start_time": "2023-03-24T06:24:58.265903Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 347
    },
    "colab_type": "code",
    "id": "ub3jiGgyYlRK",
    "outputId": "238160ce-04e7-4a02-df3a-4ef7ad6fe93a"
   },
   "outputs": [],
   "source": [
    "# lowest negative sentiment reviews (with more than 5 words)\n",
    "df[df[\"nb_words\"] >= 5].sort_values(\"neg\", ascending = False)[[\"Comments\", \"neg\"]].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:58.268140Z",
     "start_time": "2023-03-24T06:24:58.268140Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 306
    },
    "colab_type": "code",
    "id": "Fw5je005Yk-L",
    "outputId": "219cf63f-435f-4f4a-9110-dcf23a78e44b"
   },
   "outputs": [],
   "source": [
    "df['Month'] = df['Date'].dt.month\n",
    "df['Year'] = df['Date'].dt.year\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-24T06:24:58.269982Z",
     "start_time": "2023-03-24T06:24:58.269982Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 356
    },
    "colab_type": "code",
    "id": "Cgc_Qe3bhUvc",
    "outputId": "e44708e1-b901-44ea-c294-1f49f91e6cf2"
   },
   "outputs": [],
   "source": [
    "df_recent = df[(df['Year']== 2020) & (df['Month'] != 8)]\n",
    "df_recent.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sadImUqnS6h9"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "name": "Project-II.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}