Fetching Medium Articles Using Beautiful Soup and Selenium.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://github.com/Sangarshanan/webscraping-and-analysis-of-medium-articles/blob/master/Medium%20Article%20Scraping.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.request\n",
    "import requests\n",
    "import bs4\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "from selenium import webdriver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Links\n",
    "arificial_intelligence = \"https://medium.com/tag/artificial-intelligence\"\n",
    "machine_learning = \"https://medium.com/tag/machine-learning\"\n",
    "deep_learning = \"https://medium.com/tag/deep-learning\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Chrome()\n",
    "driver.get(arificial_intelligence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = driver.execute_script(\"return document.documentElement.outerHTML\")\n",
    "driver.quit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(res, 'lxml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "work=[]\n",
    "for my_tag in soup.find_all(class_=\"ds-link ds-link--styleSubtle link--darken link--accent u-accentColor--textNormal\"):\n",
    "    work.append(my_tag.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "para=[]\n",
    "for my_tag in soup.find_all(True,{'class':['graf graf--h4 graf-after--h3 graf--trailing graf--subtitle','graf graf--p graf-after--figure','graf graf--p graf-after--h3 graf--trailing']}):\n",
    "    para.append(my_tag.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "title=[]\n",
    "for my_tag in soup.find_all(True,{'class':['graf graf--h3 graf-after--figure graf--title','graf graf--h3 graf--leading graf--title','graf graf--h3 graf-after--figure graf--trailing graf--title']}):\n",
    "    title.append(my_tag.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "read=[]\n",
    "for my_tag in soup.find_all(class_=\"readingTime\"):\n",
    "    read.append(my_tag.get('title'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "upvotes=[]\n",
    "for my_tag in soup.find_all('span',{'class':'u-textAlignCenter u-relative u-background js-actionMultirecommendCount u-marginLeft5'}):\n",
    "    upvotes.append(my_tag.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "date=[]\n",
    "for my_tag in soup.find_all('time'):\n",
    "    date.append(my_tag.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = []\n",
    "for my_tag in soup.find_all(class_=\"ds-link ds-link--styleSubtle link link--darken link--accent u-accentColor--textNormal u-accentColor--textDarken\"):\n",
    "    name.append(my_tag.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "content=[]\n",
    "alllink=[]\n",
    "for my_tag in soup.find_all(class_=\"postArticle postArticle--short js-postArticle js-trackPostPresentation js-trackPostScrolls\"):\n",
    "        content.append(my_tag.text)\n",
    "        for link in my_tag.find_all('a'):\n",
    "            alllink.append(link.get('href'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The Most Undervalued Standard Python\\xa0Library'"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "title[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "for each in alllink:\n",
    "    if title[0][0:10] in each:\n",
    "        print(each)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'the most undervalued standard python\\xa0library'"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "title[0].lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://towardsdatascience.com/the-most-undervalued-standard-python-library-14021632f692?source=---------0-----------------------'"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alllink[4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(0,len(content)):\n",
    "    content[i]=content[i].replace(name[i],'')\n",
    "    content[i]=content[i].replace(date[i],'')\n",
    "    content[i]=content[i].replace(title[i],'')\n",
    "    content[i]=content[i].replace('…Read more…','*')\n",
    "    content[i]=content[i].replace('Read more…','*')\n",
    "    content[i]=content[i].replace('\\xa0',' ')\n",
    "    content[i]=content[i].replace(' in ', '')\n",
    "    for tag in work:\n",
    "        content[i] = content[i].replace(tag, '')\n",
    "    content[i]=content[i].split('*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "body=[]\n",
    "for i in range(0,len(content)):\n",
    "    body.append(content[i][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "res =[]\n",
    "for i in range(0,len(content)):\n",
    "    if (len(content[i]) == 1):\n",
    "        res.append('')\n",
    "    else:\n",
    "        res.append(content[i][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "links=[]\n",
    "for my_tag in soup.find_all(class_=\"button button--smaller button--chromeless u-baseColor--buttonNormal\"):\n",
    "        links.append(my_tag.get('href'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "data = pd.DataFrame({'Tag':'Artificial Intelligence','Name':name,'Title':title,'Body':body,'Date':date,'Comments':res})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tag</th>\n",
       "      <th>Name</th>\n",
       "      <th>Title</th>\n",
       "      <th>Body</th>\n",
       "      <th>Date</th>\n",
       "      <th>Comments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Tyler Folkman</td>\n",
       "      <td>The Most Undervalued Standard Python Library</td>\n",
       "      <td>Collections for data scientists</td>\n",
       "      <td>Oct 26</td>\n",
       "      <td>1.7K9 responses</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Thomas G. Dietterich</td>\n",
       "      <td>What does it mean for a machine to “understand”?</td>\n",
       "      <td>Critics of recent advancesartificial intellige...</td>\n",
       "      <td>Oct 27</td>\n",
       "      <td>6143 responses</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Jesus Rodriguez</td>\n",
       "      <td>Netflix Open Sources Polynote to Make Data Sci...</td>\n",
       "      <td>Notebooks are the data scientist</td>\n",
       "      <td>Oct 24</td>\n",
       "      <td>1.1K1 response</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Cassie Kozyrkov</td>\n",
       "      <td>Data Science’s Most Misunderstood Hero</td>\n",
       "      <td>Why treating analytics like a second-class cit...</td>\n",
       "      <td>Oct 19</td>\n",
       "      <td>2.5K12 responses</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Jayne Williamson-Lee</td>\n",
       "      <td>Amazon’s A.I. Emotion-Recognition Software Con...</td>\n",
       "      <td>People’s faces say a lot</td>\n",
       "      <td>Oct 27</td>\n",
       "      <td>3453 responses</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Dario Radečić</td>\n",
       "      <td>3 Pandas Functions You Should Be Using More Often</td>\n",
       "      <td>Stop reinventing the wheel. Seriously.</td>\n",
       "      <td>Oct 23</td>\n",
       "      <td>5154 responses</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Dario Radečić</td>\n",
       "      <td>IPython Notebook Support is Finally Here for V...</td>\n",
       "      <td></td>\n",
       "      <td>Oct 21</td>\n",
       "      <td>74710 responses</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Luke Posey</td>\n",
       "      <td>5 Essential AI Books</td>\n",
       "      <td>5 books on Artificial Intelligence that I’ve r...</td>\n",
       "      <td>Oct 27</td>\n",
       "      <td>911 response</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Dario Radečić</td>\n",
       "      <td>Read Text from Image with One Line of Python Code</td>\n",
       "      <td></td>\n",
       "      <td>Oct 28</td>\n",
       "      <td>431 response</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Artificial Intelligence</td>\n",
       "      <td>Lance Ulanoff</td>\n",
       "      <td>Did Google Duplex just pass the Turing Test?</td>\n",
       "      <td>This conversational AI changes everything</td>\n",
       "      <td>May 8, 2018</td>\n",
       "      <td>23K88 responses</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       Tag                  Name  \\\n",
       "0  Artificial Intelligence         Tyler Folkman   \n",
       "1  Artificial Intelligence  Thomas G. Dietterich   \n",
       "2  Artificial Intelligence       Jesus Rodriguez   \n",
       "3  Artificial Intelligence       Cassie Kozyrkov   \n",
       "4  Artificial Intelligence  Jayne Williamson-Lee   \n",
       "5  Artificial Intelligence         Dario Radečić   \n",
       "6  Artificial Intelligence         Dario Radečić   \n",
       "7  Artificial Intelligence            Luke Posey   \n",
       "8  Artificial Intelligence         Dario Radečić   \n",
       "9  Artificial Intelligence         Lance Ulanoff   \n",
       "\n",
       "                                               Title  \\\n",
       "0       The Most Undervalued Standard Python Library   \n",
       "1   What does it mean for a machine to “understand”?   \n",
       "2  Netflix Open Sources Polynote to Make Data Sci...   \n",
       "3             Data Science’s Most Misunderstood Hero   \n",
       "4  Amazon’s A.I. Emotion-Recognition Software Con...   \n",
       "5  3 Pandas Functions You Should Be Using More Often   \n",
       "6  IPython Notebook Support is Finally Here for V...   \n",
       "7                               5 Essential AI Books   \n",
       "8  Read Text from Image with One Line of Python Code   \n",
       "9       Did Google Duplex just pass the Turing Test?   \n",
       "\n",
       "                                                Body         Date  \\\n",
       "0                    Collections for data scientists       Oct 26   \n",
       "1  Critics of recent advancesartificial intellige...       Oct 27   \n",
       "2                   Notebooks are the data scientist       Oct 24   \n",
       "3  Why treating analytics like a second-class cit...       Oct 19   \n",
       "4                           People’s faces say a lot       Oct 27   \n",
       "5             Stop reinventing the wheel. Seriously.       Oct 23   \n",
       "6                                                          Oct 21   \n",
       "7  5 books on Artificial Intelligence that I’ve r...       Oct 27   \n",
       "8                                                          Oct 28   \n",
       "9          This conversational AI changes everything  May 8, 2018   \n",
       "\n",
       "           Comments  \n",
       "0   1.7K9 responses  \n",
       "1    6143 responses  \n",
       "2    1.1K1 response  \n",
       "3  2.5K12 responses  \n",
       "4    3453 responses  \n",
       "5    5154 responses  \n",
       "6   74710 responses  \n",
       "7      911 response  \n",
       "8      431 response  \n",
       "9   23K88 responses  "
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}