Pandas Web Scraping Humanist.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import csv\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n",
      "{'Server': 'ATS', 'Date': 'Wed, 23 Jun 2021 16:00:40 GMT', 'Content-Type': 'text/html; charset=utf-8', 'Content-Length': '5814', 'X-Frame-Options': 'SAMEORIGIN', 'Age': '0', 'Connection': 'keep-alive'}\n"
     ]
    }
   ],
   "source": [
    "url = \"https://humanist.kdl.kcl.ac.uk/\"\n",
    "result = requests.get(url)\n",
    "\n",
    "print(result.status_code)\n",
    "print(result.headers)\n",
    "page = result.content\n",
    "\n",
    "# How would we get all the links to volumes from the Humanist Page?\n",
    "\n",
    "soup = BeautifulSoup(page)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://humanist.kdl.kcl.ac.uk/Archives/Current/ https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://humanist.kdl.kcl.ac.uk/Archives/Current/Humanist.21.542-21.680.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Current/Humanist.vol22.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://humanist.kdl.kcl.ac.uk/Archives/Current/Humanist.vol23.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Current/Humanist.vol24.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Current/Humanist.vol25.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Current/Humanist.vol26.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://humanist.kdl.kcl.ac.uk/Archives/Current/Humanist.vol27.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://humanist.kdl.kcl.ac.uk/Archives/Current/Humanist.vol28.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://humanist.kdl.kcl.ac.uk/Archives/Current/Humanist.vol29.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Current/Humanist.vol30.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Current/Humanist.vol31.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Current/Humanist.vol32.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.1987-1988.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.1988-1989.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.1989-1990.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.1990-1991.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.1991-1992.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.1992-1993.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.1993-1994.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.1994-1995.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.1995-1996.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.1996-1997.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.1997-1998.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.1998-1999.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.1999-2000.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.2000-2001.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.2001-2002.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.2002-2003.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.2003-2004.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.2004-2005.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.2005-2006.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.2006-2007.txt\n",
      "https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/humanist.2007-2008.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.\n"
     ]
    }
   ],
   "source": [
    "links = ['https://humanist.kdl.kcl.ac.uk/Archives/Current/','https://humanist.kdl.kcl.ac.uk/Archives/Converted_Text/']\n",
    "texts = []\n",
    "for link in links:\n",
    "    result = requests.get(link)\n",
    "    page = result.content\n",
    "\n",
    "    soup = BeautifulSoup(page)\n",
    "    text_links = soup.find_all('a')\n",
    "    for txt_link in text_links:\n",
    "        url = txt_link.get('href')\n",
    "        if 'txt' in url:\n",
    "            print(link+url)\n",
    "            text_result = requests.get(link+url)\n",
    "            text_page = text_result.content\n",
    "            text_soup = BeautifulSoup(text_page)\n",
    "            dates = url.split('.')[1]\n",
    "            text = text_soup.get_text()\n",
    "            texts.append({'dates': dates, 'text': text})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Turn our web scraped data in a dataframe\n",
    "humanist_vols = pd.DataFrame.from_dict(texts, orient='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write to csv\n",
    "humanist_vols.to_csv('web_scraped_humanist_listserv.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}