notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[H\u001b[2J"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import csv\n",
    "import datetime\n",
    "import pandas as pd\n",
    "\n",
    "# Extracting data from Amazon's Comixology page using Beautiful Soup\n",
    "URL = 'https://www.amazon.com/kindle-dbs/comics-store/home/?ref_=topnav_storetab_kstore_cmx&_encoding=UTF8&slot=desktop-atf-10'\n",
    "response = requests.get(URL)\n",
    "html = response.content\n",
    "soup = BeautifulSoup(html, 'html.parser')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\"\"\"Transforming: Cleaning the data\"\"\"\n",
    "# Function finds the data in the specified 'tag' and 'class'\n",
    "def find_data(tag, attr):\n",
    "    return soup.find_all(tag, class_=attr)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function that converts the data into strings\n",
    "def find_text(text):\n",
    "    return str(text.get_text()).strip()\n",
    "\n",
    "# Function is for removing the unnecessary spaces and newlines from 'div' tags for the author data\n",
    "def find_author(name):\n",
    "    return name.replace('by', '').replace(',', ' ,').split(',')[0].replace('\\n', '').lstrip().rstrip()\n",
    "\n",
    "# Holds all the 'div' tags of the comics that will be counted\n",
    "number_of_comics = [find_text(w) for w in find_data('div', 'csw-grid-list-item-wrapper csw-inline-buy-trigger')]\n",
    "num = len(number_of_comics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Holds the titles for each comic\n",
    "titles = [find_text(title) for title in find_data('span', 'csw-title-text')]\n",
    "\n",
    "# Holds the cover arts' image source for each comic\n",
    "cover_arts = [cover_art['src'] for cover_art in find_data('img', 'csw-item-art')]\n",
    "# Holds the author for each comic\n",
    "authors = [find_author(find_text(author)) for author in find_data('div', 'a-row csw-sub-title-row')]\n",
    "\n",
    "# Holds the required data (currency, dollar, and cent) to develop a price for each comic\n",
    "prices = []\n",
    "currency = [find_text(cu) for cu in find_data('span', 'csw-item-price-currency')]\n",
    "dollars = [find_text(d) for d in find_data('span', 'csw-item-price-dollars')]\n",
    "cents = [find_text(ct) for ct in find_data('span', 'csw-item-price-cents')]\n",
    "\n",
    "# Function is for formating the price of a comic correctly with right the currency, dollar, and cent amount\n",
    "def get_prices(cur, dol, cnt):\n",
    "    for i in range(num):\n",
    "        prices.append(f'{cur[i]}{dol[i]}.{cnt[i]}')\n",
    "\n",
    "get_prices(currency, dollars, cents)\n",
    "\n",
    "# Holds all the data (Title, Author, Cover Art, and Price) for each comic\n",
    "new_releases = []\n",
    "\n",
    "# Function is used to add the data into the 'new_releases' list with the comic's title, author, cover art, and its price\n",
    "def get_new_releases(t, a, ca, p):\n",
    "    for i in range(num):\n",
    "        new_releases.append([t[i], a[i], ca[i], p[i]])\n",
    "\n",
    "get_new_releases(titles, authors, cover_arts, prices)\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"Loading the data into a predefined bucket called 'comixology-new-releases' in AWS S3\"\"\"\n",
    "header = ['Title', 'Author', 'Cover Art', 'Price']\n",
    "today = datetime.date.today()\n",
    "filename = f'{today}.csv' # \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Author</th>\n",
       "      <th>Cover Art</th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Youth Season Three (Comixology Originals) #1 (...</td>\n",
       "      <td>Curt Pires</td>\n",
       "      <td>https://m.media-amazon.com/images/I/319SE-RyNj...</td>\n",
       "      <td>$0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Teenage Mutant Ninja Turtles: The Last Ronin—T...</td>\n",
       "      <td>Kevin B. Eastman</td>\n",
       "      <td>https://m.media-amazon.com/images/I/51shHSNWPf...</td>\n",
       "      <td>$4.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Saga #61</td>\n",
       "      <td>Brian K. Vaughan</td>\n",
       "      <td>https://m.media-amazon.com/images/I/41Q1oZKJ-H...</td>\n",
       "      <td>$3.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Dudley Datson and the Forever Machine (Comixol...</td>\n",
       "      <td>Scott Snyder</td>\n",
       "      <td>https://m.media-amazon.com/images/I/51cmt669Ns...</td>\n",
       "      <td>$0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Ask For Mercy Season Four (Comixology Original...</td>\n",
       "      <td>Richard Starkings</td>\n",
       "      <td>https://m.media-amazon.com/images/I/51Ves9RzA8...</td>\n",
       "      <td>$0.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Title             Author  \\\n",
       "0  Youth Season Three (Comixology Originals) #1 (...         Curt Pires   \n",
       "1  Teenage Mutant Ninja Turtles: The Last Ronin—T...   Kevin B. Eastman   \n",
       "2                                           Saga #61   Brian K. Vaughan   \n",
       "3  Dudley Datson and the Forever Machine (Comixol...       Scott Snyder   \n",
       "4  Ask For Mercy Season Four (Comixology Original...  Richard Starkings   \n",
       "\n",
       "                                           Cover Art  Price  \n",
       "0  https://m.media-amazon.com/images/I/319SE-RyNj...  $0.00  \n",
       "1  https://m.media-amazon.com/images/I/51shHSNWPf...  $4.99  \n",
       "2  https://m.media-amazon.com/images/I/41Q1oZKJ-H...  $3.19  \n",
       "3  https://m.media-amazon.com/images/I/51cmt669Ns...  $0.00  \n",
       "4  https://m.media-amazon.com/images/I/51Ves9RzA8...  $0.00  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Creating a CSV file\n",
    "with open(f'{filename}', 'w') as file:\n",
    "    writer = csv.writer(file)\n",
    "    writer.writerow(header)\n",
    "    # Loading the data into the CSV file that was created\n",
    "    for i in new_releases:\n",
    "        writer.writerow(i)\n",
    "\n",
    "# Closing the CSV file\n",
    "file.close()\n",
    "\n",
    "dataset = pd.read_csv(filename)\n",
    "\n",
    "dataset.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "f8c028df71ed0adf6d9a4c371418625b1ea766338bd172396b5cdc1904829e39"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}