Week3-Part1.ipynb

{
    "cells": [
        {
            "cell_type": "markdown",
            "metadata": {
                "collapsed": true
            },
            "source": "# Segmenting and Clustering Neighborhoods in Toronto - Part 1"
        },
        {
            "cell_type": "code",
            "execution_count": 11,
            "metadata": {
                "scrolled": true
            },
            "outputs": [],
            "source": "from bs4 import BeautifulSoup\nimport requests\nimport numpy as np\nfrom geopy.geocoders import Nominatim # convert an address into latitude and longitude values\nfrom pandas.io.json import json_normalize  # tranform JSON file into a pandas dataframe\n\nimport pandas as pd"
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": "## Extract the list of neighborhoods from Wikipedia"
        },
        {
            "cell_type": "code",
            "execution_count": 12,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>PostalCode</th>\n      <th>Borough</th>\n      <th>Neighborhood</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>M1A\\n</td>\n      <td>Not assigned\\n</td>\n      <td>\\n</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>M2A\\n</td>\n      <td>Not assigned\\n</td>\n      <td>\\n</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>M3A\\n</td>\n      <td>North York\\n</td>\n      <td>Parkwoods\\n</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>M4A\\n</td>\n      <td>North York\\n</td>\n      <td>Victoria Village\\n</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>M5A\\n</td>\n      <td>Downtown Toronto\\n</td>\n      <td>Regent Park, Harbourfront\\n</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
                        "text/plain": "  PostalCode             Borough                 Neighborhood\n0      M1A\\n      Not assigned\\n                           \\n\n1      M2A\\n      Not assigned\\n                           \\n\n2      M3A\\n        North York\\n                  Parkwoods\\n\n3      M4A\\n        North York\\n           Victoria Village\\n\n4      M5A\\n  Downtown Toronto\\n  Regent Park, Harbourfront\\n"
                    },
                    "execution_count": 12,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": "source = requests.get(\"https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M\").text\nsoup = BeautifulSoup(source, 'lxml')\n\ntable = soup.find(\"table\")\ntable_rows = table.tbody.find_all(\"tr\")\nres = []\nfor tr in table_rows:\n    td = tr.find_all(\"td\")\n    row = [tr.text for tr in td]\n    \n    # Only process the cells that have an assigned borough. Ignore cells with a borough that is Not assigned.\n    if row != [] and row[1] != \"Not assigned\":\n        # If a cell has a borough but a \"Not assigned\" neighborhood, then the neighborhood will be the same as the borough.\n        if \"Not assigned\" in row[2]: \n            row[2] = row[1]\n        res.append(row)\n\n# Dataframe with 3 columns\ndf_wiki = pd.DataFrame(res, columns = [\"PostalCode\", \"Borough\", \"Neighborhood\"])\ndf_wiki.head()"
        },
        {
            "cell_type": "code",
            "execution_count": 13,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>PostalCode</th>\n      <th>Borough</th>\n      <th>Neighborhood</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>M1A</td>\n      <td>Not assigned</td>\n      <td></td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>M2A</td>\n      <td>Not assigned</td>\n      <td></td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>M3A</td>\n      <td>North York</td>\n      <td>Parkwoods</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>M4A</td>\n      <td>North York</td>\n      <td>Victoria Village</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>M5A</td>\n      <td>Downtown Toronto</td>\n      <td>Regent Park, Harbourfront</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
                        "text/plain": "  PostalCode           Borough               Neighborhood\n0        M1A      Not assigned                           \n1        M2A      Not assigned                           \n2        M3A        North York                  Parkwoods\n3        M4A        North York           Victoria Village\n4        M5A  Downtown Toronto  Regent Park, Harbourfront"
                    },
                    "execution_count": 13,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": "df_wiki[\"PostalCode\"]=df_wiki[\"PostalCode\"].str.replace(\"\\n\",\"\")\ndf_wiki[\"Borough\"]=df_wiki[\"Borough\"].str.replace(\"\\n\",\"\")\ndf_wiki[\"Neighborhood\"]=df_wiki[\"Neighborhood\"].str.replace(\"\\n\",\"\")\ndf_wiki.head()"
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": "## Filtered data with Borough \"Not Assigned\""
        },
        {
            "cell_type": "code",
            "execution_count": 16,
            "metadata": {},
            "outputs": [],
            "source": "df_wiki_filtered = df_wiki[df_wiki[\"Borough\"]!=\"Not assigned\"]"
        },
        {
            "cell_type": "code",
            "execution_count": 17,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/plain": "(103, 3)"
                    },
                    "execution_count": 17,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": "df_wiki_filtered.to_csv(\"./CanadaNeighborhood_List.csv\",index=False)\ndf_wiki_filtered.shape"
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": ""
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "Python 3.6",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.6.9"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 1
}