Web爬虫第一章(2).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hasattr(object, name)   \n",
    "#判断object中是否有name属性，返回BOOL值，有name特性返回True， 否则返回False,name要用括号括起来"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 设置用户代理\n",
    "from urllib.request import Request,urlopen\n",
    "from urllib import error\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://httpstat.us/500\n",
      "Download error: [Errno 11004] getaddrinfo failed\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "def download(url, user_agent = 'wswp', num_retries=2):\n",
    "    print('Downloading:', url)\n",
    "    headers = {'User-agent': user_agent}\n",
    "    request = Request(url, headers = headers)\n",
    "    try:\n",
    "        html = urlopen(request).read()\n",
    "    except error.URLError as e:\n",
    "        print('Download error:',e.reason)\n",
    "        html = None\n",
    "        if num_retries > 0:\n",
    "            if hasattr(e, 'code') and 500 <= e.code < 600:\n",
    "                # recursively retry 5xx HTTP errors\n",
    "                return download(url, user_agent,num_retries-1)\n",
    "    return html\n",
    "#print(download('http://example.webscraping.com/'))    \n",
    "print(download('http://httpstat.us/500'))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 网站地图爬虫"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 中止爬虫运行用 Ctrl + C , or CMD + C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "import urllib.request\n",
    "from urllib.error import URLError, HTTPError, ContentTooShortError\n",
    "\n",
    "\n",
    "def download(url, user_agent='wswp', num_retries=2, charset='utf-8'):\n",
    "    print('Downloading:', url)\n",
    "    request = urllib.request.Request(url)\n",
    "    request.add_header('User-agent', user_agent)\n",
    "    try:\n",
    "        resp = urllib.request.urlopen(request)\n",
    "        cs = resp.headers.get_content_charset()\n",
    "        if not cs:\n",
    "            cs = charset\n",
    "        html = resp.read().decode(cs)\n",
    "    except (URLError, HTTPError, ContentTooShortError) as e:\n",
    "        print('Download error:', e.reason)\n",
    "        html = None\n",
    "        if num_retries > 0:\n",
    "            if hasattr(e, 'code') and 500 <= e.code < 600:\n",
    "                # recursively retry 5xx HTTP errors\n",
    "                return download(url, num_retries - 1)\n",
    "    return html\n",
    "\n",
    "'''\n",
    "def crawl_site(url, max_errors=5):\n",
    "    num_errors = 0\n",
    "    for page in itertools.count(1):\n",
    "        pg_url = '{}{}'.format(url, page)\n",
    "        html = download(pg_url)\n",
    "        if html is None:\n",
    "            num_errors += 1\n",
    "            if num_errors == max_errors:\n",
    "                # reached max number of errors, so exit\n",
    "                break\n",
    "        else:\n",
    "            num_errors = 0\n",
    "            # success - can scrape the result\n",
    "           \n",
    "   \n",
    "'''\n",
    "def crawl_sitemap(url):\n",
    "    #sitemap = download(url).decode('utf-8') #download the sitemap file\n",
    "    sitemap = download(url)\n",
    "    #sitemap = sitemap.decode('utf-8')\n",
    "    links = re.findall('<loc>(.*?)</loc>', sitemap) # extract the sitemap links\n",
    "   \n",
    "    \n",
    "    for link in links: # download each link\n",
    "        \n",
    "        html = download(link) # scarpe html here'''\n",
    "        \n",
    "#crawl_sitemap('http://example.webscraping.com/sitemap.xml')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ID iteration crawler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def crawl_site(url):\n",
    "    for page in itertools.count(1):  #count() 接收两个参数，第一个参数指定开始值，默认为 0，第二个参数指定步长，默认为 1：\n",
    "        pg_url = '{}{}'.format(url, page)\n",
    "        html = download(pg_url)\n",
    "        if html is None:\n",
    "            break\n",
    "        # success - can scrape the result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://example.webscraping.com/-1\n",
      "Downloading: http://example.webscraping.com/-2\n",
      "Downloading: http://example.webscraping.com/-3\n",
      "Downloading: http://example.webscraping.com/-4\n",
      "Downloading: http://example.webscraping.com/-5\n",
      "Downloading: http://example.webscraping.com/-6\n",
      "Downloading: http://example.webscraping.com/-7\n",
      "Downloading: http://example.webscraping.com/-8\n",
      "Downloading: http://example.webscraping.com/-9\n",
      "Downloading: http://example.webscraping.com/-10\n",
      "Downloading: http://example.webscraping.com/-11\n",
      "Downloading: http://example.webscraping.com/-12\n",
      "Download error: TOO MANY REQUESTS\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(crawl_site('http://example.webscraping.com/-'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 设置发生错误下载次数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def crawl_site(url, max_errors=5):\n",
    "    num_errors = 0\n",
    "    for page in itertools.count(1):\n",
    "        pg_url = '{}{}'.format(url, page)\n",
    "        html = download(pg_url)\n",
    "        if html is None:\n",
    "            num_errors += 1\n",
    "            if num_errors == max_errors:\n",
    "                # reached max number of errors, so exit\n",
    "                break\n",
    "        else:\n",
    "            num_errors = 0\n",
    "            # success - can scrape the result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://example.webscraping.com/-1\n",
      "Downloading: http://example.webscraping.com/-2\n",
      "Downloading: http://example.webscraping.com/-3\n",
      "Downloading: http://example.webscraping.com/-4\n",
      "Downloading: http://example.webscraping.com/-5\n",
      "Downloading: http://example.webscraping.com/-6\n",
      "Downloading: http://example.webscraping.com/-7\n",
      "Downloading: http://example.webscraping.com/-8\n",
      "Downloading: http://example.webscraping.com/-9\n",
      "Downloading: http://example.webscraping.com/-10\n",
      "Downloading: http://example.webscraping.com/-11\n",
      "Downloading: http://example.webscraping.com/-12\n",
      "Download error: TOO MANY REQUESTS\n",
      "Downloading: http://example.webscraping.com/-13\n",
      "Download error: TOO MANY REQUESTS\n",
      "Downloading: http://example.webscraping.com/-14\n",
      "Download error: TOO MANY REQUESTS\n",
      "Downloading: http://example.webscraping.com/-15\n",
      "Download error: TOO MANY REQUESTS\n",
      "Downloading: http://example.webscraping.com/-16\n",
      "Download error: TOO MANY REQUESTS\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(crawl_site('http://example.webscraping.com/-'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 链接爬虫"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "'''\n",
    "def download(url, user_agent='wswp', num_retries=2, charset='utf-8'):\n",
    "    print('Downloading:', url)\n",
    "    request = urllib.request.Request(url)\n",
    "    request.add_header('User-agent', user_agent)\n",
    "    try:\n",
    "        resp = urllib.request.urlopen(request)\n",
    "        cs = resp.headers.get_content_charset()\n",
    "        if not cs:\n",
    "            cs = charset\n",
    "        html = resp.read().decode(cs)\n",
    "    except (URLError, HTTPError, ContentTooShortError) as e:\n",
    "        print('Download error:', e.reason)\n",
    "        html = None\n",
    "        if num_retries > 0:\n",
    "            if hasattr(e, 'code') and 500 <= e.code < 600:\n",
    "                # recursively retry 5xx HTTP errors\n",
    "                return download(url, num_retries - 1)\n",
    "    return html\n",
    "'''\n",
    "\n",
    "def link_crawler(start_url, link_regex):\n",
    "    \" Crawl from the given start URL following links matched by link_regex \"\n",
    "    crawl_queue = [start_url]\n",
    "    # keep track which URL's have seen before\n",
    "    seen = set(crawl_queue)\n",
    "    while crawl_queue:\n",
    "        url = crawl_queue.pop()\n",
    "        html = download(url)\n",
    "        if not html:\n",
    "            continue\n",
    "        # filter for links matching our regular expression\n",
    "        for link in get_links(html):\n",
    "            if re.match(link_regex, link):\n",
    "                abs_link = urljoin(start_url, link)\n",
    "                if abs_link not in seen:\n",
    "                    seen.add(abs_link)\n",
    "                    crawl_queue.append(abs_link)\n",
    "\n",
    "\n",
    "def get_links(html):\n",
    "    # Return a list of links from html \"\n",
    "    # a regular expression to extract all links from the webpage\n",
    "    webpage_regex = re.compile(\"\"\"<a[^>]+href=[\"'](.*?)[\"']\"\"\", re.IGNORECASE)\n",
    "    # list of all links from the webpage\n",
    "    return webpage_regex.findall(html)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: http://example.webscraping.com\n"
     ]
    }
   ],
   "source": [
    "link_crawler('http://example.webscraping.com','/(index|view)/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<generator object <genexpr> at 0x000000000524C308>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g= (x*x for  x in range(10))\n",
    "g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "4\n",
      "9\n",
      "16\n",
      "25\n",
      "36\n",
      "49\n",
      "64\n",
      "81\n"
     ]
    }
   ],
   "source": [
    "for n in g:\n",
    "    print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}