Chapter06_Web-Crawlers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#bodyContent\n",
      "/wiki/Main_Page\n",
      "/wiki/Wikipedia:Contents\n",
      "/wiki/Portal:Current_events\n",
      "/wiki/Special:Random\n",
      "/wiki/Wikipedia:About\n",
      "//en.wikipedia.org/wiki/Wikipedia:Contact_us\n",
      "https://donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en\n",
      "/wiki/Help:Contents\n",
      "/wiki/Help:Introduction\n",
      "/wiki/Wikipedia:Community_portal\n",
      "/wiki/Special:RecentChanges\n",
      "/wiki/Wikipedia:File_upload_wizard\n",
      "/wiki/Main_Page\n",
      "/wiki/Special:Search\n",
      "/w/index.php?title=Special:CreateAccount&returnto=Kevin+Bacon\n",
      "/w/index.php?title=Special:UserLogin&returnto=Kevin+Bacon\n",
      "/w/index.php?title=Special:CreateAccount&returnto=Kevin+Bacon\n",
      "/w/index.php?title=Special:UserLogin&returnto=Kevin+Bacon\n",
      "/wiki/Help:Introduction\n",
      "/wiki/Special:MyContributions\n",
      "/wiki/Special:MyTalk\n",
      "#\n",
      "#Early_life_and_education\n",
      "#Acting_career\n",
      "#Early_work\n",
      "#1980s\n",
      "#1990s\n",
      "#2000s\n",
      "#2010s\n",
      "#Advertising_work\n",
      "#Six_Degrees_of_Kevin_Bacon\n",
      "#Music\n",
      "#Personal_life\n",
      "#Accolades\n",
      "#Awards_and_nominations\n",
      "#Other_honors\n",
      "#See_also\n",
      "#References\n",
      "#External_links\n",
      "https://af.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://ar.wikipedia.org/wiki/%D9%83%D9%8A%D9%81%D9%8A%D9%86_%D8%A8%D9%8A%D9%83%D9%86\n",
      "https://an.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://ast.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://azb.wikipedia.org/wiki/%DA%A9%D9%88%DB%8C%D9%86_%D8%A8%DB%8C%DA%A9%D9%86\n",
      "https://zh-min-nan.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://bi.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://bg.wikipedia.org/wiki/%D0%9A%D0%B5%D0%B2%D0%B8%D0%BD_%D0%91%D0%B5%D0%B9%D0%BA%D1%8A%D0%BD\n",
      "https://bs.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://ca.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://cs.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://da.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://de.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://el.wikipedia.org/wiki/%CE%9A%CE%AD%CE%B2%CE%B9%CE%BD_%CE%9C%CF%80%CE%AD%CE%B9%CE%BA%CE%BF%CE%BD\n",
      "https://eml.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://es.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://eo.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://eu.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://fa.wikipedia.org/wiki/%DA%A9%D9%88%DB%8C%D9%86_%D8%A8%DB%8C%DA%A9%D9%86\n",
      "https://fr.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://ga.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://gl.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://ko.wikipedia.org/wiki/%EC%BC%80%EB%B9%88_%EB%B2%A0%EC%9D%B4%EC%BB%A8\n",
      "https://hy.wikipedia.org/wiki/%D5%94%D6%87%D5%AB%D5%B6_%D4%B2%D5%A5%D5%B5%D6%84%D5%B8%D5%B6\n",
      "https://hr.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://io.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://id.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://it.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://he.wikipedia.org/wiki/%D7%A7%D7%95%D7%95%D7%99%D7%9F_%D7%91%D7%99%D7%99%D7%A7%D7%95%D7%9F\n",
      "https://ka.wikipedia.org/wiki/%E1%83%99%E1%83%94%E1%83%95%E1%83%98%E1%83%9C_%E1%83%91%E1%83%94%E1%83%98%E1%83%99%E1%83%9D%E1%83%9C%E1%83%98\n",
      "https://kk.wikipedia.org/wiki/%D0%9A%D0%B5%D0%B2%D0%B8%D0%BD_%D0%91%D1%8D%D0%B9%D0%BA%D0%BE%D0%BD\n",
      "https://lv.wikipedia.org/wiki/Kevins_Beikons\n",
      "https://hu.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://xmf.wikipedia.org/wiki/%E1%83%99%E1%83%94%E1%83%95%E1%83%98%E1%83%9C_%E1%83%91%E1%83%94%E1%83%98%E1%83%99%E1%83%9D%E1%83%9C%E1%83%98\n",
      "https://arz.wikipedia.org/wiki/%D9%83%D9%8A%D9%81%D9%8A%D9%86_%D8%A8%D9%8A%D9%83%D9%86\n",
      "https://ms.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://mn.wikipedia.org/wiki/%D0%9A%D0%B5%D0%B2%D0%B8%D0%BD_%D0%91%D1%8D%D0%B9%D0%BA%D0%BE%D0%BD\n",
      "https://nl.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://ja.wikipedia.org/wiki/%E3%82%B1%E3%83%B4%E3%82%A3%E3%83%B3%E3%83%BB%E3%83%99%E3%83%BC%E3%82%B3%E3%83%B3\n",
      "https://no.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://oc.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://uz.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://pl.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://pt.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://ro.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://ru.wikipedia.org/wiki/%D0%91%D0%B5%D0%B9%D0%BA%D0%BE%D0%BD,_%D0%9A%D0%B5%D0%B2%D0%B8%D0%BD\n",
      "https://sco.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://simple.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://sk.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://ckb.wikipedia.org/wiki/%DA%A9%DB%8E%DA%A4%D9%86_%D8%A8%DB%95%DB%8C%DA%A9%D9%86\n",
      "https://sr.wikipedia.org/wiki/%D0%9A%D0%B5%D0%B2%D0%B8%D0%BD_%D0%91%D0%B5%D1%98%D0%BA%D0%BE%D0%BD\n",
      "https://sh.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://fi.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://sv.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://th.wikipedia.org/wiki/%E0%B9%80%E0%B8%84%E0%B8%A7%E0%B8%B4%E0%B8%99_%E0%B9%80%E0%B8%9A%E0%B8%84%E0%B8%AD%E0%B8%99\n",
      "https://tr.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://uk.wikipedia.org/wiki/%D0%9A%D0%B5%D0%B2%D1%96%D0%BD_%D0%91%D0%B5%D0%B9%D0%BA%D0%BE%D0%BD\n",
      "https://vi.wikipedia.org/wiki/Kevin_Bacon\n",
      "https://wuu.wikipedia.org/wiki/%E5%87%AF%E6%96%87%C2%B7%E8%B4%9D%E8%82%AF\n",
      "https://zh-yue.wikipedia.org/wiki/%E5%A5%87%E9%9B%B2%E8%B2%9D%E6%A0%B9\n",
      "https://zh.wikipedia.org/wiki/%E5%87%AF%E6%96%87%C2%B7%E8%B4%9D%E8%82%AF\n",
      "https://www.wikidata.org/wiki/Special:EntityPage/Q3454165#sitelinks-wikipedia\n",
      "/wiki/Kevin_Bacon\n",
      "/wiki/Talk:Kevin_Bacon\n",
      "/wiki/Kevin_Bacon\n",
      "/w/index.php?title=Kevin_Bacon&action=edit\n",
      "/w/index.php?title=Kevin_Bacon&action=history\n",
      "/wiki/Kevin_Bacon\n",
      "/w/index.php?title=Kevin_Bacon&action=edit\n",
      "/w/index.php?title=Kevin_Bacon&action=history\n",
      "/wiki/Special:WhatLinksHere/Kevin_Bacon\n",
      "/wiki/Special:RecentChangesLinked/Kevin_Bacon\n",
      "/wiki/Wikipedia:File_Upload_Wizard\n",
      "/wiki/Special:SpecialPages\n",
      "/w/index.php?title=Kevin_Bacon&oldid=1146341850\n",
      "/w/index.php?title=Kevin_Bacon&action=info\n",
      "/w/index.php?title=Special:CiteThisPage&page=Kevin_Bacon&id=1146341850&wpFormIdentifier=titleform\n",
      "https://www.wikidata.org/wiki/Special:EntityPage/Q3454165\n",
      "/w/index.php?title=Special:DownloadAsPdf&page=Kevin_Bacon&action=show-download-screen\n",
      "/w/index.php?title=Kevin_Bacon&printable=yes\n",
      "https://commons.wikimedia.org/wiki/Kevin_Bacon\n",
      "/wiki/Wikipedia:Protection_policy#semi\n",
      "/wiki/Kevin_Bacon_(disambiguation)\n",
      "/wiki/File:Kevin_Bacon_SDCC_2014.jpg\n",
      "/wiki/Philadelphia\n",
      "/wiki/Kevin_Bacon_filmography\n",
      "/wiki/Kyra_Sedgwick\n",
      "/wiki/Sosie_Bacon\n",
      "#cite_note-1\n",
      "/wiki/Edmund_Bacon_(architect)\n",
      "/wiki/Michael_Bacon_(musician)\n",
      "/wiki/Holly_Near\n",
      "/wiki/Wikipedia:Citation_needed\n",
      "http://baconbros.com/\n",
      "#cite_note-2\n",
      "#cite_note-actor-3\n",
      "/wiki/Leading_man\n",
      "/wiki/Character_actor\n",
      "/wiki/Golden_Globe_Award\n",
      "/wiki/Screen_Actors_Guild_Award\n",
      "/wiki/Primetime_Emmy_Award\n",
      "/wiki/The_Guardian\n",
      "/wiki/Academy_Award\n",
      "#cite_note-4\n",
      "/wiki/Hollywood_Walk_of_Fame\n",
      "#cite_note-5\n",
      "/wiki/National_Lampoon%27s_Animal_House\n",
      "/wiki/Footloose_(1984_film)\n",
      "/wiki/Diner_(1982_film)\n",
      "/wiki/JFK_(film)\n",
      "/wiki/A_Few_Good_Men\n",
      "/wiki/Apollo_13_(film)\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/Frost/Nixon_(film)\n",
      "/wiki/Friday_the_13th_(1980_film)\n",
      "/wiki/The_River_Wild\n",
      "/wiki/Sleepers_(film)\n",
      "/wiki/Wild_Things_(film)\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/Flatliners\n",
      "/wiki/Crazy,_Stupid,_Love\n",
      "/wiki/X-Men:_First_Class\n",
      "/wiki/Black_Mass_(film)\n",
      "/wiki/Patriots_Day_(film)\n",
      "/wiki/Losing_Chase\n",
      "/wiki/Loverboy_(2005_film)\n",
      "/wiki/Fox_Broadcasting_Company\n",
      "/wiki/The_Following\n",
      "/wiki/Michael_Strobl\n",
      "/wiki/HBO\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Miniseries_or_Television_Film\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie\n",
      "/wiki/Amazon_Prime_Video\n",
      "/wiki/I_Love_Dick_(TV_series)\n",
      "/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\n",
      "/wiki/Showtime_(TV_network)\n",
      "/wiki/City_on_a_Hill_(TV_series)\n",
      "#cite_note-6\n",
      "/wiki/Six_Degrees_of_Kevin_Bacon\n",
      "/wiki/SixDegrees.org\n",
      "#cite_note-walk-7\n",
      "/wiki/EE_Limited\n",
      "#cite_note-8\n",
      "/wiki/Philadelphia\n",
      "#cite_note-actor-3\n",
      "#cite_note-actor-3\n",
      "/wiki/Edmund_Bacon_(architect)\n",
      "/wiki/Design_of_Cities\n",
      "#cite_note-bacon-9\n",
      "/wiki/Julia_R._Masterman_High_School\n",
      "#cite_note-10\n",
      "/wiki/Pennsylvania_Governor%27s_School_for_the_Arts\n",
      "/wiki/Bucknell_University\n",
      "#cite_note-11\n",
      "/wiki/Glory_Van_Scott\n",
      "#cite_note-walk-7\n",
      "#cite_note-bacon-9\n",
      "/wiki/Kevin_Bacon_filmography\n",
      "/wiki/Circle_in_the_Square\n",
      "/wiki/Nancy_Mills\n",
      "/wiki/Cosmopolitan_(magazine)\n",
      "#cite_note-cosmo91-12\n",
      "/wiki/Fraternities_and_sororities\n",
      "/wiki/Animal_House\n",
      "#cite_note-bacon-9\n",
      "/wiki/Search_for_Tomorrow\n",
      "/wiki/Guiding_Light\n",
      "/wiki/Friday_the_13th_(1980_film)\n",
      "#cite_note-13\n",
      "/wiki/Getting_Out\n",
      "/wiki/Phoenix_Theater\n",
      "/wiki/Flux\n",
      "/wiki/Second_Stage_Theatre\n",
      "#cite_note-bio-14\n",
      "/wiki/Obie_Award\n",
      "/wiki/Forty_Deuce\n",
      "#cite_note-kevin-15\n",
      "/wiki/Slab_Boys\n",
      "/wiki/Sean_Penn\n",
      "/wiki/Val_Kilmer\n",
      "/wiki/Barry_Levinson\n",
      "/wiki/Diner_(1982_film)\n",
      "/wiki/Steve_Guttenberg\n",
      "/wiki/Daniel_Stern_(actor)\n",
      "/wiki/Mickey_Rourke\n",
      "/wiki/Tim_Daly\n",
      "/wiki/Ellen_Barkin\n",
      "#cite_note-16\n",
      "/wiki/Footloose_(1984_film)\n",
      "#cite_note-bio-14\n",
      "/wiki/James_Dean\n",
      "/wiki/Rebel_Without_a_Cause\n",
      "/wiki/Mickey_Rooney\n",
      "/wiki/Judy_Garland\n",
      "#cite_note-time84-17\n",
      "#cite_note-bacon-9\n",
      "#cite_note-18\n",
      "#cite_note-19\n",
      "/wiki/Typecasting_(acting)\n",
      "/wiki/John_Hughes_(filmmaker)\n",
      "/wiki/Planes,_Trains_and_Automobiles\n",
      "#cite_note-20\n",
      "/wiki/She%27s_Having_a_Baby\n",
      "#cite_note-bio-14\n",
      "/wiki/The_Big_Picture_(1989_film)\n",
      "#cite_note-21\n",
      "/wiki/Tremors_(1990_film)\n",
      "#cite_note-22\n",
      "/wiki/Joel_Schumacher\n",
      "/wiki/Flatliners\n",
      "#cite_note-bio-14\n",
      "/wiki/Elizabeth_Perkins\n",
      "/wiki/He_Said,_She_Said_(film)\n",
      "#cite_note-bio-14\n",
      "/wiki/The_New_York_Times\n",
      "#cite_note-nyt94-23\n",
      "/wiki/Oliver_Stone\n",
      "/wiki/JFK_(film)\n",
      "#cite_note-24\n",
      "/wiki/A_Few_Good_Men_(film)\n",
      "#cite_note-25\n",
      "/wiki/Michael_Greif\n",
      "#cite_note-bio-14\n",
      "/wiki/Golden_Globe_Award\n",
      "/wiki/The_River_Wild\n",
      "#cite_note-bio-14\n",
      "/wiki/Meryl_Streep\n",
      "/wiki/Murder_in_the_First_(film)\n",
      "#cite_note-bio-14\n",
      "/wiki/Blockbuster_(entertainment)\n",
      "/wiki/Apollo_13_(film)\n",
      "#cite_note-26\n",
      "/wiki/Sleepers_(film)\n",
      "#cite_note-27\n",
      "/wiki/Picture_Perfect_(1997_film)\n",
      "#cite_note-bio-14\n",
      "/wiki/Losing_Chase\n",
      "#cite_note-austin-28\n",
      "/wiki/Digging_to_China\n",
      "#cite_note-bio-14\n",
      "/wiki/Payola\n",
      "/wiki/Telling_Lies_in_America_(film)\n",
      "#cite_note-bio-14\n",
      "/wiki/Wild_Things_(film)\n",
      "/wiki/Stir_of_Echoes\n",
      "/wiki/David_Koepp\n",
      "#cite_note-29\n",
      "/wiki/File:Kevin_Bacon_Cannes_2004.jpg\n",
      "/wiki/File:Kevin_Bacon_Cannes_2004.jpg\n",
      "/wiki/Cannes_Film_Festival\n",
      "/wiki/Paul_Verhoeven\n",
      "/wiki/Hollow_Man\n",
      "#cite_note-30\n",
      "/wiki/Colin_Firth\n",
      "/wiki/Rachel_Blanchard\n",
      "/wiki/M%C3%A9nage_%C3%A0_trois\n",
      "/wiki/Where_the_Truth_Lies\n",
      "#cite_note-31\n",
      "/wiki/Atom_Egoyan\n",
      "/wiki/MPAA\n",
      "/wiki/MPAA_film_rating_system\n",
      "#cite_note-32\n",
      "/wiki/Sean_Penn\n",
      "/wiki/Tim_Robbins\n",
      "/wiki/Clint_Eastwood\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "#cite_note-33\n",
      "/wiki/HBO_Films\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Michael_Strobl\n",
      "/wiki/Desert_Storm\n",
      "#cite_note-34\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie\n",
      "/wiki/File:Black_Mass_PC_04_(21429102646).jpg\n",
      "/wiki/File:Black_Mass_PC_04_(21429102646).jpg\n",
      "/wiki/Toronto_Film_Festival\n",
      "/wiki/Matthew_Vaughn\n",
      "/wiki/X-Men:_First_Class\n",
      "/wiki/Sebastian_Shaw_(comics)\n",
      "#cite_note-35\n",
      "#cite_note-36\n",
      "/wiki/Dustin_Lance_Black\n",
      "/wiki/8_(play)\n",
      "/wiki/Perry_v._Brown\n",
      "/wiki/Proposition_8\n",
      "/wiki/Charles_J._Cooper\n",
      "#cite_note-8_the_play-37\n",
      "/wiki/Wilshire_Ebell_Theatre\n",
      "/wiki/American_Foundation_for_Equal_Rights\n",
      "#cite_note-8_play_video-38\n",
      "#cite_note-39\n",
      "/wiki/The_Following\n",
      "#cite_note-40\n",
      "/wiki/Saturn_Award_for_Best_Actor_on_Television\n",
      "#cite_note-41\n",
      "/wiki/Huffington_Post\n",
      "/wiki/Tremors_(1990_film)\n",
      "#cite_note-42\n",
      "/wiki/Tremors_5:_Bloodline\n",
      "/wiki/EE_(telecommunications_company)\n",
      "#cite_note-43\n",
      "#cite_note-44\n",
      "#cite_note-45\n",
      "/wiki/Six_Degrees_of_Kevin_Bacon\n",
      "/wiki/Trivia\n",
      "/wiki/Big_screen\n",
      "/wiki/Six_degrees_of_separation\n",
      "/wiki/Internet_meme\n",
      "/wiki/SixDegrees.org\n",
      "/wiki/Social_networking_service\n",
      "#cite_note-46\n",
      "/wiki/Six_Degrees_of_Kevin_Bacon\n",
      "/wiki/IMDb\n",
      "#cite_note-47\n",
      "/wiki/Paul_Erd%C5%91s\n",
      "/wiki/Erd%C5%91s_number\n",
      "/wiki/Paul_Erd%C5%91s\n",
      "/wiki/Erd%C5%91s_number\n",
      "/wiki/Erd%C5%91s%E2%80%93Bacon_number\n",
      "#cite_note-48\n",
      "/wiki/The_Bacon_Brothers\n",
      "/wiki/Michael_Bacon_(musician)\n",
      "#cite_note-49\n",
      "/wiki/Instagram\n",
      "#cite_note-50\n",
      "/wiki/Old_97%27s\n",
      "/wiki/The_Guardians_of_the_Galaxy_Holiday_Special\n",
      "#cite_note-51\n",
      "/wiki/Kyra_Sedgwick\n",
      "/wiki/PBS\n",
      "/wiki/Lanford_Wilson\n",
      "/wiki/Lemon_Sky\n",
      "#cite_note-cosmo91-12\n",
      "/wiki/Pyrates\n",
      "/wiki/Murder_in_the_First_(film)\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/Loverboy_(2005_film)\n",
      "/wiki/Sosie_Bacon\n",
      "/wiki/Upper_West_Side\n",
      "/wiki/Manhattan\n",
      "#cite_note-52\n",
      "/wiki/Tracy_Pollan\n",
      "#cite_note-53\n",
      "/wiki/Separation_of_church_and_state_in_the_United_States\n",
      "#cite_note-54\n",
      "#cite_note-55\n",
      "/wiki/The_Times\n",
      "/wiki/Atheism\n",
      "#cite_note-56\n",
      "/wiki/Antireligion\n",
      "#cite_note-57\n",
      "/wiki/Will.i.am\n",
      "/wiki/It%27s_a_New_Day_(Will.i.am_song)\n",
      "/wiki/Barack_Obama\n",
      "/wiki/Ponzi_scheme\n",
      "/wiki/Bernie_Madoff\n",
      "#cite_note-financialpost-58\n",
      "#cite_note-59\n",
      "/wiki/Finding_Your_Roots\n",
      "/wiki/Henry_Louis_Gates\n",
      "#cite_note-60\n",
      "#cite_note-61\n",
      "#cite_note-62\n",
      "#cite_note-63\n",
      "/wiki/Apollo_13_(film)\n",
      "#cite_note-64\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/Blockbuster_Entertainment_Awards\n",
      "/wiki/Blockbuster_Entertainment_Awards\n",
      "#cite_note-65\n",
      "/wiki/Hollow_Man\n",
      "/wiki/Boston_Society_of_Film_Critics\n",
      "/wiki/Boston_Society_of_Film_Critics_Award_for_Best_Cast\n",
      "#cite_note-66\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/Bravo_Otto\n",
      "/wiki/Bravo_Otto\n",
      "/wiki/Footloose_(1984_film)\n",
      "/wiki/CableACE_Award\n",
      "/wiki/CableACE_Award\n",
      "/wiki/Losing_Chase\n",
      "/wiki/Chlotrudis_Awards\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/Critics%27_Choice_Movie_Awards\n",
      "/wiki/Critics%27_Choice_Movie_Award_for_Best_Actor\n",
      "/wiki/Murder_in_the_First_(film)\n",
      "/wiki/Ghent_International_Film_Festival\n",
      "/wiki/Ghent_International_Film_Festival\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/Giffoni_Film_Festival\n",
      "/wiki/Giffoni_Film_Festival\n",
      "/wiki/Digging_to_China\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/Golden_Globe_Award\n",
      "/wiki/Golden_Globe_Award_for_Best_Supporting_Actor_%E2%80%93_Motion_Picture\n",
      "/wiki/The_River_Wild\n",
      "/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Miniseries_or_Television_Film\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\n",
      "/wiki/I_Love_Dick_(TV_series)\n",
      "/wiki/Independent_Spirit_Awards\n",
      "/wiki/Independent_Spirit_Award_for_Best_Male_Lead\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/MTV_Movie_%26_TV_Awards\n",
      "/wiki/MTV_Movie_Award_for_Best_Villain\n",
      "/wiki/Hollow_Man\n",
      "/wiki/Taking_Chance\n",
      "/wiki/The_Following\n",
      "/wiki/E!_People%27s_Choice_Awards\n",
      "/wiki/E!_People%27s_Choice_Awards\n",
      "/wiki/The_Following\n",
      "/wiki/E!_People%27s_Choice_Awards\n",
      "/wiki/The_Following\n",
      "/wiki/Primetime_Emmy_Award\n",
      "/wiki/Primetime_Emmy_Award_for_Outstanding_Lead_Actor_in_a_Limited_Series_or_Movie\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Satellite_Awards\n",
      "/wiki/Satellite_Award_for_Best_Actor_%E2%80%93_Motion_Picture\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/Satellite_Award_for_Best_Actor_%E2%80%93_Miniseries_or_Television_Film\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Saturn_Award\n",
      "/wiki/Saturn_Award_for_Best_Actor_on_Television\n",
      "/wiki/The_Following\n",
      "/wiki/Saturn_Award_for_Best_Actor_on_Television\n",
      "/wiki/The_Following\n",
      "/wiki/Scream_Awards\n",
      "/wiki/Scream_Awards\n",
      "/wiki/X-Men:_First_Class\n",
      "/wiki/Screen_Actors_Guild_Award\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Supporting_Role\n",
      "/wiki/Murder_in_the_First_(film)\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Cast_in_a_Motion_Picture\n",
      "/wiki/Apollo_13_(film)\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Cast_in_a_Motion_Picture\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Cast_in_a_Motion_Picture\n",
      "/wiki/Frost/Nixon_(film)\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Teen_Choice_Awards\n",
      "/wiki/Teen_Choice_Award_for_Choice_Movie_Villain\n",
      "/wiki/Beauty_Shop\n",
      "/wiki/Teen_Choice_Award_for_Choice_Movie_Villain\n",
      "/wiki/X-Men:_First_Class\n",
      "/wiki/TV_Guide_Award\n",
      "/wiki/TV_Guide_Award\n",
      "/wiki/The_Following\n",
      "/wiki/File:Kevin_Bacon%27s_Star_Walk_of_Fame.jpg\n",
      "/wiki/File:Kevin_Bacon%27s_Star_Walk_of_Fame.jpg\n",
      "/wiki/Hollywood_Walk_of_Fame\n",
      "/wiki/Hollywood_Walk_of_Fame\n",
      "#cite_note-67\n",
      "/wiki/Denver_Film_Festival\n",
      "#cite_note-68\n",
      "/wiki/Phoenix_Film_Festival\n",
      "#cite_note-69\n",
      "/wiki/Santa_Barbara_International_Film_Festival\n",
      "#cite_note-70\n",
      "/wiki/Broadcast_Film_Critics_Association\n",
      "#cite_note-71\n",
      "/wiki/Seattle_International_Film_Festival\n",
      "#cite_note-72\n",
      "/wiki/List_of_actors_with_Hollywood_Walk_of_Fame_motion_picture_stars\n",
      "#cite_ref-1\n",
      "http://www.geneall.net/U/per_page.php?id=1014399\n",
      "#cite_ref-2\n",
      "https://web.archive.org/web/20090113222205/http://www.newenglandancestors.org/research/services/articles_gbr78.asp\n",
      "http://www.newenglandancestors.org/research/services/articles_gbr78.asp\n",
      "#cite_ref-actor_3-0\n",
      "#cite_ref-actor_3-1\n",
      "#cite_ref-actor_3-2\n",
      "http://www.biography.com/people/kevin-bacon-9542173\n",
      "#cite_ref-4\n",
      "https://www.theguardian.com/film/filmblog/2009/feb/19/best-actors-never-nominated-for-oscars\n",
      "#cite_ref-5\n",
      "http://www.walkoffame.com/kevin-bacon\n",
      "#cite_ref-6\n",
      "https://www.hollywoodreporter.com/tv/tv-news/showtime-cancels-city-on-a-hill-3-seasons-1235250089/\n",
      "/wiki/The_Hollywood_Reporter\n",
      "#cite_ref-walk_7-0\n",
      "#cite_ref-walk_7-1\n",
      "https://web.archive.org/web/20141016202657/http://www.thebiographychannel.co.uk/biographies/kevin-bacon.html\n",
      "http://www.thebiographychannel.co.uk/biographies/kevin-bacon.html\n",
      "#cite_ref-8\n",
      "https://www.marketingweek.com/ee-unveils-six-degrees-of-bacon-launch-ads/\n",
      "#cite_ref-bacon_9-0\n",
      "#cite_ref-bacon_9-1\n",
      "#cite_ref-bacon_9-2\n",
      "#cite_ref-bacon_9-3\n",
      "https://web.archive.org/web/20190403203113/https://www.biography.com/news/kevin-bacon-biography-facts\n",
      "http://www.biography.com/news/kevin-bacon-biography-facts\n",
      "#cite_ref-10\n",
      "https://philadelphia.cbslocal.com/top-lists/stars-from-philly-to-hollywood/\n",
      "#cite_ref-11\n",
      "https://movies.yahoo.com/person/kevin-bacon/biography.html\n",
      "#cite_ref-cosmo91_12-0\n",
      "#cite_ref-cosmo91_12-1\n",
      "#cite_ref-13\n",
      "http://www.nydailynews.com/entertainment/happy-halloween-superstars-start-horror-flick-gallery-1.98345\n",
      "#cite_ref-bio_14-0\n",
      "#cite_ref-bio_14-1\n",
      "#cite_ref-bio_14-2\n",
      "#cite_ref-bio_14-3\n",
      "#cite_ref-bio_14-4\n",
      "#cite_ref-bio_14-5\n",
      "#cite_ref-bio_14-6\n",
      "#cite_ref-bio_14-7\n",
      "#cite_ref-bio_14-8\n",
      "#cite_ref-bio_14-9\n",
      "#cite_ref-bio_14-10\n",
      "https://web.archive.org/web/20160530033240/http://www.pbs.org/weta/finding-your-roots/profiles/kevin-bacon%c2%a0/\n",
      "https://www.pbs.org/wnet/finding-your-roots/profiles/kevin-bacon%C2%A0/\n",
      "#cite_ref-kevin_15-0\n",
      "https://www.tvguide.com/celebrities/kevin-bacon/bio/160550\n",
      "#cite_ref-16\n",
      "https://web.archive.org/web/20141021030336/http://news.moviefone.com/2012/03/02/diner-30th-anniversary/\n",
      "http://news.moviefone.com/2012/03/02/diner-30th-anniversary/\n",
      "#cite_ref-time84_17-0\n",
      "https://web.archive.org/web/20080830035710/http://www.time.com/time/magazine/article/0,9171,950019,00.html\n",
      "http://www.time.com/time/magazine/article/0,9171,950019,00.html\n",
      "#cite_ref-18\n",
      "http://www.huffingtonpost.com/2014/08/25/kevin-bacon-footloose_n_5710413.html\n",
      "#cite_ref-19\n",
      "https://web.archive.org/web/20090109152125/http://www.thebiographychannel.co.uk/biography_story/522%3A492/1/Kevin_Bacon.htm\n",
      "http://www.thebiographychannel.co.uk/biography_story/522:492/1/Kevin_Bacon.htm\n",
      "#cite_ref-20\n",
      "https://www.mentalfloss.com/article/71594/14-moving-facts-about-planes-trains-and-automobiles\n",
      "#cite_ref-21\n",
      "https://www.nytimes.com/1994/09/25/movies/a-second-wind-is-blowing-for-kevin-bacon.html\n",
      "#cite_ref-22\n",
      "https://www.nytimes.com/movie/review?res=9C0CE2DE1631F93AA25752C0A966958260\n",
      "#cite_ref-nyt94_23-0\n",
      "https://query.nytimes.com/gst/fullpage.html?res=9C07E6D91F3BF936A1575AC0A962958260\n",
      "#cite_ref-24\n",
      "http://www.jfk-online.com/jfkbacon.html\n",
      "#cite_ref-25\n",
      "http://www.tcm.com/this-month/article/143158%7C0/A-Few-Good-Men.html\n",
      "#cite_ref-26\n",
      "http://collider.com/kevin-bacon-commercials-footloose/\n",
      "#cite_ref-27\n",
      "http://www.rogerebert.com/reviews/sleepers-1996\n",
      "#cite_ref-austin_28-0\n",
      "http://www.austinchronicle.com/calendar/film/1997-02-07/283342/\n",
      "/wiki/The_Austin_Chronicle\n",
      "#cite_ref-29\n",
      "http://www.criminalelement.com/blogs/2013/09/under-the-raderhorror-movies-you-may-have-missed-stir-of-echoes\n",
      "#cite_ref-30\n",
      "http://www.rogerebert.com/reviews/hollow-man-2000\n",
      "#cite_ref-31\n",
      "https://web.archive.org/web/20141017080013/http://movies.about.com/od/wherethetruthlies/a/truthkb101305.htm\n",
      "http://movies.about.com/od/wherethetruthlies/a/truthkb101305.htm\n",
      "#cite_ref-32\n",
      "https://archive.today/20120604150801/http://jam.canoe.ca/Movies/2005/09/14/1216527.html\n",
      "http://jam.canoe.ca/Movies/2005/09/14/1216527.html\n",
      "#cite_ref-33\n",
      "https://web.archive.org/web/20140802203026/http://www.latimes.com/entertainment/la-et-kevin-bacon-photo6-photo.html\n",
      "http://www.latimes.com/entertainment/la-et-kevin-bacon-photo6-photo.html\n",
      "#cite_ref-34\n",
      "http://www.nydailynews.com/entertainment/tv-movies/kevin-bacon-chance-body-fallen-marine-home-article-1.392226\n",
      "#cite_ref-35\n",
      "https://web.archive.org/web/20100722010545/http://heatvision.hollywoodreporter.com/2010/07/winters-bone-star-cast-as-mystique-in-xmen-first-class.html\n",
      "http://heatvision.hollywoodreporter.com/2010/07/winters-bone-star-cast-as-mystique-in-xmen-first-class.html\n",
      "#cite_ref-36\n",
      "https://web.archive.org/web/20100720060214/http://www.forcesofgeek.com/2010/07/kevin-bacon-playing-sebastian-shaw-in-x.html\n",
      "http://www.forcesofgeek.com/2010/07/kevin-bacon-playing-sebastian-shaw-in-x.html\n",
      "#cite_ref-8_the_play_37-0\n",
      "http://www.accesshollywood.com/jesse-tyler-ferguson/glee-stars-touched-by-brad-pitt-and-george-clooneys-support-of-8_article_61543\n",
      "/wiki/Access_Hollywood\n",
      "#cite_ref-8_play_video_38-0\n",
      "https://www.youtube.com/watch?v=qlUG8F9uVgM\n",
      "https://ghostarchive.org/varchive/youtube/20211211/qlUG8F9uVgM\n",
      "#cite_ref-39\n",
      "http://www.pinknews.co.uk/2012/03/01/youtube-to-broadcast-proposition-8-play-live/\n",
      "#cite_ref-40\n",
      "http://www.fox.com/the-following/\n",
      "#cite_ref-41\n",
      "https://news.yahoo.com/blogs/trending-now/kevin-bacon-gives-millennials-a-history-lesson-about-the--80s-162525915.html\n",
      "#cite_ref-42\n",
      "http://www.huffingtonpost.com.au/entry/kevin-bacon-tremors-tv-reboot_us_5655b651e4b072e9d1c13a11\n",
      "#cite_ref-43\n",
      "http://www.campaignlive.co.uk/news/1294856/\n",
      "#cite_ref-44\n",
      "http://parade.condenast.com/269380/ashleighschmitz/kevin-bacon-reprises-his-most-iconic-film-roles-in-british-commercial/\n",
      "#cite_ref-45\n",
      "https://money.cnn.com/2015/03/13/media/kevin-bacon-eggs/index.html\n",
      "#cite_ref-46\n",
      "http://www.sixdegrees.org/\n",
      "#cite_ref-47\n",
      "http://www.webmonkey.com/2012/09/easter-egg-google-connects-the-dots-for-bacon-number-search/\n",
      "#cite_ref-48\n",
      "https://web.archive.org/web/20121112081753/http://www.telegraph.co.uk/science/science-news/4768389/And-the-winner-tonight-is.html\n",
      "https://www.telegraph.co.uk/science/science-news/4768389/And-the-winner-tonight-is.html\n",
      "#cite_ref-49\n",
      "http://baconbros.com/\n",
      "#cite_ref-50\n",
      "https://www.instagram.com/p/ClUvj92p4Qn/?hl=en\n",
      "#cite_ref-51\n",
      "https://www.marvel.com/articles/tv-shows/guardians-of-the-galaxy-holiday-special-soundtrack\n",
      "#cite_ref-52\n",
      "http://www.nydailynews.com/entertainment/tv-movies/kevin-bacon-loyalty-nyc-philly-origins-peace-bustling-city-article-1.147197\n",
      "#cite_ref-53\n",
      "http://www.people.com/people/archive/article/0,,20093025,00.html\n",
      "#cite_ref-54\n",
      "https://web.archive.org/web/20141023014658/https://www.au.org/media/church-and-state/archives/2008/05/two-thumbs-up.html\n",
      "http://www.au.org/media/church-and-state/archives/2008/05/two-thumbs-up.html\n",
      "#cite_ref-55\n",
      "https://www.washingtonpost.com/wp-dyn/content/article/2008/03/25/AR2008032503852.html\n",
      "#cite_ref-56\n",
      "#cite_ref-57\n",
      "http://www.foxnews.com/story/0,2933,343589,00.html\n",
      "#cite_ref-financialpost_58-0\n",
      "https://web.archive.org/web/20140314085857/http://economiccrisis.us/2009/06/may-god-spare-mercy-victim-tells-madoff/\n",
      "http://economiccrisis.us/2009/06/may-god-spare-mercy-victim-tells-madoff/\n",
      "#cite_ref-59\n",
      "#cite_ref-60\n",
      "http://www.huffingtonpost.com/megan-smolenyak-smolenyak/6-degrees-of-separation-k_b_900707.html\n",
      "#cite_ref-61\n",
      "https://web.archive.org/web/20130405182304/http://www.drawtheline.org/watch-stuff/\n",
      "http://www.drawtheline.org/watch-stuff\n",
      "#cite_ref-62\n",
      "http://www.drawtheline.org/sign-now/\n",
      "#cite_ref-63\n",
      "http://www.awardscircuit.com/2015/07/01/pixars-toy-story-wins-top-prize-for-1995-awards-circuit-community-awards/\n",
      "#cite_ref-64\n",
      "http://www.awardscircuit.com/acca/the-acca-nominations-2003/\n",
      "#cite_ref-65\n",
      "https://idobi.com/news/nsync-takes-home-three-blockbuster-entertainment-awards/\n",
      "#cite_ref-66\n",
      "https://www.upi.com/Boston-critics-pick-Mystic-River/24961071506935/\n",
      "#cite_ref-67\n",
      "http://www.walkoffame.com/kevin-bacon\n",
      "#cite_ref-68\n",
      "https://www.imdb.com/event/ev0000209/2004/1/\n",
      "#cite_ref-69\n",
      "https://www.imdb.com/event/ev0000536/2005/1/\n",
      "#cite_ref-70\n",
      "https://www.imdb.com/event/ev0000589/2005/1/\n",
      "#cite_ref-71\n",
      "http://www.criticschoice.com/movie-awards/critics%E2%80%99-choice-movie-awards-winners-archive/\n",
      "#cite_ref-72\n",
      "https://www.imdb.com/event/ev0000600/2015/1/\n",
      "https://commons.wikimedia.org/wiki/Category:Kevin_Bacon\n",
      "https://www.imdb.com/name/nm0000102/\n",
      "/wiki/IMDb\n",
      "https://www.ibdb.com/broadway-cast-staff/90569\n",
      "/wiki/Internet_Broadway_Database\n",
      "https://www.wikidata.org/wiki/Q3454165#P1220\n",
      "http://www.iobdb.com/CreditableEntity/5597\n",
      "/wiki/Internet_Off-Broadway_Database\n",
      "https://www.allmovie.com/artist/p3164\n",
      "/wiki/AllMovie\n",
      "http://oracleofbacon.org\n",
      "/wiki/Template:Kevin_Bacon\n",
      "/wiki/Template_talk:Kevin_Bacon\n",
      "https://en.wikipedia.org/w/index.php?title=Template:Kevin_Bacon&action=edit\n",
      "/wiki/Kevin_Bacon_filmography\n",
      "/wiki/Losing_Chase\n",
      "/wiki/Loverboy_(2005_film)\n",
      "/wiki/Kyra_Sedgwick\n",
      "/wiki/Sosie_Bacon\n",
      "/wiki/Edmund_Bacon_(architect)\n",
      "/wiki/Michael_Bacon_(musician)\n",
      "/wiki/The_Bacon_Brothers\n",
      "/wiki/Six_Degrees_of_Kevin_Bacon\n",
      "/wiki/Erd%C5%91s%E2%80%93Bacon_number\n",
      "/wiki/SixDegrees.org\n",
      "/wiki/Template:Critics%27_Choice_Movie_Award_for_Best_Actor\n",
      "/wiki/Template_talk:Critics%27_Choice_Movie_Award_for_Best_Actor\n",
      "https://en.wikipedia.org/w/index.php?title=Template:Critics%27_Choice_Movie_Award_for_Best_Actor&action=edit\n",
      "/wiki/Critics%27_Choice_Movie_Award_for_Best_Actor\n",
      "/wiki/Geoffrey_Rush\n",
      "/wiki/Jack_Nicholson\n",
      "/wiki/Ian_McKellen\n",
      "/wiki/Russell_Crowe\n",
      "/wiki/Russell_Crowe\n",
      "/wiki/Russell_Crowe\n",
      "/wiki/Daniel_Day-Lewis\n",
      "/wiki/Jack_Nicholson\n",
      "/wiki/Sean_Penn\n",
      "/wiki/Jamie_Foxx\n",
      "/wiki/Philip_Seymour_Hoffman\n",
      "/wiki/Forest_Whitaker\n",
      "/wiki/Daniel_Day-Lewis\n",
      "/wiki/Sean_Penn\n",
      "/wiki/Jeff_Bridges\n",
      "/wiki/Colin_Firth\n",
      "/wiki/George_Clooney\n",
      "/wiki/Daniel_Day-Lewis\n",
      "/wiki/Matthew_McConaughey\n",
      "/wiki/Michael_Keaton\n",
      "/wiki/Leonardo_DiCaprio\n",
      "/wiki/Casey_Affleck\n",
      "/wiki/Gary_Oldman\n",
      "/wiki/Christian_Bale\n",
      "/wiki/Joaquin_Phoenix\n",
      "/wiki/Chadwick_Boseman\n",
      "/wiki/Will_Smith\n",
      "/wiki/Brendan_Fraser\n",
      "/wiki/Template:Golden_Globe_Best_Actor_TV_Miniseries_Film\n",
      "/wiki/Template_talk:Golden_Globe_Best_Actor_TV_Miniseries_Film\n",
      "https://en.wikipedia.org/w/index.php?title=Template:Golden_Globe_Best_Actor_TV_Miniseries_Film&action=edit\n",
      "/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Miniseries_or_Television_Film\n",
      "/wiki/Mickey_Rooney\n",
      "/wiki/Anthony_Andrews\n",
      "/wiki/Richard_Chamberlain\n",
      "/wiki/Ted_Danson\n",
      "/wiki/Dustin_Hoffman\n",
      "/wiki/James_Woods\n",
      "/wiki/Randy_Quaid\n",
      "/wiki/Michael_Caine\n",
      "/wiki/Stacy_Keach\n",
      "/wiki/Robert_Duvall\n",
      "/wiki/James_Garner\n",
      "/wiki/Beau_Bridges\n",
      "/wiki/Robert_Duvall\n",
      "/wiki/James_Garner\n",
      "/wiki/Raul_Julia\n",
      "/wiki/Gary_Sinise\n",
      "/wiki/Alan_Rickman\n",
      "/wiki/Ving_Rhames\n",
      "/wiki/Stanley_Tucci\n",
      "/wiki/Jack_Lemmon\n",
      "/wiki/Brian_Dennehy\n",
      "/wiki/James_Franco\n",
      "/wiki/Albert_Finney\n",
      "/wiki/Al_Pacino\n",
      "/wiki/Geoffrey_Rush\n",
      "/wiki/Jonathan_Rhys_Meyers\n",
      "/wiki/Bill_Nighy\n",
      "/wiki/Jim_Broadbent\n",
      "/wiki/Paul_Giamatti\n",
      "/wiki/Al_Pacino\n",
      "/wiki/Idris_Elba\n",
      "/wiki/Kevin_Costner\n",
      "/wiki/Michael_Douglas\n",
      "/wiki/Billy_Bob_Thornton\n",
      "/wiki/Oscar_Isaac\n",
      "/wiki/Tom_Hiddleston\n",
      "/wiki/Ewan_McGregor\n",
      "/wiki/Darren_Criss\n",
      "/wiki/Russell_Crowe\n",
      "/wiki/Mark_Ruffalo\n",
      "/wiki/Michael_Keaton\n",
      "/wiki/Evan_Peters\n",
      "/wiki/Template:Saturn_Award_for_Best_Actor_on_Television\n",
      "/wiki/Template_talk:Saturn_Award_for_Best_Actor_on_Television\n",
      "https://en.wikipedia.org/w/index.php?title=Template:Saturn_Award_for_Best_Actor_on_Television&action=edit\n",
      "/wiki/Saturn_Award_for_Best_Actor_on_Television\n",
      "/wiki/Saturn_Award_for_Best_Actor_on_Television\n",
      "/wiki/Kyle_Chandler\n",
      "/wiki/Steven_Weber_(actor)\n",
      "/wiki/Richard_Dean_Anderson\n",
      "/wiki/David_Boreanaz\n",
      "/wiki/Robert_Patrick\n",
      "/wiki/Ben_Browder\n",
      "/wiki/David_Boreanaz\n",
      "/wiki/David_Boreanaz\n",
      "/wiki/Ben_Browder\n",
      "/wiki/Matthew_Fox\n",
      "/wiki/Michael_C._Hall\n",
      "/wiki/Matthew_Fox\n",
      "/wiki/Edward_James_Olmos\n",
      "/wiki/Josh_Holloway\n",
      "/wiki/Stephen_Moyer\n",
      "/wiki/Bryan_Cranston\n",
      "/wiki/Bryan_Cranston\n",
      "/wiki/Mads_Mikkelsen\n",
      "/wiki/Hugh_Dancy\n",
      "/wiki/Andrew_Lincoln\n",
      "/wiki/Bruce_Campbell\n",
      "/wiki/Andrew_Lincoln\n",
      "/wiki/Kyle_MacLachlan\n",
      "/wiki/Patrick_Stewart\n",
      "/wiki/Saturn_Award_for_Best_Actor_in_a_Network_or_Cable_Television_Series\n",
      "/wiki/Sam_Heughan\n",
      "/wiki/Bob_Odenkirk\n",
      "/wiki/Saturn_Award_for_Best_Actor_in_a_Streaming_Television_Series\n",
      "/wiki/Henry_Thomas\n",
      "/wiki/Oscar_Isaac\n",
      "/wiki/Template:ScreenActorsGuildAward_MaleTVMiniseriesMovie\n",
      "/wiki/Template_talk:ScreenActorsGuildAward_MaleTVMiniseriesMovie\n",
      "https://en.wikipedia.org/w/index.php?title=Template:ScreenActorsGuildAward_MaleTVMiniseriesMovie&action=edit\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie\n",
      "/wiki/Raul_Julia\n",
      "/wiki/Gary_Sinise\n",
      "/wiki/Alan_Rickman\n",
      "/wiki/Gary_Sinise\n",
      "/wiki/Christopher_Reeve\n",
      "/wiki/Jack_Lemmon\n",
      "/wiki/Brian_Dennehy\n",
      "/wiki/Ben_Kingsley\n",
      "/wiki/William_H._Macy\n",
      "/wiki/Al_Pacino\n",
      "/wiki/Geoffrey_Rush\n",
      "/wiki/Paul_Newman\n",
      "/wiki/Jeremy_Irons\n",
      "/wiki/Kevin_Kline\n",
      "/wiki/Paul_Giamatti\n",
      "/wiki/Al_Pacino\n",
      "/wiki/Paul_Giamatti\n",
      "/wiki/Kevin_Costner\n",
      "/wiki/Michael_Douglas\n",
      "/wiki/Mark_Ruffalo\n",
      "/wiki/Idris_Elba\n",
      "/wiki/Bryan_Cranston\n",
      "/wiki/Alexander_Skarsg%C3%A5rd\n",
      "/wiki/Darren_Criss\n",
      "/wiki/Sam_Rockwell\n",
      "/wiki/Mark_Ruffalo\n",
      "/wiki/Michael_Keaton\n",
      "/wiki/Sam_Elliott\n",
      "/wiki/Help:Authority_control\n",
      "https://www.wikidata.org/wiki/Q3454165#identifiers\n",
      "http://id.worldcat.org/fast/242183/\n",
      "https://isni.org/isni/0000000121291300\n",
      "https://viaf.org/viaf/39570812\n",
      "https://www.worldcat.org/identities/lccn-n88034930/\n",
      "https://authority.bibsys.no/authority/rest/authorities/html/98015093\n",
      "http://catalogo.bne.es/uhtbin/authoritybrowse.cgi?action=display&authority_id=XX1298810\n",
      "https://catalogue.bnf.fr/ark:/12148/cb139817766\n",
      "https://data.bnf.fr/ark:/12148/cb139817766\n",
      "https://d-nb.info/gnd/124109659\n",
      "http://uli.nli.org.il/F/?func=find-b&local_base=NLX10&find_code=UID&request=987007454685905171\n",
      "https://id.loc.gov/authorities/names/n88034930\n",
      "https://kopkatalogs.lv/F?func=direct&local_base=lnc10&doc_number=000249798&P_CON_LNG=ENG\n",
      "https://aleph.nkp.cz/F/?func=find-c&local_base=aut&ccl_term=ica=xx0025279&CON_LNG=ENG\n",
      "https://nla.gov.au/anbd.aut-an36021861\n",
      "https://lod.nl.go.kr/resource/KAC2020K8226\n",
      "http://data.bibliotheken.nl/id/thes/p147880998\n",
      "http://mak.bn.org.pl/cgi-bin/KHW/makwww.exe?BM=1&NU=1&IM=4&WI=9810630778405606\n",
      "https://musicbrainz.org/artist/cc0dbdfc-9b2c-4e31-8448-808412388406\n",
      "https://trove.nla.gov.au/people/1189569\n",
      "https://snaccooperative.org/ark:/99166/w6w67gw2\n",
      "https://www.idref.fr/067287832\n",
      "https://en.wikipedia.org/w/index.php?title=Kevin_Bacon&oldid=1146341850\n",
      "/wiki/Help:Category\n",
      "/wiki/Category:1958_births\n",
      "/wiki/Category:20th-century_American_male_actors\n",
      "/wiki/Category:21st-century_American_male_actors\n",
      "/wiki/Category:American_atheists\n",
      "/wiki/Category:American_male_film_actors\n",
      "/wiki/Category:American_male_soap_opera_actors\n",
      "/wiki/Category:American_male_television_actors\n",
      "/wiki/Category:American_male_voice_actors\n",
      "/wiki/Category:The_Bacon_Brothers_members\n",
      "/wiki/Category:Best_Miniseries_or_Television_Movie_Actor_Golden_Globe_winners\n",
      "/wiki/Category:Circle_in_the_Square_Theatre_School_alumni\n",
      "/wiki/Category:Living_people\n",
      "/wiki/Category:Male_actors_from_Philadelphia\n",
      "/wiki/Category:Obie_Award_recipients\n",
      "/wiki/Category:Outstanding_Performance_by_a_Cast_in_a_Motion_Picture_Screen_Actors_Guild_Award_winners\n",
      "/wiki/Category:Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie_Screen_Actors_Guild_Award_winners\n",
      "/wiki/Category:Sedgwick_family\n",
      "/wiki/Category:People_from_the_Upper_West_Side\n",
      "/wiki/Category:Articles_with_short_description\n",
      "/wiki/Category:Short_description_is_different_from_Wikidata\n",
      "/wiki/Category:Wikipedia_indefinitely_semi-protected_biographies_of_living_people\n",
      "/wiki/Category:Use_mdy_dates_from_October_2022\n",
      "/wiki/Category:All_articles_with_unsourced_statements\n",
      "/wiki/Category:Articles_with_unsourced_statements_from_October_2021\n",
      "/wiki/Category:Articles_with_hCards\n",
      "/wiki/Category:Commons_category_link_from_Wikidata\n",
      "/wiki/Category:IBDB_name_template_using_Wikidata\n",
      "/wiki/Category:Internet_Off-Broadway_Database_person_ID_same_as_Wikidata\n",
      "/wiki/Category:Articles_with_FAST_identifiers\n",
      "/wiki/Category:Articles_with_ISNI_identifiers\n",
      "/wiki/Category:Articles_with_VIAF_identifiers\n",
      "/wiki/Category:Articles_with_WorldCat_identifiers\n",
      "/wiki/Category:Articles_with_BIBSYS_identifiers\n",
      "/wiki/Category:Articles_with_BNE_identifiers\n",
      "/wiki/Category:Articles_with_BNF_identifiers\n",
      "/wiki/Category:Articles_with_BNFdata_identifiers\n",
      "/wiki/Category:Articles_with_GND_identifiers\n",
      "/wiki/Category:Articles_with_J9U_identifiers\n",
      "/wiki/Category:Articles_with_LCCN_identifiers\n",
      "/wiki/Category:Articles_with_LNB_identifiers\n",
      "/wiki/Category:Articles_with_NKC_identifiers\n",
      "/wiki/Category:Articles_with_NLA_identifiers\n",
      "/wiki/Category:Articles_with_NLK_identifiers\n",
      "/wiki/Category:Articles_with_NTA_identifiers\n",
      "/wiki/Category:Articles_with_PLWABN_identifiers\n",
      "/wiki/Category:Articles_with_MusicBrainz_identifiers\n",
      "/wiki/Category:Articles_with_Trove_identifiers\n",
      "/wiki/Category:Articles_with_SNAC-ID_identifiers\n",
      "/wiki/Category:Articles_with_SUDOC_identifiers\n",
      "//en.wikipedia.org/wiki/Wikipedia:Text_of_the_Creative_Commons_Attribution-ShareAlike_3.0_Unported_License\n",
      "//creativecommons.org/licenses/by-sa/3.0/\n",
      "//foundation.wikimedia.org/wiki/Terms_of_Use\n",
      "//foundation.wikimedia.org/wiki/Privacy_policy\n",
      "//www.wikimediafoundation.org/\n",
      "https://foundation.wikimedia.org/wiki/Privacy_policy\n",
      "/wiki/Wikipedia:About\n",
      "/wiki/Wikipedia:General_disclaimer\n",
      "//en.wikipedia.org/wiki/Wikipedia:Contact_us\n",
      "//en.m.wikipedia.org/w/index.php?title=Kevin_Bacon&mobileaction=toggle_view_mobile\n",
      "https://developer.wikimedia.org\n",
      "https://stats.wikimedia.org/#/en.wikipedia.org\n",
      "https://foundation.wikimedia.org/wiki/Cookie_statement\n",
      "https://wikimediafoundation.org/\n",
      "https://www.mediawiki.org/\n"
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup \n",
    "\n",
    "html = urlopen('http://en.wikipedia.org/wiki/Kevin_Bacon')\n",
    "bs = BeautifulSoup(html, 'html.parser')\n",
    "for link in bs.find_all('a'):\n",
    "    if 'href' in link.attrs:\n",
    "        print(link.attrs['href'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieving Articles Only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/wiki/Kevin_Bacon_(disambiguation)\n",
      "/wiki/Philadelphia\n",
      "/wiki/Kevin_Bacon_filmography\n",
      "/wiki/Kyra_Sedgwick\n",
      "/wiki/Sosie_Bacon\n",
      "/wiki/Edmund_Bacon_(architect)\n",
      "/wiki/Michael_Bacon_(musician)\n",
      "/wiki/Leading_man\n",
      "/wiki/Character_actor\n",
      "/wiki/Golden_Globe_Award\n",
      "/wiki/Screen_Actors_Guild_Award\n",
      "/wiki/Primetime_Emmy_Award\n",
      "/wiki/National_Lampoon%27s_Animal_House\n",
      "/wiki/Footloose_(1984_film)\n",
      "/wiki/Diner_(1982_film)\n",
      "/wiki/JFK_(film)\n",
      "/wiki/A_Few_Good_Men\n",
      "/wiki/Apollo_13_(film)\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/Frost/Nixon_(film)\n",
      "/wiki/Friday_the_13th_(1980_film)\n",
      "/wiki/Tremors_(1990_film)\n",
      "/wiki/The_River_Wild\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/Crazy,_Stupid,_Love\n",
      "/wiki/Patriots_Day_(film)\n",
      "/wiki/Losing_Chase\n",
      "/wiki/Loverboy_(2005_film)\n",
      "/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Miniseries_or_Television_Film\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie\n",
      "/wiki/Michael_Strobl\n",
      "/wiki/HBO\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Fox_Broadcasting_Company\n",
      "/wiki/The_Following\n",
      "/wiki/Amazon_Prime_Video\n",
      "/wiki/I_Love_Dick_(TV_series)\n",
      "/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\n",
      "/wiki/Showtime_(TV_network)\n",
      "/wiki/City_on_a_Hill_(TV_series)\n",
      "/wiki/The_Guardian\n",
      "/wiki/Academy_Award\n",
      "/wiki/Hollywood_Walk_of_Fame\n",
      "/wiki/Six_Degrees_of_Kevin_Bacon\n",
      "/wiki/EE_Limited\n",
      "/wiki/Kyra_Sedgwick\n",
      "/wiki/Philadelphia\n",
      "/wiki/Edmund_Bacon_(architect)\n",
      "/wiki/Urban_planning\n",
      "/wiki/Design_of_Cities\n",
      "/wiki/Julia_R._Masterman_School\n",
      "/wiki/Spring_Garden,_Philadelphia\n",
      "/wiki/Pennsylvania_Governor%27s_School_for_the_Arts\n",
      "/wiki/Bucknell_University\n",
      "/wiki/Lewisburg,_Pennsylvania\n",
      "/wiki/Glory_Van_Scott\n",
      "/wiki/Kevin_Bacon_filmography\n",
      "/wiki/Circle_in_the_Square\n",
      "/wiki/Nancy_Mills\n",
      "/wiki/Cosmopolitan_(magazine)\n",
      "/wiki/Fraternities_and_sororities\n",
      "/wiki/Animal_House\n",
      "/wiki/Search_for_Tomorrow\n",
      "/wiki/Guiding_Light\n",
      "/wiki/Friday_the_13th_(1980_film)\n",
      "/wiki/Getting_Out\n",
      "/wiki/Phoenix_Theater\n",
      "/wiki/Flux\n",
      "/wiki/Second_Stage_Theatre\n",
      "/wiki/Obie_Award\n",
      "/wiki/Forty_Deuce\n",
      "/wiki/Slab_Boys\n",
      "/wiki/Sean_Penn\n",
      "/wiki/Val_Kilmer\n",
      "/wiki/Barry_Levinson\n",
      "/wiki/Diner_(1982_film)\n",
      "/wiki/Steve_Guttenberg\n",
      "/wiki/Daniel_Stern_(actor)\n",
      "/wiki/Mickey_Rourke\n",
      "/wiki/Tim_Daly\n",
      "/wiki/Ellen_Barkin\n",
      "/wiki/Footloose_(1984_film)\n",
      "/wiki/James_Dean\n",
      "/wiki/Rebel_Without_a_Cause\n",
      "/wiki/Mickey_Rooney\n",
      "/wiki/Judy_Garland\n",
      "/wiki/Typecasting_(acting)\n",
      "/wiki/John_Hughes_(filmmaker)\n",
      "/wiki/Planes,_Trains_and_Automobiles\n",
      "/wiki/She%27s_Having_a_Baby\n",
      "/wiki/The_Big_Picture_(1989_film)\n",
      "/wiki/Tremors_(1990_film)\n",
      "/wiki/Joel_Schumacher\n",
      "/wiki/Flatliners\n",
      "/wiki/Elizabeth_Perkins\n",
      "/wiki/He_Said,_She_Said_(film)\n",
      "/wiki/The_New_York_Times\n",
      "/wiki/Oliver_Stone\n",
      "/wiki/JFK_(film)\n",
      "/wiki/A_Few_Good_Men_(film)\n",
      "/wiki/Michael_Greif\n",
      "/wiki/Golden_Globe_Award\n",
      "/wiki/The_River_Wild\n",
      "/wiki/Meryl_Streep\n",
      "/wiki/Murder_in_the_First_(film)\n",
      "/wiki/Blockbuster_(entertainment)\n",
      "/wiki/Apollo_13_(film)\n",
      "/wiki/Sleepers_(film)\n",
      "/wiki/Picture_Perfect_(1997_film)\n",
      "/wiki/Losing_Chase\n",
      "/wiki/Digging_to_China\n",
      "/wiki/Payola\n",
      "/wiki/Telling_Lies_in_America_(film)\n",
      "/wiki/Wild_Things_(film)\n",
      "/wiki/Stir_of_Echoes\n",
      "/wiki/David_Koepp\n",
      "/wiki/Cannes_Film_Festival\n",
      "/wiki/Paul_Verhoeven\n",
      "/wiki/Hollow_Man\n",
      "/wiki/Colin_Firth\n",
      "/wiki/Rachel_Blanchard\n",
      "/wiki/M%C3%A9nage_%C3%A0_trois\n",
      "/wiki/Where_the_Truth_Lies\n",
      "/wiki/Atom_Egoyan\n",
      "/wiki/MPAA\n",
      "/wiki/MPAA_film_rating_system\n",
      "/wiki/Sean_Penn\n",
      "/wiki/Tim_Robbins\n",
      "/wiki/Clint_Eastwood\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/HBO_Films\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Michael_Strobl\n",
      "/wiki/Desert_Storm\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie\n",
      "/wiki/Toronto_Film_Festival\n",
      "/wiki/Matthew_Vaughn\n",
      "/wiki/Sebastian_Shaw_(comics)\n",
      "/wiki/Dustin_Lance_Black\n",
      "/wiki/8_(play)\n",
      "/wiki/Perry_v._Brown\n",
      "/wiki/Proposition_8\n",
      "/wiki/Charles_J._Cooper\n",
      "/wiki/Wilshire_Ebell_Theatre\n",
      "/wiki/American_Foundation_for_Equal_Rights\n",
      "/wiki/The_Following\n",
      "/wiki/Saturn_Award_for_Best_Actor_on_Television\n",
      "/wiki/Huffington_Post\n",
      "/wiki/Tremors_(1990_film)\n",
      "/wiki/The_Bacon_Brothers\n",
      "/wiki/Michael_Bacon_(musician)\n",
      "/wiki/Instagram\n",
      "/wiki/Old_97%27s\n",
      "/wiki/The_Guardians_of_the_Galaxy_Holiday_Special\n",
      "/wiki/EE_(telecommunications_company)\n",
      "/wiki/Six_Degrees_of_Kevin_Bacon\n",
      "/wiki/Trivia\n",
      "/wiki/Big_screen\n",
      "/wiki/Six_degrees_of_separation\n",
      "/wiki/Internet_meme\n",
      "/wiki/SixDegrees.org\n",
      "/wiki/Social_networking_service\n",
      "/wiki/Six_Degrees_of_Kevin_Bacon\n",
      "/wiki/IMDb\n",
      "/wiki/Paul_Erd%C5%91s\n",
      "/wiki/Erd%C5%91s_number\n",
      "/wiki/Paul_Erd%C5%91s\n",
      "/wiki/Erd%C5%91s_number\n",
      "/wiki/Erd%C5%91s%E2%80%93Bacon_number\n",
      "/wiki/Kyra_Sedgwick\n",
      "/wiki/PBS\n",
      "/wiki/Lanford_Wilson\n",
      "/wiki/Lemon_Sky\n",
      "/wiki/Pyrates\n",
      "/wiki/Murder_in_the_First_(film)\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/Loverboy_(2005_film)\n",
      "/wiki/Sosie_Bacon\n",
      "/wiki/Upper_West_Side\n",
      "/wiki/Manhattan\n",
      "/wiki/Tracy_Pollan\n",
      "/wiki/Separation_of_church_and_state_in_the_United_States\n",
      "/wiki/The_Times\n",
      "/wiki/Atheism\n",
      "/wiki/Antireligion\n",
      "/wiki/Will.i.am\n",
      "/wiki/It%27s_a_New_Day_(Will.i.am_song)\n",
      "/wiki/Barack_Obama\n",
      "/wiki/Ponzi_scheme\n",
      "/wiki/Bernie_Madoff\n",
      "/wiki/Finding_Your_Roots\n",
      "/wiki/Henry_Louis_Gates\n",
      "/wiki/Apollo_13_(film)\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/Blockbuster_Entertainment_Awards\n",
      "/wiki/Blockbuster_Entertainment_Awards\n",
      "/wiki/Hollow_Man\n",
      "/wiki/Boston_Society_of_Film_Critics\n",
      "/wiki/Boston_Society_of_Film_Critics_Award_for_Best_Cast\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/Bravo_Otto\n",
      "/wiki/Bravo_Otto\n",
      "/wiki/Footloose_(1984_film)\n",
      "/wiki/CableACE_Award\n",
      "/wiki/CableACE_Award\n",
      "/wiki/Losing_Chase\n",
      "/wiki/Chlotrudis_Awards\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/Critics%27_Choice_Movie_Awards\n",
      "/wiki/Critics%27_Choice_Movie_Award_for_Best_Actor\n",
      "/wiki/Murder_in_the_First_(film)\n",
      "/wiki/Ghent_International_Film_Festival\n",
      "/wiki/Ghent_International_Film_Festival\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/Giffoni_Film_Festival\n",
      "/wiki/Giffoni_Film_Festival\n",
      "/wiki/Digging_to_China\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/Golden_Globe_Award\n",
      "/wiki/Golden_Globe_Award_for_Best_Supporting_Actor_%E2%80%93_Motion_Picture\n",
      "/wiki/The_River_Wild\n",
      "/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Miniseries_or_Television_Film\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Television_Series_Musical_or_Comedy\n",
      "/wiki/I_Love_Dick_(TV_series)\n",
      "/wiki/Independent_Spirit_Awards\n",
      "/wiki/Independent_Spirit_Award_for_Best_Male_Lead\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/MTV_Movie_%26_TV_Awards\n",
      "/wiki/MTV_Movie_Award_for_Best_Villain\n",
      "/wiki/Hollow_Man\n",
      "/wiki/Taking_Chance\n",
      "/wiki/The_Following\n",
      "/wiki/E!_People%27s_Choice_Awards\n",
      "/wiki/E!_People%27s_Choice_Awards\n",
      "/wiki/The_Following\n",
      "/wiki/E!_People%27s_Choice_Awards\n",
      "/wiki/The_Following\n",
      "/wiki/Primetime_Emmy_Award\n",
      "/wiki/Primetime_Emmy_Award_for_Outstanding_Lead_Actor_in_a_Limited_Series_or_Movie\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Satellite_Awards\n",
      "/wiki/Satellite_Award_for_Best_Actor_%E2%80%93_Motion_Picture\n",
      "/wiki/The_Woodsman_(2004_film)\n",
      "/wiki/Satellite_Award_for_Best_Actor_%E2%80%93_Miniseries_or_Television_Film\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Saturn_Award\n",
      "/wiki/Saturn_Award_for_Best_Actor_on_Television\n",
      "/wiki/The_Following\n",
      "/wiki/Saturn_Award_for_Best_Actor_on_Television\n",
      "/wiki/The_Following\n",
      "/wiki/Scream_Awards\n",
      "/wiki/Scream_Awards\n",
      "/wiki/Screen_Actors_Guild_Award\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Supporting_Role\n",
      "/wiki/Murder_in_the_First_(film)\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Cast_in_a_Motion_Picture\n",
      "/wiki/Apollo_13_(film)\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Cast_in_a_Motion_Picture\n",
      "/wiki/Mystic_River_(film)\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Cast_in_a_Motion_Picture\n",
      "/wiki/Frost/Nixon_(film)\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie\n",
      "/wiki/Taking_Chance\n",
      "/wiki/Teen_Choice_Awards\n",
      "/wiki/Teen_Choice_Award_for_Choice_Movie_Villain\n",
      "/wiki/Beauty_Shop\n",
      "/wiki/Teen_Choice_Award_for_Choice_Movie_Villain\n",
      "/wiki/TV_Guide_Award\n",
      "/wiki/TV_Guide_Award\n",
      "/wiki/The_Following\n",
      "/wiki/Hollywood_Walk_of_Fame\n",
      "/wiki/Hollywood_Walk_of_Fame\n",
      "/wiki/Denver_Film_Festival\n",
      "/wiki/Phoenix_Film_Festival\n",
      "/wiki/Santa_Barbara_International_Film_Festival\n",
      "/wiki/Broadcast_Film_Critics_Association\n",
      "/wiki/Seattle_International_Film_Festival\n",
      "/wiki/List_of_actors_with_Hollywood_Walk_of_Fame_motion_picture_stars\n",
      "/wiki/The_Hollywood_Reporter\n",
      "/wiki/The_Austin_Chronicle\n",
      "/wiki/Access_Hollywood\n",
      "/wiki/CNN\n",
      "/wiki/IMDb_(identifier)\n",
      "/wiki/Internet_Broadway_Database\n",
      "/wiki/Internet_Off-Broadway_Database\n",
      "/wiki/AllMovie\n",
      "/wiki/Kevin_Bacon_filmography\n",
      "/wiki/Losing_Chase\n",
      "/wiki/Loverboy_(2005_film)\n",
      "/wiki/Kyra_Sedgwick\n",
      "/wiki/Sosie_Bacon\n",
      "/wiki/Edmund_Bacon_(architect)\n",
      "/wiki/Michael_Bacon_(musician)\n",
      "/wiki/The_Bacon_Brothers\n",
      "/wiki/Six_Degrees_of_Kevin_Bacon\n",
      "/wiki/Erd%C5%91s%E2%80%93Bacon_number\n",
      "/wiki/SixDegrees.org\n",
      "/wiki/Critics%27_Choice_Movie_Award_for_Best_Actor\n",
      "/wiki/Geoffrey_Rush\n",
      "/wiki/Jack_Nicholson\n",
      "/wiki/Ian_McKellen\n",
      "/wiki/Russell_Crowe\n",
      "/wiki/Russell_Crowe\n",
      "/wiki/Russell_Crowe\n",
      "/wiki/Daniel_Day-Lewis\n",
      "/wiki/Jack_Nicholson\n",
      "/wiki/Sean_Penn\n",
      "/wiki/Jamie_Foxx\n",
      "/wiki/Philip_Seymour_Hoffman\n",
      "/wiki/Forest_Whitaker\n",
      "/wiki/Daniel_Day-Lewis\n",
      "/wiki/Sean_Penn\n",
      "/wiki/Jeff_Bridges\n",
      "/wiki/Colin_Firth\n",
      "/wiki/George_Clooney\n",
      "/wiki/Daniel_Day-Lewis\n",
      "/wiki/Matthew_McConaughey\n",
      "/wiki/Michael_Keaton\n",
      "/wiki/Leonardo_DiCaprio\n",
      "/wiki/Casey_Affleck\n",
      "/wiki/Gary_Oldman\n",
      "/wiki/Christian_Bale\n",
      "/wiki/Joaquin_Phoenix\n",
      "/wiki/Chadwick_Boseman\n",
      "/wiki/Will_Smith\n",
      "/wiki/Brendan_Fraser\n",
      "/wiki/Golden_Globe_Award_for_Best_Actor_%E2%80%93_Miniseries_or_Television_Film\n",
      "/wiki/Mickey_Rooney\n",
      "/wiki/Anthony_Andrews\n",
      "/wiki/Richard_Chamberlain\n",
      "/wiki/Ted_Danson\n",
      "/wiki/Dustin_Hoffman\n",
      "/wiki/James_Woods\n",
      "/wiki/Randy_Quaid\n",
      "/wiki/Michael_Caine\n",
      "/wiki/Stacy_Keach\n",
      "/wiki/Robert_Duvall\n",
      "/wiki/James_Garner\n",
      "/wiki/Beau_Bridges\n",
      "/wiki/Robert_Duvall\n",
      "/wiki/James_Garner\n",
      "/wiki/Raul_Julia\n",
      "/wiki/Gary_Sinise\n",
      "/wiki/Alan_Rickman\n",
      "/wiki/Ving_Rhames\n",
      "/wiki/Stanley_Tucci\n",
      "/wiki/Jack_Lemmon\n",
      "/wiki/Brian_Dennehy\n",
      "/wiki/James_Franco\n",
      "/wiki/Albert_Finney\n",
      "/wiki/Al_Pacino\n",
      "/wiki/Geoffrey_Rush\n",
      "/wiki/Jonathan_Rhys_Meyers\n",
      "/wiki/Bill_Nighy\n",
      "/wiki/Jim_Broadbent\n",
      "/wiki/Paul_Giamatti\n",
      "/wiki/Al_Pacino\n",
      "/wiki/Idris_Elba\n",
      "/wiki/Kevin_Costner\n",
      "/wiki/Michael_Douglas\n",
      "/wiki/Billy_Bob_Thornton\n",
      "/wiki/Oscar_Isaac\n",
      "/wiki/Tom_Hiddleston\n",
      "/wiki/Ewan_McGregor\n",
      "/wiki/Darren_Criss\n",
      "/wiki/Russell_Crowe\n",
      "/wiki/Mark_Ruffalo\n",
      "/wiki/Michael_Keaton\n",
      "/wiki/Evan_Peters\n",
      "/wiki/Saturn_Award_for_Best_Actor_on_Television\n",
      "/wiki/Saturn_Award_for_Best_Actor_on_Television\n",
      "/wiki/Kyle_Chandler\n",
      "/wiki/Steven_Weber_(actor)\n",
      "/wiki/Richard_Dean_Anderson\n",
      "/wiki/David_Boreanaz\n",
      "/wiki/Robert_Patrick\n",
      "/wiki/Ben_Browder\n",
      "/wiki/David_Boreanaz\n",
      "/wiki/David_Boreanaz\n",
      "/wiki/Ben_Browder\n",
      "/wiki/Matthew_Fox\n",
      "/wiki/Michael_C._Hall\n",
      "/wiki/Matthew_Fox\n",
      "/wiki/Edward_James_Olmos\n",
      "/wiki/Josh_Holloway\n",
      "/wiki/Stephen_Moyer\n",
      "/wiki/Bryan_Cranston\n",
      "/wiki/Bryan_Cranston\n",
      "/wiki/Mads_Mikkelsen\n",
      "/wiki/Hugh_Dancy\n",
      "/wiki/Andrew_Lincoln\n",
      "/wiki/Bruce_Campbell\n",
      "/wiki/Andrew_Lincoln\n",
      "/wiki/Kyle_MacLachlan\n",
      "/wiki/Patrick_Stewart\n",
      "/wiki/Saturn_Award_for_Best_Actor_in_a_Network_or_Cable_Television_Series\n",
      "/wiki/Sam_Heughan\n",
      "/wiki/Bob_Odenkirk\n",
      "/wiki/Saturn_Award_for_Best_Actor_in_a_Streaming_Television_Series\n",
      "/wiki/Henry_Thomas\n",
      "/wiki/Oscar_Isaac\n",
      "/wiki/Screen_Actors_Guild_Award_for_Outstanding_Performance_by_a_Male_Actor_in_a_Miniseries_or_Television_Movie\n",
      "/wiki/Raul_Julia\n",
      "/wiki/Gary_Sinise\n",
      "/wiki/Alan_Rickman\n",
      "/wiki/Gary_Sinise\n",
      "/wiki/Christopher_Reeve\n",
      "/wiki/Jack_Lemmon\n",
      "/wiki/Brian_Dennehy\n",
      "/wiki/Ben_Kingsley\n",
      "/wiki/William_H._Macy\n",
      "/wiki/Al_Pacino\n",
      "/wiki/Geoffrey_Rush\n",
      "/wiki/Paul_Newman\n",
      "/wiki/Jeremy_Irons\n",
      "/wiki/Kevin_Kline\n",
      "/wiki/Paul_Giamatti\n",
      "/wiki/Al_Pacino\n",
      "/wiki/Paul_Giamatti\n",
      "/wiki/Kevin_Costner\n",
      "/wiki/Michael_Douglas\n",
      "/wiki/Mark_Ruffalo\n",
      "/wiki/Idris_Elba\n",
      "/wiki/Bryan_Cranston\n",
      "/wiki/Alexander_Skarsg%C3%A5rd\n",
      "/wiki/Darren_Criss\n",
      "/wiki/Sam_Rockwell\n",
      "/wiki/Mark_Ruffalo\n",
      "/wiki/Michael_Keaton\n",
      "/wiki/Sam_Elliott\n"
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen \n",
    "from bs4 import BeautifulSoup \n",
    "import re\n",
    "\n",
    "html = urlopen('http://en.wikipedia.org/wiki/Kevin_Bacon')\n",
    "bs = BeautifulSoup(html, 'html.parser')\n",
    "for link in bs.find('div', {'id':'bodyContent'}).find_all(\n",
    "    'a', href=re.compile('^(/wiki/)((?!:).)*$')):\n",
    "    print(link.attrs['href'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Walk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/wiki/Julia_R._Masterman_High_School\n",
      "/wiki/Edwin_M._Stanton_School_(Philadelphia)\n",
      "/wiki/Robert_Ralston_School\n",
      "/wiki/Thaddeus_Stevens_School_of_Observation\n",
      "/wiki/Delaplaine_McDaniel_School\n",
      "/wiki/National_Register_of_Historic_Places_listings_in_Greene_County,_Pennsylvania\n",
      "/wiki/National_Register_of_Historic_Places_listings_in_Fayette_County,_Pennsylvania\n",
      "/wiki/National_Register_of_Historic_Places_listings_in_Blair_County,_Pennsylvania\n",
      "/wiki/National_Register_of_Historic_Places_listings_in_Montour_County,_Pennsylvania\n",
      "/wiki/National_Register_of_Historic_Places_listings_in_Pittsburgh,_Pennsylvania\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [4]\u001b[0m, in \u001b[0;36m<cell line: 14>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     15\u001b[0m newArticle \u001b[38;5;241m=\u001b[39m links[random\u001b[38;5;241m.\u001b[39mrandint(\u001b[38;5;241m0\u001b[39m, \u001b[38;5;28mlen\u001b[39m(links)\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)]\u001b[38;5;241m.\u001b[39mattrs[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhref\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m     16\u001b[0m \u001b[38;5;28mprint\u001b[39m(newArticle)\n\u001b[0;32m---> 17\u001b[0m links \u001b[38;5;241m=\u001b[39m \u001b[43mgetLinks\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnewArticle\u001b[49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [4]\u001b[0m, in \u001b[0;36mgetLinks\u001b[0;34m(articleUrl)\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mgetLinks\u001b[39m(articleUrl):\n\u001b[1;32m      9\u001b[0m     html \u001b[38;5;241m=\u001b[39m urlopen(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttp://en.wikipedia.org\u001b[39m\u001b[38;5;132;01m{\u001b[39;00marticleUrl\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m---> 10\u001b[0m     bs \u001b[38;5;241m=\u001b[39m \u001b[43mBeautifulSoup\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhtml\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhtml.parser\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m     11\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m bs\u001b[38;5;241m.\u001b[39mfind(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdiv\u001b[39m\u001b[38;5;124m'\u001b[39m, {\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbodyContent\u001b[39m\u001b[38;5;124m'\u001b[39m})\u001b[38;5;241m.\u001b[39mfind_all(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ma\u001b[39m\u001b[38;5;124m'\u001b[39m, href\u001b[38;5;241m=\u001b[39mre\u001b[38;5;241m.\u001b[39mcompile(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m^(/wiki/)((?!:).)*$\u001b[39m\u001b[38;5;124m'\u001b[39m))\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/bs4/__init__.py:312\u001b[0m, in \u001b[0;36mBeautifulSoup.__init__\u001b[0;34m(self, markup, features, builder, parse_only, from_encoding, exclude_encodings, element_classes, **kwargs)\u001b[0m\n\u001b[1;32m    309\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mparse_only \u001b[38;5;241m=\u001b[39m parse_only\n\u001b[1;32m    311\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(markup, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mread\u001b[39m\u001b[38;5;124m'\u001b[39m):        \u001b[38;5;66;03m# It's a file-type object.\u001b[39;00m\n\u001b[0;32m--> 312\u001b[0m     markup \u001b[38;5;241m=\u001b[39m \u001b[43mmarkup\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    313\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(markup) \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m256\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m (\n\u001b[1;32m    314\u001b[0m         (\u001b[38;5;28misinstance\u001b[39m(markup, \u001b[38;5;28mbytes\u001b[39m) \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m<\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;129;01min\u001b[39;00m markup)\n\u001b[1;32m    315\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m (\u001b[38;5;28misinstance\u001b[39m(markup, \u001b[38;5;28mstr\u001b[39m) \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m<\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;129;01min\u001b[39;00m markup)\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    319\u001b[0m     \u001b[38;5;66;03m# Beautiful Soup will still parse the input as markup,\u001b[39;00m\n\u001b[1;32m    320\u001b[0m     \u001b[38;5;66;03m# since that is sometimes the intended behavior.\u001b[39;00m\n\u001b[1;32m    321\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_markup_is_url(markup):\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:470\u001b[0m, in \u001b[0;36mHTTPResponse.read\u001b[0;34m(self, amt)\u001b[0m\n\u001b[1;32m    465\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    466\u001b[0m     \u001b[38;5;66;03m# Amount is not given (unbounded read) so we must check self.length\u001b[39;00m\n\u001b[1;32m    467\u001b[0m     \u001b[38;5;66;03m# and self.chunked\u001b[39;00m\n\u001b[1;32m    469\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mchunked:\n\u001b[0;32m--> 470\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_readall_chunked\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    472\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlength \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    473\u001b[0m         s \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfp\u001b[38;5;241m.\u001b[39mread()\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:577\u001b[0m, in \u001b[0;36mHTTPResponse._readall_chunked\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    575\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    576\u001b[0m     \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 577\u001b[0m         chunk_left \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_chunk_left\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    578\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m chunk_left \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    579\u001b[0m             \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:560\u001b[0m, in \u001b[0;36mHTTPResponse._get_chunk_left\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    558\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_safe_read(\u001b[38;5;241m2\u001b[39m)  \u001b[38;5;66;03m# toss the CRLF at the end of the chunk\u001b[39;00m\n\u001b[1;32m    559\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 560\u001b[0m     chunk_left \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_next_chunk_size\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    561\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m:\n\u001b[1;32m    562\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m IncompleteRead(\u001b[38;5;124mb\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:520\u001b[0m, in \u001b[0;36mHTTPResponse._read_next_chunk_size\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    518\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_read_next_chunk_size\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    519\u001b[0m     \u001b[38;5;66;03m# Read the next chunk size from the file\u001b[39;00m\n\u001b[0;32m--> 520\u001b[0m     line \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreadline\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_MAXLINE\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m    521\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(line) \u001b[38;5;241m>\u001b[39m _MAXLINE:\n\u001b[1;32m    522\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m LineTooLong(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mchunk size\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/socket.py:704\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    702\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m    703\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 704\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    705\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n\u001b[1;32m    706\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_timeout_occurred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/ssl.py:1241\u001b[0m, in \u001b[0;36mSSLSocket.recv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m   1237\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m flags \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m   1238\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   1239\u001b[0m           \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnon-zero flags not allowed in calls to recv_into() on \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m\n\u001b[1;32m   1240\u001b[0m           \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m)\n\u001b[0;32m-> 1241\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnbytes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1242\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1243\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mrecv_into(buffer, nbytes, flags)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/ssl.py:1099\u001b[0m, in \u001b[0;36mSSLSocket.read\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m   1097\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1098\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m buffer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1099\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1100\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1101\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sslobj\u001b[38;5;241m.\u001b[39mread(\u001b[38;5;28mlen\u001b[39m)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import datetime\n",
    "import random\n",
    "import re\n",
    "\n",
    "random.seed(datetime.datetime.now().strftime('%s'))\n",
    "def getLinks(articleUrl):\n",
    "    html = urlopen(f'http://en.wikipedia.org{articleUrl}')\n",
    "    bs = BeautifulSoup(html, 'html.parser')\n",
    "    return bs.find('div', {'id':'bodyContent'}).find_all('a', href=re.compile('^(/wiki/)((?!:).)*$'))\n",
    "\n",
    "links = getLinks('/wiki/Kevin_Bacon')\n",
    "while len(links) > 0:\n",
    "    newArticle = links[random.randint(0, len(links)-1)].attrs['href']\n",
    "    print(newArticle)\n",
    "    links = getLinks(newArticle)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recursively crawling an entire site"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/wiki/Main_Page\n",
      "/wiki/Special:Search\n",
      "/wiki/Help:Introduction\n",
      "/wiki/Special:MyTalk\n",
      "/wiki/Special:MyContributions\n",
      "/wiki/Wikipedia:Contents\n",
      "/wiki/Portal:Current_events\n",
      "/wiki/Special:Random\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [7]\u001b[0m, in \u001b[0;36m<cell line: 18>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m                 pages\u001b[38;5;241m.\u001b[39madd(newPage)\n\u001b[1;32m     17\u001b[0m                 getLinks(newPage)\n\u001b[0;32m---> 18\u001b[0m \u001b[43mgetLinks\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [7]\u001b[0m, in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[38;5;28mprint\u001b[39m(newPage)\n\u001b[1;32m     16\u001b[0m pages\u001b[38;5;241m.\u001b[39madd(newPage)\n\u001b[0;32m---> 17\u001b[0m \u001b[43mgetLinks\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnewPage\u001b[49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [7]\u001b[0m, in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[38;5;28mprint\u001b[39m(newPage)\n\u001b[1;32m     16\u001b[0m pages\u001b[38;5;241m.\u001b[39madd(newPage)\n\u001b[0;32m---> 17\u001b[0m \u001b[43mgetLinks\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnewPage\u001b[49m\u001b[43m)\u001b[49m\n",
      "    \u001b[0;31m[... skipping similar frames: getLinks at line 17 (5 times)]\u001b[0m\n",
      "Input \u001b[0;32mIn [7]\u001b[0m, in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[38;5;28mprint\u001b[39m(newPage)\n\u001b[1;32m     16\u001b[0m pages\u001b[38;5;241m.\u001b[39madd(newPage)\n\u001b[0;32m---> 17\u001b[0m \u001b[43mgetLinks\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnewPage\u001b[49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [7]\u001b[0m, in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mgetLinks\u001b[39m(pageUrl):\n\u001b[1;32m      7\u001b[0m     \u001b[38;5;28;01mglobal\u001b[39;00m pages\n\u001b[0;32m----> 8\u001b[0m     html \u001b[38;5;241m=\u001b[39m \u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhttp://en.wikipedia.org\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mpageUrl\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      9\u001b[0m     bs \u001b[38;5;241m=\u001b[39m BeautifulSoup(html, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhtml.parser\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     10\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m link \u001b[38;5;129;01min\u001b[39;00m bs\u001b[38;5;241m.\u001b[39mfind_all(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ma\u001b[39m\u001b[38;5;124m'\u001b[39m, href\u001b[38;5;241m=\u001b[39mre\u001b[38;5;241m.\u001b[39mcompile(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m^(/wiki/)\u001b[39m\u001b[38;5;124m'\u001b[39m)):\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:214\u001b[0m, in \u001b[0;36murlopen\u001b[0;34m(url, data, timeout, cafile, capath, cadefault, context)\u001b[0m\n\u001b[1;32m    212\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    213\u001b[0m     opener \u001b[38;5;241m=\u001b[39m _opener\n\u001b[0;32m--> 214\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mopener\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:523\u001b[0m, in \u001b[0;36mOpenerDirector.open\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    521\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m processor \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocess_response\u001b[38;5;241m.\u001b[39mget(protocol, []):\n\u001b[1;32m    522\u001b[0m     meth \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(processor, meth_name)\n\u001b[0;32m--> 523\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[43mmeth\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    525\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:632\u001b[0m, in \u001b[0;36mHTTPErrorProcessor.http_response\u001b[0;34m(self, request, response)\u001b[0m\n\u001b[1;32m    629\u001b[0m \u001b[38;5;66;03m# According to RFC 2616, \"2xx\" code indicates that the client's\u001b[39;00m\n\u001b[1;32m    630\u001b[0m \u001b[38;5;66;03m# request was successfully received, understood, and accepted.\u001b[39;00m\n\u001b[1;32m    631\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;241m200\u001b[39m \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m code \u001b[38;5;241m<\u001b[39m \u001b[38;5;241m300\u001b[39m):\n\u001b[0;32m--> 632\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43merror\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    633\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhttp\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhdrs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    635\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:555\u001b[0m, in \u001b[0;36mOpenerDirector.error\u001b[0;34m(self, proto, *args)\u001b[0m\n\u001b[1;32m    553\u001b[0m     http_err \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m    554\u001b[0m args \u001b[38;5;241m=\u001b[39m (\u001b[38;5;28mdict\u001b[39m, proto, meth_name) \u001b[38;5;241m+\u001b[39m args\n\u001b[0;32m--> 555\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_chain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    556\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m result:\n\u001b[1;32m    557\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:494\u001b[0m, in \u001b[0;36mOpenerDirector._call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    492\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m handler \u001b[38;5;129;01min\u001b[39;00m handlers:\n\u001b[1;32m    493\u001b[0m     func \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(handler, meth_name)\n\u001b[0;32m--> 494\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    495\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m result \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    496\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:747\u001b[0m, in \u001b[0;36mHTTPRedirectHandler.http_error_302\u001b[0;34m(self, req, fp, code, msg, headers)\u001b[0m\n\u001b[1;32m    744\u001b[0m fp\u001b[38;5;241m.\u001b[39mread()\n\u001b[1;32m    745\u001b[0m fp\u001b[38;5;241m.\u001b[39mclose()\n\u001b[0;32m--> 747\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:517\u001b[0m, in \u001b[0;36mOpenerDirector.open\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    514\u001b[0m     req \u001b[38;5;241m=\u001b[39m meth(req)\n\u001b[1;32m    516\u001b[0m sys\u001b[38;5;241m.\u001b[39maudit(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124murllib.Request\u001b[39m\u001b[38;5;124m'\u001b[39m, req\u001b[38;5;241m.\u001b[39mfull_url, req\u001b[38;5;241m.\u001b[39mdata, req\u001b[38;5;241m.\u001b[39mheaders, req\u001b[38;5;241m.\u001b[39mget_method())\n\u001b[0;32m--> 517\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    519\u001b[0m \u001b[38;5;66;03m# post-process response\u001b[39;00m\n\u001b[1;32m    520\u001b[0m meth_name \u001b[38;5;241m=\u001b[39m protocol\u001b[38;5;241m+\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_response\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:534\u001b[0m, in \u001b[0;36mOpenerDirector._open\u001b[0;34m(self, req, data)\u001b[0m\n\u001b[1;32m    531\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m result\n\u001b[1;32m    533\u001b[0m protocol \u001b[38;5;241m=\u001b[39m req\u001b[38;5;241m.\u001b[39mtype\n\u001b[0;32m--> 534\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_chain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_open\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprotocol\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprotocol\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\n\u001b[1;32m    535\u001b[0m \u001b[43m                          \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m_open\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    536\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m result:\n\u001b[1;32m    537\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:494\u001b[0m, in \u001b[0;36mOpenerDirector._call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    492\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m handler \u001b[38;5;129;01min\u001b[39;00m handlers:\n\u001b[1;32m    493\u001b[0m     func \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(handler, meth_name)\n\u001b[0;32m--> 494\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    495\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m result \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    496\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:1389\u001b[0m, in \u001b[0;36mHTTPSHandler.https_open\u001b[0;34m(self, req)\u001b[0m\n\u001b[1;32m   1388\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mhttps_open\u001b[39m(\u001b[38;5;28mself\u001b[39m, req):\n\u001b[0;32m-> 1389\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhttp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mHTTPSConnection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1390\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_context\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcheck_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_check_hostname\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:1350\u001b[0m, in \u001b[0;36mAbstractHTTPHandler.do_open\u001b[0;34m(self, http_class, req, **http_conn_args)\u001b[0m\n\u001b[1;32m   1348\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err: \u001b[38;5;66;03m# timeout error\u001b[39;00m\n\u001b[1;32m   1349\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m URLError(err)\n\u001b[0;32m-> 1350\u001b[0m     r \u001b[38;5;241m=\u001b[39m \u001b[43mh\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetresponse\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1351\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n\u001b[1;32m   1352\u001b[0m     h\u001b[38;5;241m.\u001b[39mclose()\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:1377\u001b[0m, in \u001b[0;36mHTTPConnection.getresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1375\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1376\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1377\u001b[0m         \u001b[43mresponse\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbegin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1378\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m:\n\u001b[1;32m   1379\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclose()\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:320\u001b[0m, in \u001b[0;36mHTTPResponse.begin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    318\u001b[0m \u001b[38;5;66;03m# read until we get a non-100 response\u001b[39;00m\n\u001b[1;32m    319\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 320\u001b[0m     version, status, reason \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    321\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m status \u001b[38;5;241m!=\u001b[39m CONTINUE:\n\u001b[1;32m    322\u001b[0m         \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:281\u001b[0m, in \u001b[0;36mHTTPResponse._read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    280\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_read_status\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 281\u001b[0m     line \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreadline\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_MAXLINE\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124miso-8859-1\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    282\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(line) \u001b[38;5;241m>\u001b[39m _MAXLINE:\n\u001b[1;32m    283\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m LineTooLong(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstatus line\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/socket.py:704\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    702\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m    703\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 704\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    705\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n\u001b[1;32m    706\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_timeout_occurred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/ssl.py:1241\u001b[0m, in \u001b[0;36mSSLSocket.recv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m   1237\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m flags \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m   1238\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   1239\u001b[0m           \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnon-zero flags not allowed in calls to recv_into() on \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m\n\u001b[1;32m   1240\u001b[0m           \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m)\n\u001b[0;32m-> 1241\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnbytes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1242\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1243\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mrecv_into(buffer, nbytes, flags)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/ssl.py:1099\u001b[0m, in \u001b[0;36mSSLSocket.read\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m   1097\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1098\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m buffer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1099\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1100\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1101\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sslobj\u001b[38;5;241m.\u001b[39mread(\u001b[38;5;28mlen\u001b[39m)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "\n",
    "pages = set()\n",
    "def getLinks(pageUrl):\n",
    "    html = urlopen(f'http://en.wikipedia.org{pageUrl}')\n",
    "    bs = BeautifulSoup(html, 'html.parser')\n",
    "    for link in bs.find_all('a', href=re.compile('^(/wiki/)')):\n",
    "        if 'href' in link.attrs:\n",
    "            if link.attrs['href'] not in pages:\n",
    "                #We have encountered a new page\n",
    "                newPage = link.attrs['href']\n",
    "                print(newPage)\n",
    "                pages.add(newPage)\n",
    "                getLinks(newPage)\n",
    "getLinks('')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collecting Data Across an Entire Site"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "General-purpose programming language\n",
      "<p>In <a class=\"mw-redirect\" href=\"/wiki/Computer_software\" title=\"Computer software\">computer software</a>, a <b>general-purpose programming language (GPL)</b> is a <a href=\"/wiki/Programming_language\" title=\"Programming language\">programming language</a> for building <a href=\"/wiki/Software\" title=\"Software\">software</a> in a wide variety of application <a href=\"/wiki/Domain_(software_engineering)\" title=\"Domain (software engineering)\">domains</a>. Conversely, a <a class=\"mw-redirect\" href=\"/wiki/Domain-specific_programming_language\" title=\"Domain-specific programming language\">domain-specific programming language</a> is used within a specific area. For example, <a href=\"/wiki/SQL\" title=\"SQL\">SQL</a> was specifically designed for <a href=\"/wiki/Query_language\" title=\"Query language\">querying relational databases</a>.\n",
      "</p>\n",
      "/w/index.php?title=General-purpose_programming_language&action=edit\n",
      "--------------------\n",
      "/wiki/Main_Page\n",
      "Main Page\n",
      "<p>\"<b><a href=\"/wiki/I_Need_You_(Paris_Hilton_song)\" title=\"I Need You (Paris Hilton song)\">I Need You</a></b>\" is a song recorded by American socialite <a href=\"/wiki/Paris_Hilton\" title=\"Paris Hilton\">Paris Hilton</a> <i>(pictured)</i>. Michael Green produced the song and co-wrote it with Hilton and <a href=\"/wiki/Simon_Wilcox\" title=\"Simon Wilcox\">Simon Wilcox</a>. Heiress Records released it as a <a href=\"/wiki/Music_download\" title=\"Music download\">digital download</a> on February 14, 2018. A version of the song <a href=\"/wiki/Music_leak\" title=\"Music leak\">leaked</a> in 2010, but Hilton said she re-recorded portions of the vocals and instrumental as a dedication to her then-fiancé <a href=\"/wiki/Chris_Zylka\" title=\"Chris Zylka\">Chris Zylka</a>. It is a <a href=\"/wiki/Doo-wop\" title=\"Doo-wop\">doo-wop</a> and <a href=\"/wiki/Pop_music\" title=\"Pop music\">pop</a> ballad with holiday-related <a href=\"/wiki/Pun\" title=\"Pun\">puns</a> about love. Green based the <a href=\"/wiki/Sentimental_ballad\" title=\"Sentimental ballad\">ballad</a>'s style on 1950s music. Critics praised \"I Need You\", partially as a departure from Hilton's <a href=\"/wiki/Dance_music\" title=\"Dance music\">dance</a> and <a href=\"/wiki/Electronic_dance_music\" title=\"Electronic dance music\">EDM</a> songs. The single peaked at number 32 on the <a href=\"/wiki/Dance_Club_Songs\" title=\"Dance Club Songs\">Dance Club Songs</a> chart, Hilton's fifth appearance on it. A music video, directed by Zylka, features Hilton wearing lingerie, posing in a bed covered with red rose petals, and <a href=\"/wiki/Pop_out_cake\" title=\"Pop out cake\">popping out of a cake</a>. It was inspired by <a href=\"/wiki/Marilyn_Monroe\" title=\"Marilyn Monroe\">Marilyn Monroe</a>'s performance of \"<a href=\"/wiki/I_Wanna_Be_Loved_by_You\" title=\"I Wanna Be Loved by You\">I Wanna Be Loved by You</a>\" and <a href=\"/wiki/Jessica_Rabbit\" title=\"Jessica Rabbit\">Jessica Rabbit</a>. Critics described the bed scene as an homage to the 1999 film <i><a href=\"/wiki/American_Beauty_(1999_film)\" title=\"American Beauty (1999 film)\">American Beauty</a></i>. (<b><a href=\"/wiki/I_Need_You_(Paris_Hilton_song)\" title=\"I Need You (Paris Hilton song)\">Full article...</a></b>)\n",
      "</p>\n",
      "This page is missing something! Continuing.\n",
      "--------------------\n",
      "/wiki/Special:Search\n",
      "Search\n",
      "This page is missing something! Continuing.\n",
      "--------------------\n",
      "/wiki/Help:Introduction\n",
      "Help:Introduction\n",
      "<p><style data-mw-deduplicate=\"TemplateStyles:r1053965116\">.mw-parser-output .introtosingle__main{position:relative;box-sizing:border-box;box-shadow:2px 2px 2px #CCC;max-width:100%;overflow:hidden;border:1px solid black;margin:auto;padding-bottom:20px}.mw-parser-output .introtosingle__main p{margin-bottom:2.0em}.mw-parser-output .introtosingle__main-withbackground{background-image:url(\"https://upload.wikimedia.org/wikipedia/commons/9/94/Wikipedia-logo-v2_%2810_percent_opacity%29.svg\");background-position:center -500px;background-repeat:no-repeat;background-size:auto 150%}.mw-parser-output .introtosingle__main-title{font-size:250%;line-height:150%;background:#777;color:#FFF;text-align:center;align-items:center;justify-content:center}.mw-parser-output .introtosingle__lead{background-color:#EEE;background-color:rgba(221,221,221,0.5);padding:30px 60px;margin-bottom:15px}.mw-parser-output .introtosingle__base{box-sizing:border-box;max-width:1100px;min-height:55px;margin:auto;padding:5px 20px;font-size:1.1em;background:#EEE;border:1px solid lightgrey}.mw-parser-output .introtosingle__columns{display:flex;flex-direction:row;flex-wrap:wrap-reverse;justify-content:center}.mw-parser-output .introtosingle__columns-left,.mw-parser-output .introtosingle__columns-left-noborder,.mw-parser-output .introtosingle__columns-right{display:inline-block;flex:1 1 0;align-self:flex-end;vertical-align:top;min-width:200px;max-width:300px;padding:10px}.mw-parser-output .introtosingle__columns-left{text-align:right;justify-content:right;border-right:solid 1px #ddd}.mw-parser-output .introtosingle__columns-left-noborder{text-align:right;justify-content:right}.mw-parser-output .introtosingle__columns-right{text-align:left;justify-content:left}@media screen and (min-width:1101px){.mw-parser-output .introtosingle__main{max-width:1100px}}</style>\n",
      "</p>\n",
      "This page is missing something! Continuing.\n",
      "--------------------\n",
      "/wiki/Special:MyTalk\n",
      "User talk:72.93.95.254\n",
      "<p>People on Wikipedia can use this <a href=\"/wiki/Wikipedia:User_pages\" title=\"Wikipedia:User pages\">talk page</a> to post a public message about edits made from the IP address you are currently using.\n",
      "</p>\n",
      "/w/index.php?title=User_talk:72.93.95.254&action=edit\n",
      "--------------------\n",
      "/wiki/Special:MyContributions\n",
      "User contributions for 72.93.95.254\n",
      "This page is missing something! Continuing.\n",
      "--------------------\n",
      "/wiki/Wikipedia:Contents\n",
      "Wikipedia:Contents\n",
      "<p class=\"mw-empty-elt\">\n",
      "</p>\n",
      "This page is missing something! Continuing.\n",
      "--------------------\n",
      "/wiki/Portal:Current_events\n",
      "Portal:Current events\n",
      "<p><span class=\"noprint\" id=\"coordinates\"><a href=\"/wiki/Portal:Current_events/Edit_instructions\" title=\"Portal:Current events/Edit instructions\">Edit instructions</a></span>\n",
      "</p>\n",
      "This page is missing something! Continuing.\n",
      "--------------------\n",
      "/wiki/Special:Random\n",
      "1995 Missouri Valley Conference men's basketball tournament\n",
      "<p>The <b>1995 <a href=\"/wiki/Missouri_Valley_Conference_men%27s_basketball_tournament\" title=\"Missouri Valley Conference men's basketball tournament\">Missouri Valley Conference men's basketball tournament</a></b> was played after the conclusion of the 1994–1995 regular season at the <a class=\"mw-redirect\" href=\"/wiki/Kiel_Center\" title=\"Kiel Center\">Kiel Center</a> in <a class=\"mw-redirect\" href=\"/wiki/St._Louis,_Missouri\" title=\"St. Louis, Missouri\">St. Louis, Missouri</a>.\n",
      "</p>\n",
      "/w/index.php?title=1995_Missouri_Valley_Conference_men%27s_basketball_tournament&action=edit\n",
      "--------------------\n",
      "/wiki/Wikipedia:About\n",
      "Wikipedia:About\n",
      "<p class=\"mw-empty-elt\">\n",
      "</p>\n",
      "This page is missing something! Continuing.\n",
      "--------------------\n",
      "/wiki/Help:Contents\n",
      "Help:Contents\n",
      "<p><br/>\n",
      "<style data-mw-deduplicate=\"TemplateStyles:r1012508797\">.mw-parser-output .helpContents-wrapper{display:flex;flex-wrap:wrap}.mw-parser-output .helpContents-header{flex:1 0 100%;padding-bottom:1em;border-bottom:1px solid #a2a9b1;text-align:center}.mw-parser-output .helpContents-section{position:relative;flex:1 0 50%;min-width:380px;box-sizing:border-box;margin:1em 0;padding-left:30px;padding-right:20px}.mw-parser-output .helpContents-section h2{font-size:21px;padding:0;margin-top:1em;margin-bottom:0.25em;line-height:1.5}.mw-parser-output .helpContents-section ul{margin-left:0;padding-left:0}.mw-parser-output .helpContents-icon{position:absolute;top:25px;left:0;line-height:1;opacity:0.7}.mw-parser-output .helpContents-additional-search{text-align:center;margin-bottom:1em}</style>\n",
      "</p>\n",
      "This page is missing something! Continuing.\n",
      "--------------------\n",
      "/wiki/Wikipedia:Community_portal\n",
      "Wikipedia:Community portal\n",
      "<p class=\"mw-empty-elt\">\n",
      "</p>\n",
      "This page is missing something! Continuing.\n",
      "--------------------\n",
      "/wiki/Special:RecentChanges\n",
      "Recent changes\n",
      "<p>This is a list of recent changes to Wikipedia.\n",
      "</p>\n",
      "This page is missing something! Continuing.\n",
      "--------------------\n",
      "/wiki/Wikipedia:File_upload_wizard\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wikipedia:File upload wizard\n",
      "<p>Thank you for offering to contribute an image or other media file for use on Wikipedia. This wizard will guide you through a questionnaire prompting you for the appropriate copyright and sourcing information for each file. Please ensure you understand <a href=\"/wiki/Wikipedia:Copyrights\" title=\"Wikipedia:Copyrights\">copyright</a> and the <a href=\"/wiki/Wikipedia:Image_use_policy\" title=\"Wikipedia:Image use policy\">image use policy</a> before proceeding.\n",
      "</p>\n",
      "This page is missing something! Continuing.\n",
      "--------------------\n",
      "/wiki/Special:WhatLinksHere/Wikipedia:File_upload_wizard\n",
      "Pages that link to \"Wikipedia:File upload wizard\"\n",
      "<p>The following pages link to <b id=\"specialDeleteTarget\"><a href=\"/wiki/Wikipedia:File_upload_wizard\" title=\"Wikipedia:File upload wizard\">Wikipedia:File upload wizard</a></b> <span id=\"specialDeleteLink\"></span>\n",
      "</p>\n",
      "This page is missing something! Continuing.\n",
      "--------------------\n",
      "/wiki/Wikipedia:File_Upload_Wizard\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [13]\u001b[0m, in \u001b[0;36m<cell line: 29>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     27\u001b[0m                 pages\u001b[38;5;241m.\u001b[39madd(newPage)\n\u001b[1;32m     28\u001b[0m                 getLinks(newPage)\n\u001b[0;32m---> 29\u001b[0m \u001b[43mgetLinks\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m/wiki/General-purpose_programming_language\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [13]\u001b[0m, in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[38;5;28mprint\u001b[39m(newPage)\n\u001b[1;32m     27\u001b[0m pages\u001b[38;5;241m.\u001b[39madd(newPage)\n\u001b[0;32m---> 28\u001b[0m \u001b[43mgetLinks\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnewPage\u001b[49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [13]\u001b[0m, in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[38;5;28mprint\u001b[39m(newPage)\n\u001b[1;32m     27\u001b[0m pages\u001b[38;5;241m.\u001b[39madd(newPage)\n\u001b[0;32m---> 28\u001b[0m \u001b[43mgetLinks\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnewPage\u001b[49m\u001b[43m)\u001b[49m\n",
      "    \u001b[0;31m[... skipping similar frames: getLinks at line 28 (12 times)]\u001b[0m\n",
      "Input \u001b[0;32mIn [13]\u001b[0m, in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[38;5;28mprint\u001b[39m(newPage)\n\u001b[1;32m     27\u001b[0m pages\u001b[38;5;241m.\u001b[39madd(newPage)\n\u001b[0;32m---> 28\u001b[0m \u001b[43mgetLinks\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnewPage\u001b[49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [13]\u001b[0m, in \u001b[0;36mgetLinks\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mgetLinks\u001b[39m(pageUrl):\n\u001b[1;32m      7\u001b[0m     \u001b[38;5;28;01mglobal\u001b[39;00m pages\n\u001b[0;32m----> 8\u001b[0m     html \u001b[38;5;241m=\u001b[39m \u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhttp://en.wikipedia.org\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mpageUrl\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      9\u001b[0m     bs \u001b[38;5;241m=\u001b[39m BeautifulSoup(html, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhtml.parser\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     10\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:214\u001b[0m, in \u001b[0;36murlopen\u001b[0;34m(url, data, timeout, cafile, capath, cadefault, context)\u001b[0m\n\u001b[1;32m    212\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    213\u001b[0m     opener \u001b[38;5;241m=\u001b[39m _opener\n\u001b[0;32m--> 214\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mopener\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:523\u001b[0m, in \u001b[0;36mOpenerDirector.open\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    521\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m processor \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocess_response\u001b[38;5;241m.\u001b[39mget(protocol, []):\n\u001b[1;32m    522\u001b[0m     meth \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(processor, meth_name)\n\u001b[0;32m--> 523\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[43mmeth\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    525\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:632\u001b[0m, in \u001b[0;36mHTTPErrorProcessor.http_response\u001b[0;34m(self, request, response)\u001b[0m\n\u001b[1;32m    629\u001b[0m \u001b[38;5;66;03m# According to RFC 2616, \"2xx\" code indicates that the client's\u001b[39;00m\n\u001b[1;32m    630\u001b[0m \u001b[38;5;66;03m# request was successfully received, understood, and accepted.\u001b[39;00m\n\u001b[1;32m    631\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;241m200\u001b[39m \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m code \u001b[38;5;241m<\u001b[39m \u001b[38;5;241m300\u001b[39m):\n\u001b[0;32m--> 632\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43merror\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    633\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhttp\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhdrs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    635\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:555\u001b[0m, in \u001b[0;36mOpenerDirector.error\u001b[0;34m(self, proto, *args)\u001b[0m\n\u001b[1;32m    553\u001b[0m     http_err \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m    554\u001b[0m args \u001b[38;5;241m=\u001b[39m (\u001b[38;5;28mdict\u001b[39m, proto, meth_name) \u001b[38;5;241m+\u001b[39m args\n\u001b[0;32m--> 555\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_chain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    556\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m result:\n\u001b[1;32m    557\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:494\u001b[0m, in \u001b[0;36mOpenerDirector._call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    492\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m handler \u001b[38;5;129;01min\u001b[39;00m handlers:\n\u001b[1;32m    493\u001b[0m     func \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(handler, meth_name)\n\u001b[0;32m--> 494\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    495\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m result \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    496\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:747\u001b[0m, in \u001b[0;36mHTTPRedirectHandler.http_error_302\u001b[0;34m(self, req, fp, code, msg, headers)\u001b[0m\n\u001b[1;32m    744\u001b[0m fp\u001b[38;5;241m.\u001b[39mread()\n\u001b[1;32m    745\u001b[0m fp\u001b[38;5;241m.\u001b[39mclose()\n\u001b[0;32m--> 747\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:517\u001b[0m, in \u001b[0;36mOpenerDirector.open\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    514\u001b[0m     req \u001b[38;5;241m=\u001b[39m meth(req)\n\u001b[1;32m    516\u001b[0m sys\u001b[38;5;241m.\u001b[39maudit(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124murllib.Request\u001b[39m\u001b[38;5;124m'\u001b[39m, req\u001b[38;5;241m.\u001b[39mfull_url, req\u001b[38;5;241m.\u001b[39mdata, req\u001b[38;5;241m.\u001b[39mheaders, req\u001b[38;5;241m.\u001b[39mget_method())\n\u001b[0;32m--> 517\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    519\u001b[0m \u001b[38;5;66;03m# post-process response\u001b[39;00m\n\u001b[1;32m    520\u001b[0m meth_name \u001b[38;5;241m=\u001b[39m protocol\u001b[38;5;241m+\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_response\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:534\u001b[0m, in \u001b[0;36mOpenerDirector._open\u001b[0;34m(self, req, data)\u001b[0m\n\u001b[1;32m    531\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m result\n\u001b[1;32m    533\u001b[0m protocol \u001b[38;5;241m=\u001b[39m req\u001b[38;5;241m.\u001b[39mtype\n\u001b[0;32m--> 534\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_chain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_open\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprotocol\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprotocol\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\n\u001b[1;32m    535\u001b[0m \u001b[43m                          \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m_open\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    536\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m result:\n\u001b[1;32m    537\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:494\u001b[0m, in \u001b[0;36mOpenerDirector._call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    492\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m handler \u001b[38;5;129;01min\u001b[39;00m handlers:\n\u001b[1;32m    493\u001b[0m     func \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(handler, meth_name)\n\u001b[0;32m--> 494\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    495\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m result \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    496\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:1389\u001b[0m, in \u001b[0;36mHTTPSHandler.https_open\u001b[0;34m(self, req)\u001b[0m\n\u001b[1;32m   1388\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mhttps_open\u001b[39m(\u001b[38;5;28mself\u001b[39m, req):\n\u001b[0;32m-> 1389\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhttp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mHTTPSConnection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1390\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_context\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcheck_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_check_hostname\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:1346\u001b[0m, in \u001b[0;36mAbstractHTTPHandler.do_open\u001b[0;34m(self, http_class, req, **http_conn_args)\u001b[0m\n\u001b[1;32m   1344\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1345\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1346\u001b[0m         \u001b[43mh\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_method\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mselector\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1347\u001b[0m \u001b[43m                  \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhas_header\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mTransfer-encoding\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1348\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err: \u001b[38;5;66;03m# timeout error\u001b[39;00m\n\u001b[1;32m   1349\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m URLError(err)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:1285\u001b[0m, in \u001b[0;36mHTTPConnection.request\u001b[0;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[1;32m   1282\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrequest\u001b[39m(\u001b[38;5;28mself\u001b[39m, method, url, body\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, headers\u001b[38;5;241m=\u001b[39m{}, \u001b[38;5;241m*\u001b[39m,\n\u001b[1;32m   1283\u001b[0m             encode_chunked\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[1;32m   1284\u001b[0m     \u001b[38;5;124;03m\"\"\"Send a complete request to the server.\"\"\"\u001b[39;00m\n\u001b[0;32m-> 1285\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:1331\u001b[0m, in \u001b[0;36mHTTPConnection._send_request\u001b[0;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[1;32m   1327\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(body, \u001b[38;5;28mstr\u001b[39m):\n\u001b[1;32m   1328\u001b[0m     \u001b[38;5;66;03m# RFC 2616 Section 3.7.1 says that text default has a\u001b[39;00m\n\u001b[1;32m   1329\u001b[0m     \u001b[38;5;66;03m# default charset of iso-8859-1.\u001b[39;00m\n\u001b[1;32m   1330\u001b[0m     body \u001b[38;5;241m=\u001b[39m _encode(body, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbody\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m-> 1331\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mendheaders\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:1280\u001b[0m, in \u001b[0;36mHTTPConnection.endheaders\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m   1278\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1279\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m CannotSendHeader()\n\u001b[0;32m-> 1280\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_output\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:1040\u001b[0m, in \u001b[0;36mHTTPConnection._send_output\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m   1038\u001b[0m msg \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\r\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_buffer)\n\u001b[1;32m   1039\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_buffer[:]\n\u001b[0;32m-> 1040\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1042\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m message_body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   1043\u001b[0m \n\u001b[1;32m   1044\u001b[0m     \u001b[38;5;66;03m# create a consistent interface to message_body\u001b[39;00m\n\u001b[1;32m   1045\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(message_body, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mread\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[1;32m   1046\u001b[0m         \u001b[38;5;66;03m# Let file-like take precedence over byte-like.  This\u001b[39;00m\n\u001b[1;32m   1047\u001b[0m         \u001b[38;5;66;03m# is needed to allow the current position of mmap'ed\u001b[39;00m\n\u001b[1;32m   1048\u001b[0m         \u001b[38;5;66;03m# files to be taken into account.\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:980\u001b[0m, in \u001b[0;36mHTTPConnection.send\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m    978\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msock \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    979\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mauto_open:\n\u001b[0;32m--> 980\u001b[0m         \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    981\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    982\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m NotConnected()\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:1447\u001b[0m, in \u001b[0;36mHTTPSConnection.connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1444\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m   1445\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConnect to a host on a given (SSL) port.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m-> 1447\u001b[0m     \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1449\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_tunnel_host:\n\u001b[1;32m   1450\u001b[0m         server_hostname \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_tunnel_host\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:946\u001b[0m, in \u001b[0;36mHTTPConnection.connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    944\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    945\u001b[0m     \u001b[38;5;124;03m\"\"\"Connect to the host and port specified in __init__.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 946\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msock \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_create_connection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    947\u001b[0m \u001b[43m        \u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhost\u001b[49m\u001b[43m,\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mport\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msource_address\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    948\u001b[0m     \u001b[38;5;66;03m# Might fail in OSs that don't implement TCP_NODELAY\u001b[39;00m\n\u001b[1;32m    949\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/socket.py:832\u001b[0m, in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address)\u001b[0m\n\u001b[1;32m    830\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m source_address:\n\u001b[1;32m    831\u001b[0m     sock\u001b[38;5;241m.\u001b[39mbind(source_address)\n\u001b[0;32m--> 832\u001b[0m \u001b[43msock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43msa\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    833\u001b[0m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n\u001b[1;32m    834\u001b[0m err \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "\n",
    "pages = set()\n",
    "def getLinks(pageUrl):\n",
    "    html = urlopen(f'http://en.wikipedia.org{pageUrl}')\n",
    "    bs = BeautifulSoup(html, 'html.parser')\n",
    "    try:\n",
    "        print(bs.h1.get_text())\n",
    "        #mw-parser-output\n",
    "        bodyContent = bs.find('div', {'id':'bodyContent'}).find_all('p')\n",
    "        if len(bodyContent):\n",
    "            print(bodyContent[0])\n",
    "        print(bs.find(id='ca-edit').find('a').attrs['href'])\n",
    "    except AttributeError:\n",
    "        print('This page is missing something! Continuing.')\n",
    "    \n",
    "    for link in bs.find_all('a', href=re.compile('^(/wiki/)')):\n",
    "        if 'href' in link.attrs:\n",
    "            if link.attrs['href'] not in pages:\n",
    "                #We have encountered a new page\n",
    "                newPage = link.attrs['href']\n",
    "                print('-'*20)\n",
    "                print(newPage)\n",
    "                pages.add(newPage)\n",
    "                getLinks(newPage)\n",
    "getLinks('/wiki/General-purpose_programming_language') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Crawling across the Internet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random external link is: https://learning.oreilly.com/search/?query=author%3A%22Kelsey%20Hightower%22&extended_publisher_data=true&highlight=true&include_assessments=false&include_case_studies=true&include_courses=true&include_playlists=true&include_collections=true&include_notebooks=true&include_sandboxes=true&include_scenarios=true&is_academic_institution_account=false&source=user&sort=date_added&facet_json=true&json_facets=true&page=0&include_facets=false\n",
      "Random external link is: https://play.google.com/store/apps/details?id=com.safariflow.queue\n",
      "Random external link is: https://maps.google.com/?q=1005%20Gravenstein%20Highway%20North%0ASebastopol,%20CA%2095472%0AUSA\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [37]\u001b[0m, in \u001b[0;36m<cell line: 51>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     47\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mRandom external link is: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mexternalLink\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     48\u001b[0m     followExternalOnly(externalLink)\n\u001b[0;32m---> 51\u001b[0m \u001b[43mfollowExternalOnly\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhttps://www.oreilly.com/\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [37]\u001b[0m, in \u001b[0;36mfollowExternalOnly\u001b[0;34m(startingSite)\u001b[0m\n\u001b[1;32m     46\u001b[0m externalLink \u001b[38;5;241m=\u001b[39m getRandomExternalLink(startingSite)\n\u001b[1;32m     47\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mRandom external link is: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mexternalLink\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m---> 48\u001b[0m \u001b[43mfollowExternalOnly\u001b[49m\u001b[43m(\u001b[49m\u001b[43mexternalLink\u001b[49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [37]\u001b[0m, in \u001b[0;36mfollowExternalOnly\u001b[0;34m(startingSite)\u001b[0m\n\u001b[1;32m     46\u001b[0m externalLink \u001b[38;5;241m=\u001b[39m getRandomExternalLink(startingSite)\n\u001b[1;32m     47\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mRandom external link is: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mexternalLink\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m---> 48\u001b[0m \u001b[43mfollowExternalOnly\u001b[49m\u001b[43m(\u001b[49m\u001b[43mexternalLink\u001b[49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [37]\u001b[0m, in \u001b[0;36mfollowExternalOnly\u001b[0;34m(startingSite)\u001b[0m\n\u001b[1;32m     46\u001b[0m externalLink \u001b[38;5;241m=\u001b[39m getRandomExternalLink(startingSite)\n\u001b[1;32m     47\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mRandom external link is: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mexternalLink\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m---> 48\u001b[0m \u001b[43mfollowExternalOnly\u001b[49m\u001b[43m(\u001b[49m\u001b[43mexternalLink\u001b[49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [37]\u001b[0m, in \u001b[0;36mfollowExternalOnly\u001b[0;34m(startingSite)\u001b[0m\n\u001b[1;32m     45\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfollowExternalOnly\u001b[39m(startingSite):\n\u001b[0;32m---> 46\u001b[0m     externalLink \u001b[38;5;241m=\u001b[39m \u001b[43mgetRandomExternalLink\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstartingSite\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     47\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mRandom external link is: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mexternalLink\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     48\u001b[0m     followExternalOnly(externalLink)\n",
      "Input \u001b[0;32mIn [37]\u001b[0m, in \u001b[0;36mgetRandomExternalLink\u001b[0;34m(startingPage)\u001b[0m\n\u001b[1;32m     35\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mgetRandomExternalLink\u001b[39m(startingPage):\n\u001b[0;32m---> 36\u001b[0m     bs \u001b[38;5;241m=\u001b[39m BeautifulSoup(\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstartingPage\u001b[49m\u001b[43m)\u001b[49m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhtml.parser\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     37\u001b[0m     externalLinks \u001b[38;5;241m=\u001b[39m getExternalLinks(bs, startingPage)\n\u001b[1;32m     38\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(externalLinks):\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:214\u001b[0m, in \u001b[0;36murlopen\u001b[0;34m(url, data, timeout, cafile, capath, cadefault, context)\u001b[0m\n\u001b[1;32m    212\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    213\u001b[0m     opener \u001b[38;5;241m=\u001b[39m _opener\n\u001b[0;32m--> 214\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mopener\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:517\u001b[0m, in \u001b[0;36mOpenerDirector.open\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    514\u001b[0m     req \u001b[38;5;241m=\u001b[39m meth(req)\n\u001b[1;32m    516\u001b[0m sys\u001b[38;5;241m.\u001b[39maudit(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124murllib.Request\u001b[39m\u001b[38;5;124m'\u001b[39m, req\u001b[38;5;241m.\u001b[39mfull_url, req\u001b[38;5;241m.\u001b[39mdata, req\u001b[38;5;241m.\u001b[39mheaders, req\u001b[38;5;241m.\u001b[39mget_method())\n\u001b[0;32m--> 517\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    519\u001b[0m \u001b[38;5;66;03m# post-process response\u001b[39;00m\n\u001b[1;32m    520\u001b[0m meth_name \u001b[38;5;241m=\u001b[39m protocol\u001b[38;5;241m+\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_response\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:534\u001b[0m, in \u001b[0;36mOpenerDirector._open\u001b[0;34m(self, req, data)\u001b[0m\n\u001b[1;32m    531\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m result\n\u001b[1;32m    533\u001b[0m protocol \u001b[38;5;241m=\u001b[39m req\u001b[38;5;241m.\u001b[39mtype\n\u001b[0;32m--> 534\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_chain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_open\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprotocol\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprotocol\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\n\u001b[1;32m    535\u001b[0m \u001b[43m                          \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m_open\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    536\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m result:\n\u001b[1;32m    537\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:494\u001b[0m, in \u001b[0;36mOpenerDirector._call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    492\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m handler \u001b[38;5;129;01min\u001b[39;00m handlers:\n\u001b[1;32m    493\u001b[0m     func \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(handler, meth_name)\n\u001b[0;32m--> 494\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    495\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m result \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    496\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:1389\u001b[0m, in \u001b[0;36mHTTPSHandler.https_open\u001b[0;34m(self, req)\u001b[0m\n\u001b[1;32m   1388\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mhttps_open\u001b[39m(\u001b[38;5;28mself\u001b[39m, req):\n\u001b[0;32m-> 1389\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhttp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mHTTPSConnection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1390\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_context\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcheck_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_check_hostname\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:1350\u001b[0m, in \u001b[0;36mAbstractHTTPHandler.do_open\u001b[0;34m(self, http_class, req, **http_conn_args)\u001b[0m\n\u001b[1;32m   1348\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err: \u001b[38;5;66;03m# timeout error\u001b[39;00m\n\u001b[1;32m   1349\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m URLError(err)\n\u001b[0;32m-> 1350\u001b[0m     r \u001b[38;5;241m=\u001b[39m \u001b[43mh\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetresponse\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1351\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n\u001b[1;32m   1352\u001b[0m     h\u001b[38;5;241m.\u001b[39mclose()\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:1377\u001b[0m, in \u001b[0;36mHTTPConnection.getresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1375\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1376\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1377\u001b[0m         \u001b[43mresponse\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbegin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1378\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m:\n\u001b[1;32m   1379\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclose()\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:320\u001b[0m, in \u001b[0;36mHTTPResponse.begin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    318\u001b[0m \u001b[38;5;66;03m# read until we get a non-100 response\u001b[39;00m\n\u001b[1;32m    319\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 320\u001b[0m     version, status, reason \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    321\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m status \u001b[38;5;241m!=\u001b[39m CONTINUE:\n\u001b[1;32m    322\u001b[0m         \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:281\u001b[0m, in \u001b[0;36mHTTPResponse._read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    280\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_read_status\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 281\u001b[0m     line \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreadline\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_MAXLINE\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124miso-8859-1\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    282\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(line) \u001b[38;5;241m>\u001b[39m _MAXLINE:\n\u001b[1;32m    283\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m LineTooLong(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstatus line\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/socket.py:704\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    702\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m    703\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 704\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    705\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n\u001b[1;32m    706\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_timeout_occurred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/ssl.py:1241\u001b[0m, in \u001b[0;36mSSLSocket.recv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m   1237\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m flags \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m   1238\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   1239\u001b[0m           \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnon-zero flags not allowed in calls to recv_into() on \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m\n\u001b[1;32m   1240\u001b[0m           \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m)\n\u001b[0;32m-> 1241\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnbytes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1242\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1243\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mrecv_into(buffer, nbytes, flags)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/ssl.py:1099\u001b[0m, in \u001b[0;36mSSLSocket.read\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m   1097\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1098\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m buffer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1099\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1100\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1101\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sslobj\u001b[38;5;241m.\u001b[39mread(\u001b[38;5;28mlen\u001b[39m)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from urllib.parse import urlparse\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import datetime\n",
    "import random\n",
    "\n",
    "#Retrieves a list of all Internal links found on a page\n",
    "def getInternalLinks(bs, url):\n",
    "    netloc = urlparse(url).netloc\n",
    "    scheme = urlparse(url).scheme\n",
    "    internalLinks = set()\n",
    "    for link in bs.find_all('a'):\n",
    "        if not link.attrs.get('href'):\n",
    "            continue\n",
    "        parsed = urlparse(link.attrs['href'])\n",
    "        if parsed.netloc == '':\n",
    "            internalLinks.add(f'{scheme}://{netloc}/{link.attrs[\"href\"].strip(\"/\")}')\n",
    "        elif parsed.netloc == netloc:\n",
    "            internalLinks.add(link.attrs['href'])\n",
    "    return list(internalLinks)\n",
    "            \n",
    "#Retrieves a list of all external links found on a page\n",
    "def getExternalLinks(bs, url):\n",
    "    netloc = urlparse(url).netloc\n",
    "    externalLinks = set()\n",
    "    for link in bs.find_all('a'):\n",
    "        if not link.attrs.get('href'):\n",
    "            continue\n",
    "        parsed = urlparse(link.attrs['href'])\n",
    "        if parsed.netloc != '' and parsed.netloc != netloc:\n",
    "            externalLinks.add(link.attrs['href'])\n",
    "    return list(externalLinks)\n",
    "\n",
    "def getRandomExternalLink(startingPage):\n",
    "    bs = BeautifulSoup(urlopen(startingPage), 'html.parser')\n",
    "    externalLinks = getExternalLinks(bs, startingPage)\n",
    "    if not len(externalLinks):\n",
    "        print('No external links, looking around the site for one')\n",
    "        internalLinks = getInternalLinks(bs, startingPage)\n",
    "        return getRandomExternalLink(random.choice(internalLinks))\n",
    "    else:\n",
    "        return random.choice(externalLinks)\n",
    "    \n",
    "def followExternalOnly(startingSite):\n",
    "    externalLink = getRandomExternalLink(startingSite)\n",
    "    print(f'Random external link is: {externalLink}')\n",
    "    followExternalOnly(externalLink)\n",
    "\n",
    "\n",
    "followExternalOnly('https://www.oreilly.com/')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect all External Links from a Site"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://itunes.apple.com/us/app/safari-to-go/id881697395\n",
      "https://learning.oreilly.com/search/?query=author%3A%22Sari%20Greene%22&extended_publisher_data=true&highlight=true&include_assessments=false&include_case_studies=true&include_courses=true&include_playlists=true&include_collections=true&include_notebooks=true&include_sandboxes=true&include_scenarios=true&is_academic_institution_account=false&source=user&sort=date_added&facet_json=true&json_facets=true&page=0&include_facets=false\n",
      "https://twitter.com/oreillymedia\n",
      "https://www.linkedin.com/company/oreilly-media\n",
      "https://oreilly.hk/\n",
      "https://oreilly.id/\n",
      "https://learning.oreilly.com/search/?query=author%3A%22Neal%20Ford%22&extended_publisher_data=true&highlight=true&include_assessments=false&include_case_studies=true&include_courses=true&include_playlists=true&include_collections=true&include_notebooks=true&include_sandboxes=true&include_scenarios=true&is_academic_institution_account=false&source=user&sort=date_added&facet_json=true&json_facets=true&page=0&include_facets=false\n",
      "https://learning.oreilly.com/search/?query=author%3A%22Kelsey%20Hightower%22&extended_publisher_data=true&highlight=true&include_assessments=false&include_case_studies=true&include_courses=true&include_playlists=true&include_collections=true&include_notebooks=true&include_sandboxes=true&include_scenarios=true&is_academic_institution_account=false&source=user&sort=date_added&facet_json=true&json_facets=true&page=0&include_facets=false\n",
      "https://play.google.com/store/apps/details?id=com.safariflow.queue\n",
      "https://www.youtube.com/user/OreillyMedia\n",
      "https://learning.oreilly.com/search/?query=author%3A%22Bruno%20Gon%C3%A7alves%22&extended_publisher_data=true&highlight=true&include_assessments=false&include_case_studies=true&include_courses=true&include_playlists=true&include_collections=true&include_notebooks=true&include_sandboxes=true&include_scenarios=true&is_academic_institution_account=false&source=user&sort=date_added&facet_json=true&json_facets=true&page=0&include_facets=false\n",
      "https://www.oreilly.co.jp/index.shtml\n",
      "https://channelstore.roku.com/details/c8a2d0096693eb9455f6ac165003ee06/oreilly\n",
      "https://learning.oreilly.com/search/?query=author%3A%22Arianne%20Dee%22&extended_publisher_data=true&highlight=true&include_assessments=false&include_case_studies=true&include_courses=true&include_playlists=true&include_collections=true&include_notebooks=true&include_sandboxes=true&include_scenarios=true&is_academic_institution_account=false&source=suggestion&sort=date_added&facet_json=true&json_facets=true&page=0&include_facets=false\n",
      "https://www.amazon.com/OReilly-Media-Inc/dp/B087YYHL5C/ref=sr_1_2?dchild=1&keywords=oreilly&qid=1604964116&s=mobile-apps&sr=1-2\n",
      "https://learning.oreilly.com/start-trial/\n",
      "https://oreillylearning.in/\n",
      "https://learning.oreilly.com/search/?query=author%3A%22Ken%20Kousen%22&extended_publisher_data=true&highlight=true&include_assessments=false&include_case_studies=true&include_courses=true&include_playlists=true&include_collections=true&include_notebooks=true&include_sandboxes=true&include_scenarios=true&is_academic_institution_account=false&source=user&sort=date_added&facet_json=true&json_facets=true&page=0&include_facets=false\n",
      "http://twitter.com/timoreilly\n",
      "http://www.oreillynet.com/pub/au/27\n",
      "http://oreilly.com/oreilly/tim_bio.csp\n",
      "http://www.web2expo.com/\n",
      "http://toccon.com\n",
      "http://www.oreillynet.com/pub/au/1242\n",
      "http://missingmanuals.com\n",
      "http://twitter.com/sarahm\n",
      "http://tim.oreilly.com\n",
      "http://shop.oreilly.com/product/0636920022336.do\n",
      "https://www.applytosupply.digitalmarketplace.service.gov.uk/g-cloud/services/460643971613582\n",
      "https://www.gsaelibrary.gsa.gov/ElibMain/contractorInfo.do?contractNumber=47QTCA19D0022&contractorName=O%27REILLY+MEDIA%2C+INC&executeQuery=YES\n",
      "https://cdn.oreillystatic.com/pdf/OReilly_Capabilities_Statement.pdf\n",
      "https://apps.apple.com/app/apple-store/id881697395?pt=206943&ct=oreilly-apps-page&mt=8\n",
      "https://chrome.google.com/webstore/detail/oreilly/honddgdnobfcbldlmimekbmphjnehbbb?hl=en\n",
      "https://learning.oreilly.com/signup/\n",
      "https://learning.oreilly.com/interactive/\n",
      "https://www.amazon.com/OReilly-Media-Inc/dp/B087YYHL5C/ref=sr_1_2?dchild=1&amp;keywords=oreilly&amp;qid=1604964116&amp;s=mobile-apps&amp;sr=1-2\n",
      "https://members.oreilly.com/\n",
      "https://www.facebook.com/OReilly\n",
      "https://learning.oreilly.com/register/\n",
      "https://oreillymedia.zoom.us/webinar/register/WN_BcLRmUGUTVWGRa-LvcxqHg\n",
      "https://oreillymedia.zoom.us/webinar/register/WN_MBTdQH-PTtyRLIuRa7eHYA\n",
      "https://oreillymedia.zoom.us/webinar/register/WN_hOecnEtERj-d5rMmw2feOQ\n",
      "https://oreillymedia.zoom.us/webinar/register/WN_sT5HhW89R4aaRTaKaMYxGQ\n",
      "https://oreillymedia.zoom.us/webinar/register/WN_9-GFj1gbQ66hZCenhisXKA\n",
      "https://learning.oreilly.com/team-setup/\n",
      "https://oreillymedia.zoom.us/webinar/register/WN_h8bJNEN4TsC54uo0o-zu4g\n",
      "https://learning.oreilly.com/live-events/first-steps-in-machine-learning-misfit-models-and-fixes/0636920087879/0636920087877/\n",
      "https://learning.oreilly.com/live-events/explainable-machine-learning-models-with-interactivity/0636920061943/0636920085001/\n",
      "https://learning.oreilly.com/live-events/ai-superstream-designing-machine-learning-systems/0636920086385/0636920086384/\n",
      "https://learning.oreilly.com/live-events/machine-learning-interviews-in-3-weeks/0636920080937/0636920085271/\n",
      "https://learning.oreilly.com/scenarios/regularization-in-regression/9781492091424/\n",
      "https://learning.oreilly.com/live-events/scaling-machine-learning-in-three-weeks/0636920081997/0636920087562/\n",
      "https://learning.oreilly.com/scenarios/serve-machine-learning/9781492080725/\n",
      "https://learning.oreilly.com/attend/\n",
      "https://learning.oreilly.com/scenarios/multiple-linear-regression/9781492091400/\n",
      "https://learning.oreilly.com/library/view/introduction-to-machine/9781449369880/\n",
      "https://learning.oreilly.com/search/?query=&extended_publisher_data=true&formats=book&highlight=true&include_assessments=false&include_case_studies=true&include_courses=true&include_orioles=true&include_playlists=true&is_academic_institution_account=false&publishers=O%27Reilly%20Media%2C%20Inc.&topics=Machine%20Learning&sort=popularity\n",
      "https://learning.oreilly.com/library/view/hands-on-machine-learning/9781098125967/\n",
      "https://learning.oreilly.com/live-events/machine-learning-algorithms-for-trading-bootcamp/0636920082023/0636920088138/\n",
      "https://learning.oreilly.com/scenarios/challenge-make-a/9781098113933/\n",
      "https://learning.oreilly.com/library/view/hands-on-machine-learning/9781492032632/\n",
      "https://learning.oreilly.com/scenarios/deploy-machine-learning/9781492080732/\n",
      "https://learning.oreilly.com/library/view/designing-machine-learning/9781098107956/\n",
      "https://learning.oreilly.com/scenarios/machine-learning-with/9781098142582/\n",
      "http://archive.oreilly.com/oreillyschool/privacy-policy/\n",
      "https://www.privacyshield.gov/article?id=ANNEX-I-introduction\n",
      "https://www.privacyshield.gov\n",
      "https://www.edoeb.admin.ch/edoeb/en/home/the-fdpic/task.html\n",
      "https://www.safaribooksonline.com/membership-agreement/\n",
      "https://ico.org.uk/\n",
      "https://www.safaribooksonline.com/static/legal/SafariPrivacyPolicy_v3.3_13June2017.a4d9478408f5.pdf\n",
      "http://archive.oreilly.com/oreillyschool/privacy-policy-version2/\n",
      "https://ec.europa.eu/info/law/law-topic/data-protection/data-transfers-outside-eu/model-contracts-transfer-personal-data-third-countries_en\n",
      "https://ec.europa.eu/info/law/law-topic/data-protection/data-transfers-outside-eu/adequacy-protection-personal-data-non-eu-countries_en\n",
      "https://edpo.com/gdpr-data-request/\n",
      "https://www.safaribooksonline.com/terms/\n",
      "https://tools.google.com/dlpage/gaoptout/\n",
      "https://learning.oreilly.com/membership-agreement/\n",
      "https://cdn.oreillystatic.com/pdf/SafariPrivacyPolicy_v3.4_23October2017.pdf\n",
      "https://www.privacyshield.gov/article?id=How-to-Submit-a-Complaint\n",
      "https://edpb.europa.eu/about-edpb/about-edpb/members_en\n",
      "//privacy.truste.com/privacy-seal/validation?rid=b1a9b95e-186e-4498-aaa0-2a7b98c52af0\n",
      "https://ec.europa.eu/info/law/law-topic/data-protection/data-transfers-outside-eu/eu-us-privacy-shield_en\n",
      "https://feedback-form.truste.com/watchdog/request\n",
      "//privacy.truste.com/privacy-seal/validation?rid=0bdead8c-1d9c-429d-ae6a-5ee92e15a471\n",
      "https://smetrics.oreilly.com/optout.html?optout=1&confirm_change=1\n",
      "http://www.oreilly.com.cn/\n",
      "http://www.oreilly.co.jp\n"
     ]
    },
    {
     "ename": "HTTPError",
     "evalue": "HTTP Error 404: Not Found",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mHTTPError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [38]\u001b[0m, in \u001b[0;36m<cell line: 22>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     18\u001b[0m             getAllExternalLinks(link)\n\u001b[1;32m     21\u001b[0m allIntLinks\u001b[38;5;241m.\u001b[39mappend(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttps://oreilly.com\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m---> 22\u001b[0m \u001b[43mgetAllExternalLinks\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhttps://www.oreilly.com/\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [38]\u001b[0m, in \u001b[0;36mgetAllExternalLinks\u001b[0;34m(url)\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m link \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m allIntLinks:\n\u001b[1;32m     17\u001b[0m     allIntLinks\u001b[38;5;241m.\u001b[39mappend(link)\n\u001b[0;32m---> 18\u001b[0m     \u001b[43mgetAllExternalLinks\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlink\u001b[49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [38]\u001b[0m, in \u001b[0;36mgetAllExternalLinks\u001b[0;34m(url)\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m link \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m allIntLinks:\n\u001b[1;32m     17\u001b[0m     allIntLinks\u001b[38;5;241m.\u001b[39mappend(link)\n\u001b[0;32m---> 18\u001b[0m     \u001b[43mgetAllExternalLinks\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlink\u001b[49m\u001b[43m)\u001b[49m\n",
      "    \u001b[0;31m[... skipping similar frames: getAllExternalLinks at line 18 (20 times)]\u001b[0m\n",
      "Input \u001b[0;32mIn [38]\u001b[0m, in \u001b[0;36mgetAllExternalLinks\u001b[0;34m(url)\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m link \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m allIntLinks:\n\u001b[1;32m     17\u001b[0m     allIntLinks\u001b[38;5;241m.\u001b[39mappend(link)\n\u001b[0;32m---> 18\u001b[0m     \u001b[43mgetAllExternalLinks\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlink\u001b[49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [38]\u001b[0m, in \u001b[0;36mgetAllExternalLinks\u001b[0;34m(url)\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mgetAllExternalLinks\u001b[39m(url):\n\u001b[0;32m----> 7\u001b[0m     bs \u001b[38;5;241m=\u001b[39m BeautifulSoup(\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m)\u001b[49m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhtml.parser\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m      8\u001b[0m     internalLinks \u001b[38;5;241m=\u001b[39m getInternalLinks(bs, url)\n\u001b[1;32m      9\u001b[0m     externalLinks \u001b[38;5;241m=\u001b[39m getExternalLinks(bs, url)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:214\u001b[0m, in \u001b[0;36murlopen\u001b[0;34m(url, data, timeout, cafile, capath, cadefault, context)\u001b[0m\n\u001b[1;32m    212\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    213\u001b[0m     opener \u001b[38;5;241m=\u001b[39m _opener\n\u001b[0;32m--> 214\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mopener\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:523\u001b[0m, in \u001b[0;36mOpenerDirector.open\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    521\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m processor \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocess_response\u001b[38;5;241m.\u001b[39mget(protocol, []):\n\u001b[1;32m    522\u001b[0m     meth \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(processor, meth_name)\n\u001b[0;32m--> 523\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[43mmeth\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    525\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:632\u001b[0m, in \u001b[0;36mHTTPErrorProcessor.http_response\u001b[0;34m(self, request, response)\u001b[0m\n\u001b[1;32m    629\u001b[0m \u001b[38;5;66;03m# According to RFC 2616, \"2xx\" code indicates that the client's\u001b[39;00m\n\u001b[1;32m    630\u001b[0m \u001b[38;5;66;03m# request was successfully received, understood, and accepted.\u001b[39;00m\n\u001b[1;32m    631\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;241m200\u001b[39m \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m code \u001b[38;5;241m<\u001b[39m \u001b[38;5;241m300\u001b[39m):\n\u001b[0;32m--> 632\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43merror\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    633\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhttp\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhdrs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    635\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:555\u001b[0m, in \u001b[0;36mOpenerDirector.error\u001b[0;34m(self, proto, *args)\u001b[0m\n\u001b[1;32m    553\u001b[0m     http_err \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m    554\u001b[0m args \u001b[38;5;241m=\u001b[39m (\u001b[38;5;28mdict\u001b[39m, proto, meth_name) \u001b[38;5;241m+\u001b[39m args\n\u001b[0;32m--> 555\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_chain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    556\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m result:\n\u001b[1;32m    557\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:494\u001b[0m, in \u001b[0;36mOpenerDirector._call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    492\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m handler \u001b[38;5;129;01min\u001b[39;00m handlers:\n\u001b[1;32m    493\u001b[0m     func \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(handler, meth_name)\n\u001b[0;32m--> 494\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    495\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m result \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    496\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:747\u001b[0m, in \u001b[0;36mHTTPRedirectHandler.http_error_302\u001b[0;34m(self, req, fp, code, msg, headers)\u001b[0m\n\u001b[1;32m    744\u001b[0m fp\u001b[38;5;241m.\u001b[39mread()\n\u001b[1;32m    745\u001b[0m fp\u001b[38;5;241m.\u001b[39mclose()\n\u001b[0;32m--> 747\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:523\u001b[0m, in \u001b[0;36mOpenerDirector.open\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    521\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m processor \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocess_response\u001b[38;5;241m.\u001b[39mget(protocol, []):\n\u001b[1;32m    522\u001b[0m     meth \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(processor, meth_name)\n\u001b[0;32m--> 523\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[43mmeth\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    525\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:632\u001b[0m, in \u001b[0;36mHTTPErrorProcessor.http_response\u001b[0;34m(self, request, response)\u001b[0m\n\u001b[1;32m    629\u001b[0m \u001b[38;5;66;03m# According to RFC 2616, \"2xx\" code indicates that the client's\u001b[39;00m\n\u001b[1;32m    630\u001b[0m \u001b[38;5;66;03m# request was successfully received, understood, and accepted.\u001b[39;00m\n\u001b[1;32m    631\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;241m200\u001b[39m \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m code \u001b[38;5;241m<\u001b[39m \u001b[38;5;241m300\u001b[39m):\n\u001b[0;32m--> 632\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43merror\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    633\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhttp\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhdrs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    635\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:561\u001b[0m, in \u001b[0;36mOpenerDirector.error\u001b[0;34m(self, proto, *args)\u001b[0m\n\u001b[1;32m    559\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m http_err:\n\u001b[1;32m    560\u001b[0m     args \u001b[38;5;241m=\u001b[39m (\u001b[38;5;28mdict\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdefault\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttp_error_default\u001b[39m\u001b[38;5;124m'\u001b[39m) \u001b[38;5;241m+\u001b[39m orig_args\n\u001b[0;32m--> 561\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_chain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:494\u001b[0m, in \u001b[0;36mOpenerDirector._call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    492\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m handler \u001b[38;5;129;01min\u001b[39;00m handlers:\n\u001b[1;32m    493\u001b[0m     func \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(handler, meth_name)\n\u001b[0;32m--> 494\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    495\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m result \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    496\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:641\u001b[0m, in \u001b[0;36mHTTPDefaultErrorHandler.http_error_default\u001b[0;34m(self, req, fp, code, msg, hdrs)\u001b[0m\n\u001b[1;32m    640\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mhttp_error_default\u001b[39m(\u001b[38;5;28mself\u001b[39m, req, fp, code, msg, hdrs):\n\u001b[0;32m--> 641\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m HTTPError(req\u001b[38;5;241m.\u001b[39mfull_url, code, msg, hdrs, fp)\n",
      "\u001b[0;31mHTTPError\u001b[0m: HTTP Error 404: Not Found"
     ]
    }
   ],
   "source": [
    "# Collects a list of all external URLs found on the site\n",
    "allExtLinks = []\n",
    "allIntLinks = []\n",
    "\n",
    "\n",
    "def getAllExternalLinks(url):\n",
    "    bs = BeautifulSoup(urlopen(url), 'html.parser')\n",
    "    internalLinks = getInternalLinks(bs, url)\n",
    "    externalLinks = getExternalLinks(bs, url)\n",
    "    for link in externalLinks:\n",
    "        if link not in allExtLinks:\n",
    "            allExtLinks.append(link)\n",
    "            print(link)\n",
    "\n",
    "    for link in internalLinks:\n",
    "        if link not in allIntLinks:\n",
    "            allIntLinks.append(link)\n",
    "            getAllExternalLinks(link)\n",
    "\n",
    "\n",
    "allIntLinks.append('https://oreilly.com')\n",
    "getAllExternalLinks('https://www.oreilly.com/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}