Chapter15_API-Scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import json\n",
    "import datetime\n",
    "import re\n",
    "import random\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------\n",
      "history url is: http://en.wikipedia.org/w/index.php?title=Programming_paradigm&action=history\n",
      "2405:201:2009:80b0:41bc:366f:a49c:52f2\n",
      "115.186.189.53\n",
      "103.252.145.68\n",
      "2405:201:400b:7058:b128:89fd:5248:f249\n",
      "172.115.220.47\n",
      "2806:1016:d:54b6:8950:4501:c00b:507a\n",
      "36.255.87.160\n",
      "2603:6011:1100:a1d0:31bd:8a11:a0c8:e4c3\n",
      "2806:108e:d:bd2c:a577:db4f:2867:2b5c\n",
      "2409:4042:e8f:8d39:b50c:f4ca:91b8:eb9d\n",
      "107.190.108.84\n",
      "--------------------\n",
      "history url is: http://en.wikipedia.org/w/index.php?title=Multi-paradigm_programming_language&action=history\n",
      "98.197.198.46\n",
      "75.139.254.117\n",
      "--------------------\n",
      "history url is: http://en.wikipedia.org/w/index.php?title=Object-oriented_programming&action=history\n",
      "86.186.86.97\n",
      "82.151.115.17\n",
      "2a00:23c8:2c00:4d01:9d37:3187:6b3c:6d7b\n",
      "2a01:cb1e:75:31ab:a2:ba91:d0e:6965\n",
      "67.182.232.160\n",
      "92.253.43.187\n",
      "195.12.232.246\n",
      "104.156.69.18\n",
      "115.241.201.77\n",
      "2405:9800:b923:2b6:b4b7:730b:405b:a1e0\n",
      "130.208.188.159\n",
      "2a00:23ee:1050:29af:6020:e55f:8546:e258\n",
      "150.135.165.11\n",
      "12.183.178.250\n",
      "--------------------\n",
      "history url is: http://en.wikipedia.org/w/index.php?title=Procedural_programming&action=history\n",
      "112.134.214.53\n",
      "2607:fea8:235e:b600:cad6:9572:d5a8:1639\n",
      "174.90.223.214\n",
      "2600:1000:b045:9be1:67c4:f788:c2fb:461d\n",
      "83.173.200.121\n",
      "216.225.124.244\n",
      "2601:2c3:867f:e810:5b9b:58e7:b427:e352\n",
      "162.208.181.210\n",
      "117.220.135.30\n",
      "2001:44c8:45d2:11cd:1:0:2e69:c996\n",
      "2001:8f8:1e35:fa30:52b:190e:146a:4b48\n",
      "112.79.118.122\n",
      "2607:fea8:9560:8ae:e155:3e6f:7747:5544\n",
      "192.208.125.161\n",
      "50.201.78.245\n",
      "151.255.244.223\n",
      "--------------------\n",
      "history url is: http://en.wikipedia.org/w/index.php?title=Imperative_programming&action=history\n",
      "27.2.20.10\n",
      "97.121.170.107\n",
      "117.207.179.184\n",
      "2600:8803:c40a:800:b47e:b314:65fe:8b6c\n",
      "173.225.40.198\n",
      "2a02:c7f:c28c:f400:e510:1e3e:d3d2:a21e\n",
      "207.255.75.92\n",
      "223.24.167.37\n",
      "49.207.196.35\n",
      "106.201.59.250\n",
      "189.204.196.201\n",
      "2a02:c7f:74c3:c300:5c5a:ce83:7325:4429\n",
      "2603:6010:df00:21d5:8dbc:53ba:9aa1:6267\n",
      "2405:204:a3a7:c3bd::f0b:a0ac\n",
      "2001:4454:38f:fc00:c8f1:b67e:da95:ab6f\n",
      "--------------------\n",
      "history url is: http://en.wikipedia.org/w/index.php?title=Functional_programming&action=history\n",
      "62.175.103.210\n",
      "2607:fea8:235f:9730::1232\n",
      "2600:6c4a:4d7f:e999:d21:9ad5:a5e4:e20d\n",
      "105.12.5.186\n",
      "134.204.220.36\n",
      "2607:fea8:2363:8f00:a000:dce0:5c45:2642\n",
      "2804:14c:1c2:821c:4970:1293:d1dc:3f00\n",
      "23.121.65.56\n",
      "104.152.255.71\n",
      "109.125.120.234\n",
      "91.243.4.220\n",
      "--------------------\n",
      "history url is: http://en.wikipedia.org/w/index.php?title=Structured_programming&action=history\n",
      "83.223.241.43\n",
      "2401:4900:c16:df76:13e:4e10:599a:642e\n",
      "165.82.134.144\n",
      "2409:4051:291:a148:159a:ee30:ad1b:2179\n",
      "2600:8805:a981:4300:65d3:9258:af92:d91c\n",
      "156.199.200.186\n",
      "76.8.32.203\n",
      "223.196.166.69\n",
      "64.119.192.201\n",
      "103.164.47.189\n",
      "87.254.97.74\n",
      "41.89.61.3\n",
      "206.214.57.47\n",
      "--------------------\n",
      "history url is: http://en.wikipedia.org/w/index.php?title=Reflective_programming&action=history\n",
      "174.88.46.109\n",
      "2804:14d:72a3:c4:6d48:d3fb:77ea:5e3\n",
      "2409:4064:4db3:5f54::13c8:b209\n",
      "172.72.221.49\n",
      "194.96.27.238\n",
      "95.60.233.218\n",
      "2601:444:8280:6880:b0c0:1c65:8627:243\n",
      "2600:1700:87d0:5fd0:ac5c:5d73:f9a6:6fe2\n",
      "94.21.10.204\n",
      "2804:14d:3291:8807::2\n",
      "--------------------\n",
      "history url is: http://en.wikipedia.org/w/index.php?title=Software_design&action=history\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [3]\u001b[0m, in \u001b[0;36m<cell line: 28>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m link \u001b[38;5;129;01min\u001b[39;00m links:\n\u001b[1;32m     30\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m-\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m20\u001b[39m) \n\u001b[0;32m---> 31\u001b[0m     historyIPs \u001b[38;5;241m=\u001b[39m \u001b[43mgetHistoryIPs\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlink\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mattrs\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhref\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     32\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m historyIP \u001b[38;5;129;01min\u001b[39;00m historyIPs:\n\u001b[1;32m     33\u001b[0m         \u001b[38;5;28mprint\u001b[39m(historyIP)\n",
      "Input \u001b[0;32mIn [3]\u001b[0m, in \u001b[0;36mgetHistoryIPs\u001b[0;34m(pageUrl)\u001b[0m\n\u001b[1;32m     18\u001b[0m historyUrl \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttp://en.wikipedia.org/w/index.php?title=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpageUrl\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m&action=history\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m     19\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhistory url is: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mhistoryUrl\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m---> 20\u001b[0m bs \u001b[38;5;241m=\u001b[39m BeautifulSoup(\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhistoryUrl\u001b[49m\u001b[43m)\u001b[49m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhtml.parser\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     21\u001b[0m \u001b[38;5;66;03m#finds only the links with class \"mw-anonuserlink\" which has IP addresses \u001b[39;00m\n\u001b[1;32m     22\u001b[0m \u001b[38;5;66;03m#instead of usernames\u001b[39;00m\n\u001b[1;32m     23\u001b[0m ipAddresses \u001b[38;5;241m=\u001b[39m bs\u001b[38;5;241m.\u001b[39mfindAll(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ma\u001b[39m\u001b[38;5;124m'\u001b[39m, {\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mclass\u001b[39m\u001b[38;5;124m'\u001b[39m:\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmw-anonuserlink\u001b[39m\u001b[38;5;124m'\u001b[39m})\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:214\u001b[0m, in \u001b[0;36murlopen\u001b[0;34m(url, data, timeout, cafile, capath, cadefault, context)\u001b[0m\n\u001b[1;32m    212\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    213\u001b[0m     opener \u001b[38;5;241m=\u001b[39m _opener\n\u001b[0;32m--> 214\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mopener\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:523\u001b[0m, in \u001b[0;36mOpenerDirector.open\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    521\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m processor \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocess_response\u001b[38;5;241m.\u001b[39mget(protocol, []):\n\u001b[1;32m    522\u001b[0m     meth \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(processor, meth_name)\n\u001b[0;32m--> 523\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[43mmeth\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    525\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:632\u001b[0m, in \u001b[0;36mHTTPErrorProcessor.http_response\u001b[0;34m(self, request, response)\u001b[0m\n\u001b[1;32m    629\u001b[0m \u001b[38;5;66;03m# According to RFC 2616, \"2xx\" code indicates that the client's\u001b[39;00m\n\u001b[1;32m    630\u001b[0m \u001b[38;5;66;03m# request was successfully received, understood, and accepted.\u001b[39;00m\n\u001b[1;32m    631\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;241m200\u001b[39m \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m code \u001b[38;5;241m<\u001b[39m \u001b[38;5;241m300\u001b[39m):\n\u001b[0;32m--> 632\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43merror\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    633\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhttp\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhdrs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    635\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:555\u001b[0m, in \u001b[0;36mOpenerDirector.error\u001b[0;34m(self, proto, *args)\u001b[0m\n\u001b[1;32m    553\u001b[0m     http_err \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m    554\u001b[0m args \u001b[38;5;241m=\u001b[39m (\u001b[38;5;28mdict\u001b[39m, proto, meth_name) \u001b[38;5;241m+\u001b[39m args\n\u001b[0;32m--> 555\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_chain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    556\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m result:\n\u001b[1;32m    557\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:494\u001b[0m, in \u001b[0;36mOpenerDirector._call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    492\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m handler \u001b[38;5;129;01min\u001b[39;00m handlers:\n\u001b[1;32m    493\u001b[0m     func \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(handler, meth_name)\n\u001b[0;32m--> 494\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    495\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m result \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    496\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:747\u001b[0m, in \u001b[0;36mHTTPRedirectHandler.http_error_302\u001b[0;34m(self, req, fp, code, msg, headers)\u001b[0m\n\u001b[1;32m    744\u001b[0m fp\u001b[38;5;241m.\u001b[39mread()\n\u001b[1;32m    745\u001b[0m fp\u001b[38;5;241m.\u001b[39mclose()\n\u001b[0;32m--> 747\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnew\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:517\u001b[0m, in \u001b[0;36mOpenerDirector.open\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    514\u001b[0m     req \u001b[38;5;241m=\u001b[39m meth(req)\n\u001b[1;32m    516\u001b[0m sys\u001b[38;5;241m.\u001b[39maudit(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124murllib.Request\u001b[39m\u001b[38;5;124m'\u001b[39m, req\u001b[38;5;241m.\u001b[39mfull_url, req\u001b[38;5;241m.\u001b[39mdata, req\u001b[38;5;241m.\u001b[39mheaders, req\u001b[38;5;241m.\u001b[39mget_method())\n\u001b[0;32m--> 517\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    519\u001b[0m \u001b[38;5;66;03m# post-process response\u001b[39;00m\n\u001b[1;32m    520\u001b[0m meth_name \u001b[38;5;241m=\u001b[39m protocol\u001b[38;5;241m+\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_response\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:534\u001b[0m, in \u001b[0;36mOpenerDirector._open\u001b[0;34m(self, req, data)\u001b[0m\n\u001b[1;32m    531\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m result\n\u001b[1;32m    533\u001b[0m protocol \u001b[38;5;241m=\u001b[39m req\u001b[38;5;241m.\u001b[39mtype\n\u001b[0;32m--> 534\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_chain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_open\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprotocol\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprotocol\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\n\u001b[1;32m    535\u001b[0m \u001b[43m                          \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m_open\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    536\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m result:\n\u001b[1;32m    537\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:494\u001b[0m, in \u001b[0;36mOpenerDirector._call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    492\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m handler \u001b[38;5;129;01min\u001b[39;00m handlers:\n\u001b[1;32m    493\u001b[0m     func \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(handler, meth_name)\n\u001b[0;32m--> 494\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    495\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m result \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    496\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:1389\u001b[0m, in \u001b[0;36mHTTPSHandler.https_open\u001b[0;34m(self, req)\u001b[0m\n\u001b[1;32m   1388\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mhttps_open\u001b[39m(\u001b[38;5;28mself\u001b[39m, req):\n\u001b[0;32m-> 1389\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhttp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mHTTPSConnection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1390\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_context\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcheck_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_check_hostname\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:1350\u001b[0m, in \u001b[0;36mAbstractHTTPHandler.do_open\u001b[0;34m(self, http_class, req, **http_conn_args)\u001b[0m\n\u001b[1;32m   1348\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err: \u001b[38;5;66;03m# timeout error\u001b[39;00m\n\u001b[1;32m   1349\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m URLError(err)\n\u001b[0;32m-> 1350\u001b[0m     r \u001b[38;5;241m=\u001b[39m \u001b[43mh\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetresponse\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1351\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n\u001b[1;32m   1352\u001b[0m     h\u001b[38;5;241m.\u001b[39mclose()\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:1377\u001b[0m, in \u001b[0;36mHTTPConnection.getresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1375\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1376\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1377\u001b[0m         \u001b[43mresponse\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbegin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1378\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m:\n\u001b[1;32m   1379\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclose()\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:320\u001b[0m, in \u001b[0;36mHTTPResponse.begin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    318\u001b[0m \u001b[38;5;66;03m# read until we get a non-100 response\u001b[39;00m\n\u001b[1;32m    319\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 320\u001b[0m     version, status, reason \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    321\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m status \u001b[38;5;241m!=\u001b[39m CONTINUE:\n\u001b[1;32m    322\u001b[0m         \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:281\u001b[0m, in \u001b[0;36mHTTPResponse._read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    280\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_read_status\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 281\u001b[0m     line \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreadline\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_MAXLINE\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124miso-8859-1\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    282\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(line) \u001b[38;5;241m>\u001b[39m _MAXLINE:\n\u001b[1;32m    283\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m LineTooLong(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstatus line\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/socket.py:704\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    702\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m    703\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 704\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    705\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n\u001b[1;32m    706\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_timeout_occurred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/ssl.py:1241\u001b[0m, in \u001b[0;36mSSLSocket.recv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m   1237\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m flags \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m   1238\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   1239\u001b[0m           \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnon-zero flags not allowed in calls to recv_into() on \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m\n\u001b[1;32m   1240\u001b[0m           \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m)\n\u001b[0;32m-> 1241\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnbytes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1242\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1243\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mrecv_into(buffer, nbytes, flags)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/ssl.py:1099\u001b[0m, in \u001b[0;36mSSLSocket.read\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m   1097\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1098\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m buffer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1099\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1100\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1101\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sslobj\u001b[38;5;241m.\u001b[39mread(\u001b[38;5;28mlen\u001b[39m)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "def getLinks(articleUrl):\n",
    "    html = urlopen(f'http://en.wikipedia.org{articleUrl}')\n",
    "    bs = BeautifulSoup(html, 'html.parser')\n",
    "    return bs.find('div', {'id':'bodyContent'}).findAll('a', \n",
    "        href=re.compile('^(/wiki/)((?!:).)*$'))\n",
    "\n",
    "def getHistoryIPs(pageUrl):\n",
    "    #Format of revision history pages is: \n",
    "    #http://en.wikipedia.org/w/index.php?title=Title_in_URL&action=history\n",
    "    pageUrl = pageUrl.replace('/wiki/', '')\n",
    "    historyUrl = f'http://en.wikipedia.org/w/index.php?title={pageUrl}&action=history'\n",
    "    print(f'history url is: {historyUrl}')\n",
    "    bs = BeautifulSoup(urlopen(historyUrl), 'html.parser')\n",
    "    #finds only the links with class \"mw-anonuserlink\" which has IP addresses \n",
    "    #instead of usernames\n",
    "    ipAddresses = bs.findAll('a', {'class':'mw-anonuserlink'})\n",
    "    return set([ip.get_text() for ip in ipAddresses])\n",
    "\n",
    "links = getLinks('/wiki/Python_(programming_language)')\n",
    "\n",
    "while(len(links) > 0):\n",
    "    for link in links:\n",
    "        print('-'*20) \n",
    "        historyIPs = getHistoryIPs(link.attrs['href'])\n",
    "        for historyIP in historyIPs:\n",
    "            print(historyIP)\n",
    "\n",
    "    newLink = links[random.randint(0, len(links)-1)].attrs['href']\n",
    "    links = getLinks(newLink)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------\n",
      "history url is: http://en.wikipedia.org/w/index.php?title=Programming_paradigm&action=history\n",
      "2405:201:2009:80b0:41bc:366f:a49c:52f2 is from IN\n",
      "115.186.189.53 is from PK\n",
      "103.252.145.68 is from IN\n",
      "2405:201:400b:7058:b128:89fd:5248:f249 is from IN\n",
      "172.115.220.47 is from US\n",
      "2806:1016:d:54b6:8950:4501:c00b:507a is from MX\n",
      "36.255.87.160 is from IN\n",
      "2603:6011:1100:a1d0:31bd:8a11:a0c8:e4c3 is from US\n",
      "2806:108e:d:bd2c:a577:db4f:2867:2b5c is from MX\n",
      "2409:4042:e8f:8d39:b50c:f4ca:91b8:eb9d is from IN\n",
      "107.190.108.84 is from CA\n",
      "--------------------\n",
      "history url is: http://en.wikipedia.org/w/index.php?title=Multi-paradigm_programming_language&action=history\n",
      "98.197.198.46 is from US\n",
      "75.139.254.117 is from US\n",
      "--------------------\n",
      "history url is: http://en.wikipedia.org/w/index.php?title=Object-oriented_programming&action=history\n",
      "86.186.86.97 is from GB\n",
      "82.151.115.17 is from RU\n",
      "2a00:23c8:2c00:4d01:9d37:3187:6b3c:6d7b is from GB\n",
      "2a01:cb1e:75:31ab:a2:ba91:d0e:6965 is from FR\n",
      "67.182.232.160 is from US\n",
      "92.253.43.187 is from JO\n",
      "195.12.232.246 is from DE\n",
      "104.156.69.18 is from CA\n",
      "115.241.201.77 is from IN\n",
      "2405:9800:b923:2b6:b4b7:730b:405b:a1e0 is from TH\n",
      "130.208.188.159 is from IS\n",
      "2a00:23ee:1050:29af:6020:e55f:8546:e258 is from GB\n",
      "150.135.165.11 is from US\n",
      "12.183.178.250 is from US\n",
      "--------------------\n",
      "history url is: http://en.wikipedia.org/w/index.php?title=Procedural_programming&action=history\n",
      "112.134.214.53 is from LK\n",
      "2607:fea8:235e:b600:cad6:9572:d5a8:1639 is from CA\n",
      "174.90.223.214 is from CA\n",
      "2600:1000:b045:9be1:67c4:f788:c2fb:461d is from US\n",
      "83.173.200.121 is from CH\n",
      "216.225.124.244 is from US\n",
      "2601:2c3:867f:e810:5b9b:58e7:b427:e352 is from US\n",
      "162.208.181.210 is from CA\n",
      "117.220.135.30 is from IN\n",
      "2001:44c8:45d2:11cd:1:0:2e69:c996 is from TH\n",
      "2001:8f8:1e35:fa30:52b:190e:146a:4b48 is from AE\n",
      "112.79.118.122 is from IN\n",
      "2607:fea8:9560:8ae:e155:3e6f:7747:5544 is from CA\n",
      "192.208.125.161 is from US\n",
      "50.201.78.245 is from US\n",
      "151.255.244.223 is from SA\n",
      "--------------------\n",
      "history url is: http://en.wikipedia.org/w/index.php?title=Imperative_programming&action=history\n",
      "27.2.20.10 is from VN\n",
      "97.121.170.107 is from US\n",
      "117.207.179.184 is from IN\n",
      "2600:8803:c40a:800:b47e:b314:65fe:8b6c is from US\n",
      "173.225.40.198 is from US\n",
      "2a02:c7f:c28c:f400:e510:1e3e:d3d2:a21e is from GB\n",
      "207.255.75.92 is from US\n",
      "223.24.167.37 is from TH\n",
      "49.207.196.35 is from IN\n",
      "106.201.59.250 is from IN\n",
      "189.204.196.201 is from MX\n",
      "2a02:c7f:74c3:c300:5c5a:ce83:7325:4429 is from GB\n",
      "2603:6010:df00:21d5:8dbc:53ba:9aa1:6267 is from US\n",
      "2405:204:a3a7:c3bd::f0b:a0ac is from IN\n",
      "2001:4454:38f:fc00:c8f1:b67e:da95:ab6f is from PH\n",
      "--------------------\n",
      "history url is: http://en.wikipedia.org/w/index.php?title=Functional_programming&action=history\n",
      "62.175.103.210 is from ES\n",
      "2607:fea8:235f:9730::1232 is from CA\n",
      "2600:6c4a:4d7f:e999:d21:9ad5:a5e4:e20d is from US\n",
      "105.12.5.186 is from ZA\n",
      "134.204.220.36 is from US\n",
      "2607:fea8:2363:8f00:a000:dce0:5c45:2642 is from CA\n",
      "2804:14c:1c2:821c:4970:1293:d1dc:3f00 is from BR\n",
      "23.121.65.56 is from US\n",
      "104.152.255.71 is from US\n",
      "109.125.120.234 is from DE\n",
      "91.243.4.220 is from UA\n",
      "--------------------\n",
      "history url is: http://en.wikipedia.org/w/index.php?title=Structured_programming&action=history\n",
      "83.223.241.43 is from PT\n",
      "2401:4900:c16:df76:13e:4e10:599a:642e is from IN\n",
      "165.82.134.144 is from US\n",
      "2409:4051:291:a148:159a:ee30:ad1b:2179 is from IN\n",
      "2600:8805:a981:4300:65d3:9258:af92:d91c is from US\n",
      "156.199.200.186 is from EG\n",
      "76.8.32.203 is from BM\n",
      "223.196.166.69 is from IN\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'HTTPError' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [5]\u001b[0m, in \u001b[0;36mgetCountry\u001b[0;34m(ipAddress)\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m----> 3\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhttps://ipwho.is/\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mipAddress\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mread()\u001b[38;5;241m.\u001b[39mdecode(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mutf-8\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m HTTPError:\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:214\u001b[0m, in \u001b[0;36murlopen\u001b[0;34m(url, data, timeout, cafile, capath, cadefault, context)\u001b[0m\n\u001b[1;32m    213\u001b[0m     opener \u001b[38;5;241m=\u001b[39m _opener\n\u001b[0;32m--> 214\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mopener\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:517\u001b[0m, in \u001b[0;36mOpenerDirector.open\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m    516\u001b[0m sys\u001b[38;5;241m.\u001b[39maudit(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124murllib.Request\u001b[39m\u001b[38;5;124m'\u001b[39m, req\u001b[38;5;241m.\u001b[39mfull_url, req\u001b[38;5;241m.\u001b[39mdata, req\u001b[38;5;241m.\u001b[39mheaders, req\u001b[38;5;241m.\u001b[39mget_method())\n\u001b[0;32m--> 517\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    519\u001b[0m \u001b[38;5;66;03m# post-process response\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:534\u001b[0m, in \u001b[0;36mOpenerDirector._open\u001b[0;34m(self, req, data)\u001b[0m\n\u001b[1;32m    533\u001b[0m protocol \u001b[38;5;241m=\u001b[39m req\u001b[38;5;241m.\u001b[39mtype\n\u001b[0;32m--> 534\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_chain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_open\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprotocol\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprotocol\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\n\u001b[1;32m    535\u001b[0m \u001b[43m                          \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m_open\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    536\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m result:\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:494\u001b[0m, in \u001b[0;36mOpenerDirector._call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m    493\u001b[0m func \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(handler, meth_name)\n\u001b[0;32m--> 494\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    495\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m result \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:1389\u001b[0m, in \u001b[0;36mHTTPSHandler.https_open\u001b[0;34m(self, req)\u001b[0m\n\u001b[1;32m   1388\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mhttps_open\u001b[39m(\u001b[38;5;28mself\u001b[39m, req):\n\u001b[0;32m-> 1389\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhttp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mHTTPSConnection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1390\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_context\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcheck_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_check_hostname\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/urllib/request.py:1346\u001b[0m, in \u001b[0;36mAbstractHTTPHandler.do_open\u001b[0;34m(self, http_class, req, **http_conn_args)\u001b[0m\n\u001b[1;32m   1345\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1346\u001b[0m     \u001b[43mh\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_method\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mselector\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1347\u001b[0m \u001b[43m              \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhas_header\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mTransfer-encoding\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1348\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err: \u001b[38;5;66;03m# timeout error\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:1285\u001b[0m, in \u001b[0;36mHTTPConnection.request\u001b[0;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[1;32m   1284\u001b[0m \u001b[38;5;124;03m\"\"\"Send a complete request to the server.\"\"\"\u001b[39;00m\n\u001b[0;32m-> 1285\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:1331\u001b[0m, in \u001b[0;36mHTTPConnection._send_request\u001b[0;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[1;32m   1330\u001b[0m     body \u001b[38;5;241m=\u001b[39m _encode(body, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbody\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m-> 1331\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mendheaders\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:1280\u001b[0m, in \u001b[0;36mHTTPConnection.endheaders\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m   1279\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m CannotSendHeader()\n\u001b[0;32m-> 1280\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_output\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:1040\u001b[0m, in \u001b[0;36mHTTPConnection._send_output\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m   1039\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_buffer[:]\n\u001b[0;32m-> 1040\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1042\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m message_body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   1043\u001b[0m \n\u001b[1;32m   1044\u001b[0m     \u001b[38;5;66;03m# create a consistent interface to message_body\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:980\u001b[0m, in \u001b[0;36mHTTPConnection.send\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m    979\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mauto_open:\n\u001b[0;32m--> 980\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    981\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:1454\u001b[0m, in \u001b[0;36mHTTPSConnection.connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1452\u001b[0m     server_hostname \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhost\n\u001b[0;32m-> 1454\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msock \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_context\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrap_socket\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1455\u001b[0m \u001b[43m                                      \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_hostname\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/ssl.py:500\u001b[0m, in \u001b[0;36mSSLContext.wrap_socket\u001b[0;34m(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session)\u001b[0m\n\u001b[1;32m    494\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrap_socket\u001b[39m(\u001b[38;5;28mself\u001b[39m, sock, server_side\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m    495\u001b[0m                 do_handshake_on_connect\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m    496\u001b[0m                 suppress_ragged_eofs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m    497\u001b[0m                 server_hostname\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, session\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[1;32m    498\u001b[0m     \u001b[38;5;66;03m# SSLSocket class handles server_hostname encoding before it calls\u001b[39;00m\n\u001b[1;32m    499\u001b[0m     \u001b[38;5;66;03m# ctx._wrap_socket()\u001b[39;00m\n\u001b[0;32m--> 500\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msslsocket_class\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_create\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    501\u001b[0m \u001b[43m        \u001b[49m\u001b[43msock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    502\u001b[0m \u001b[43m        \u001b[49m\u001b[43mserver_side\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_side\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    503\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdo_handshake_on_connect\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdo_handshake_on_connect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    504\u001b[0m \u001b[43m        \u001b[49m\u001b[43msuppress_ragged_eofs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msuppress_ragged_eofs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    505\u001b[0m \u001b[43m        \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_hostname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    506\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    507\u001b[0m \u001b[43m        \u001b[49m\u001b[43msession\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msession\u001b[49m\n\u001b[1;32m    508\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/ssl.py:1040\u001b[0m, in \u001b[0;36mSSLSocket._create\u001b[0;34m(cls, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, context, session)\u001b[0m\n\u001b[1;32m   1039\u001b[0m             \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdo_handshake_on_connect should not be specified for non-blocking sockets\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1040\u001b[0m         \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_handshake\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1041\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (\u001b[38;5;167;01mOSError\u001b[39;00m, \u001b[38;5;167;01mValueError\u001b[39;00m):\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/ssl.py:1309\u001b[0m, in \u001b[0;36mSSLSocket.do_handshake\u001b[0;34m(self, block)\u001b[0m\n\u001b[1;32m   1308\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msettimeout(\u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m-> 1309\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_handshake\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1310\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [5]\u001b[0m, in \u001b[0;36m<cell line: 11>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     14\u001b[0m     historyIPs \u001b[38;5;241m=\u001b[39m getHistoryIPs(link\u001b[38;5;241m.\u001b[39mattrs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhref\u001b[39m\u001b[38;5;124m\"\u001b[39m])\n\u001b[1;32m     15\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m historyIP \u001b[38;5;129;01min\u001b[39;00m historyIPs:\n\u001b[0;32m---> 16\u001b[0m         \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mhistoryIP\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m is from \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mgetCountry(historyIP)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m     18\u001b[0m newLink \u001b[38;5;241m=\u001b[39m links[random\u001b[38;5;241m.\u001b[39mrandint(\u001b[38;5;241m0\u001b[39m, \u001b[38;5;28mlen\u001b[39m(links)\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m)]\u001b[38;5;241m.\u001b[39mattrs[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhref\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m     19\u001b[0m links \u001b[38;5;241m=\u001b[39m getLinks(newLink)\n",
      "Input \u001b[0;32mIn [5]\u001b[0m, in \u001b[0;36mgetCountry\u001b[0;34m(ipAddress)\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m      3\u001b[0m     response \u001b[38;5;241m=\u001b[39m urlopen(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttps://ipwho.is/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mipAddress\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\u001b[38;5;241m.\u001b[39mread()\u001b[38;5;241m.\u001b[39mdecode(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mutf-8\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m----> 4\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[43mHTTPError\u001b[49m:\n\u001b[1;32m      5\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m      6\u001b[0m responseJson \u001b[38;5;241m=\u001b[39m json\u001b[38;5;241m.\u001b[39mloads(response)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'HTTPError' is not defined"
     ]
    }
   ],
   "source": [
    "def getCountry(ipAddress):\n",
    "    try:\n",
    "        response = urlopen(f'https://ipwho.is/{ipAddress}').read().decode('utf-8')\n",
    "    except HTTPError:\n",
    "        return None\n",
    "    responseJson = json.loads(response)\n",
    "    return responseJson.get('country_code')\n",
    "    \n",
    "links = getLinks('/wiki/Python_(programming_language)')\n",
    "\n",
    "while(len(links) > 0):\n",
    "    for link in links:\n",
    "        print('-'*20) \n",
    "        historyIPs = getHistoryIPs(link.attrs[\"href\"])\n",
    "        for historyIP in historyIPs:\n",
    "            print(f'{historyIP} is from {getCountry(historyIP)}')\n",
    "\n",
    "    newLink = links[random.randint(0, len(links)-1)].attrs['href']\n",
    "    links = getLinks(newLink)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}