micrograd.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math, random\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "41.120000000000005"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def f(x):\n",
    "    return 3*x**2 - 4*x+5\n",
    "\n",
    "f(4.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x2336a6335b0>]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkUUlEQVR4nO3dd3jV5f3/8ec7e0EGCTskQBBkCEhIETe4q4jUhVVxtKi1rtKvpcsuq1ZbR+uoG6zWgaNuVGSIAzCAzEAIOxAgAZIwErLu3x85+KOVEbI+Z7we18VFzuecw+d1rvZ65fY+9+f+mHMOEREJLmFeBxARkeanchcRCUIqdxGRIKRyFxEJQip3EZEgFOF1AIDU1FSXmZnpdQwRkYAyf/78Eudc2sGe84tyz8zMJDc31+sYIiIBxczWH+o5TcuIiAQhlbuISBBSuYuIBCGVu4hIEFK5i4gEoSOWu5k9Z2bbzGzpAcdSzOwTM1vl+zvZd9zM7O9mVmBmi83s+JYMLyIiB9eQkfsk4Jz/OTYR+NQ51wv41PcY4Fygl+/PeOCJ5okpIiJH44jl7pz7DNjxP4cvBCb7fp4MjD7g+Auu3hwgycw6NVPW71hSWMZfpq5A2xaLiPy3xs65d3DOFfl+3gJ08P3cBdh4wOsKfce+w8zGm1mumeUWFxc3KsTCjTt5YuZqctfvbNT7RUSCVZO/UHX1w+ajHjo7555yzmU757LT0g569ewRXTIkneS4SJ6ctaZR7xcRCVaNLfet+6dbfH9v8x3fBKQf8LquvmMtIjYqnKtOyGRa3lYKtu1qqdOIiAScxpb7O8A438/jgLcPOH61b9XMMKDsgOmbFjHuhAyiI8J4+rO1LXkaEZGA0pClkC8DXwG9zazQzK4H7gPONLNVwBm+xwAfAGuAAuBp4CctkvoA7RKiuSS7K28t3MS28sqWPp2ISEA44q6Qzrmxh3hq5EFe64CbmxrqaP3opB68NHcDk75cx53n9Gnt04uI+J2guEI1MzWec/p15MU569m9r8brOCIinguKcgcYf0oPyitrePXrjUd+sYhIkAuach/cLZmczBSe+3wt1bV1XscREfFU0JQ71I/eN5VW8MGSFl2gIyLi94Kq3Ef0aU/PtHj+OWuNtiQQkZAWVOUeFmaMP6UHeUXlfF5Q4nUcERHPBFW5A4we3IW0NtE89Zm2JBCR0BV05R4dEc41wzOZvaqEZZvLvI4jIuKJoCt3gCu/l0F8VDhPa/QuIiEqKMs9MS6Sy3O68e7iIjaVVngdR0Sk1QVluQNcd1J3AJ77XBuKiUjoCdpy75IUy6iBnXl53gZ27qnyOo6ISKsK2nIHuPHUnuytquX5L9d5HUVEpFUFdbn37tiGs/p2YNIXa7WhmIiElKAud4CbT8+ivLKGF+es9zqKiEirCfpyH5iexMm9Unlm9loqq2u9jiMi0iqCvtwBfnJaFiW79/FarrYDFpHQEBLlPqxHCkMyknly1hptBywiISEkyt3M+OnpWWwqreA/Czd5HUdEpMWFRLkDnNY7jb6d2vLEzNXU1mk7YBEJbiFT7mbGzadnsaZkD1OXbvE6johIiwqZcgc4p39HeqTF8+iMAt3MQ0SCWkiVe3iYcdOpPckrKmfGym1exxERaTEhVe5QfzOPLkmxPDpdo3cRCV4hV+6R4WHccGoPFmwoZc6aHV7HERFpESFX7gCXZqeTmhDN4zMLvI4iItIiQrLcYyLD+dHJ3Zm9qoRFG0u9jiMi0uxCstwBrhyWQWJsJP+YvsrrKCIizS5kyz0hOoLrT+rOtLxtLCnUjbRFJLiEbLkDXHtiJomxkTzyab7XUUREmlVIl3ubmEh+fHL96H1xYanXcUREmk1IlzvAuOGZJMVF8vA0zb2LSPBoUrmb2R1mtszMlprZy2YWY2bdzWyumRWY2atmFtVcYVtC/ei9B9NXbOMbrZwRkSDR6HI3sy7ArUC2c64/EA5cDvwFeMg5lwXsBK5vjqAtaf/o/ZFpmnsXkeDQ1GmZCCDWzCKAOKAIGAG87nt+MjC6iedocQnREfz45B7MWFnMwg07vY4jItJkjS5359wm4K/ABupLvQyYD5Q652p8LysEuhzs/WY23sxyzSy3uLi4sTGazbjhmSRr7l1EgkRTpmWSgQuB7kBnIB44p6Hvd8495ZzLds5lp6WlNTZGs0mIjmD8KT2ZlV/MAo3eRSTANWVa5gxgrXOu2DlXDbwJnAgk+aZpALoCAXNfu6tPyCAlPkqjdxEJeE0p9w3AMDOLMzMDRgLLgRnAxb7XjAPeblrE1hMfHcH4U3rwWX4x89dr9C4igaspc+5zqf/idAGwxPdvPQX8AviZmRUA7YBnmyFnq7n6hAzaxUfxsFbOiEgAa9JqGefc75xzfZxz/Z1zVznn9jnn1jjncpxzWc65S5xz+5orbGuIi4rghlN7MHtVCbnrtN+7iASmkL9C9WCuHJZBaoLm3kUkcKncDyIuKoIbTunJ5wUlfK3Ru4gEIJX7IVw5LIO0NtE8MHWl7rUqIgFH5X4IsVHh3Doii3nrdjAz3/uLrEREjobK/TAuG9qN9JRYHpi6kro6jd5FJHCo3A8jKiKMCWf2ZnlROe8vKfI6johIg6ncj2DUwM706diGv328kuraOq/jiIg0iMr9CMLCjP87uzfrtu9lSm6h13FERBpE5d4AI/q0Z0hGMo98mk9lda3XcUREjkjl3gBmxi/O6cPW8n1M/nKd13FERI5I5d5AOd1TOK13Go/PXE1ZRbXXcUREDkvlfhT+7+zelFVU8/Rna7yOIiJyWCr3o9CvcyIXDOzMs5+vpXhXQO2HJiIhRuV+lH525jFU1dbx6HRtKiYi/kvlfpS6p8Zz2dB0/j1vAxt37PU6jojIQancG+HWEb0IM+OhT3RDDxHxTyr3RuiYGMM1J2by1jebWLGl3Os4IiLfoXJvpJtO7UnbmEju+WCF11FERL5D5d5ISXFR3DIii8/yi5mlLYFFxM+o3Jvg6hMyyWgXxz3v51GrLYFFxI+o3JsgKiKMief0YeXWXbyWu9HrOCIi31K5N9E5/TsyNDOZv32cz+59NV7HEREBVO5NZmb8+vt9Kdm9jydnrfY6jogIoHJvFoPSkxg1sDNPz15DUVmF13FERFTuzeX/zu5NnYMHPlrpdRQREZV7c0lPiePaEzN5c8Emlm4q8zqOiIQ4lXszuvn0LFLio7j7/eU4p6WRIuIdlXszahsTye1n9GLOmh1My9vmdRwRCWEq92Y2NqcbPdLiuffDPKpr67yOIyIhSuXezCLDw/jVuceypngPL8/b4HUcEQlRKvcWMPLY9pzQox0PfZJP2V7db1VEWp/KvQWYGb85/1jKKqp5aJr2fBeR1tekcjezJDN73cxWmFmemZ1gZilm9omZrfL9ndxcYQNJv86JXPG9bvxrznrt+S4ira6pI/dHgKnOuT7AQCAPmAh86pzrBXzqexySJpzZmzYxEfzu7WVaGikirarR5W5micApwLMAzrkq51wpcCEw2feyycDopkUMXMnxUfzf2b2Zu3YH7y0u8jqOiISQpozcuwPFwPNmttDMnjGzeKCDc25/k20BOhzszWY23sxyzSy3uDh4b3Zx+dBu9O/Slns+yGOPdo0UkVbSlHKPAI4HnnDODQb28D9TMK5+LuKg8xHOuaecc9nOuey0tLQmxPBv4WHGH0b1o6isksdmFHgdR0RCRFPKvRAodM7N9T1+nfqy32pmnQB8f4f8pZpDMlIYM7gLz8xey7qSPV7HEZEQ0Ohyd85tATaaWW/foZHAcuAdYJzv2Djg7SYlDBITz+1DVEQYf3xvuddRRCQENHW1zC3AS2a2GBgE3APcB5xpZquAM3yPQ177tjHcOjKL6Su2MX3FVq/jiEiQi2jKm51z3wDZB3lqZFP+3WB1zfDuvPr1Rv7w7nKG90wlJjLc60giEqR0hWoriooI4/ej+rF++16e/Xyt13FEJIip3FvZyb3SOLtfBx6dXsDmUt2ST0RahsrdA7/5fl/qnOPPH+R5HUVEgpTK3QPpKXH85LQs3l9cxMyVIb9SVERagMrdIzee1oMeafH89u2lVFTVeh1HRIKMyt0j0RHh3HPRADbuqOCRT1d5HUdEgozK3UPDerTjkiFdeWb2Gm0LLBJinHM8Mm0VRWUts7BC5e6xX513LG1jI/nlm0uoq9O2wCKh4s0Fm3hoWj7T8lrmezeVu8eS46P47fnHsnBDKS/NXe91HBFpBSW79/Gn95czJCOZH+Z0a5FzqNz9wOhBXTgxqx33T13J1vJKr+OISAv747vL2buvlvvGDCAszFrkHCp3P2Bm/Hn0APbV1vGHd5d5HUdEWtD0FVt5Z9Fmbj49i14d2rTYeVTufiIzNZ5bR2TxwZItfJqnjcVEgtHufTX85q2lHNMhgZtO69mi51K5+5Hxp/TkmA4J3PX2Mt21SSQI/fWjlRSVV3LvmOOIimjZ+lW5+5GoiDDuuWgAm0oreHhavtdxRKQZzV+/k8lfrWPcCZkMyUhu8fOp3P1MdmYKY3O68dwX61i6qczrOCLSDKpq6pj4xmI6tY3h52f3PvIbmoHK3Q9NPKcPyXFR/OKNxVTX1nkdR0Sa6ImZq1m1bTd3X9SfhOgm3UajwVTufigxLpK7R/dj2eZynpi52us4ItIEq7bu4tEZqxg1sDMj+nRotfOq3P3UOf07MWpgZ/4xfRV5RdqaQCQQ1dU5Jr65hPjoCO66oG+rnlvl7sf+MKofibFRTHhtkaZnRALQS3PXM3/9Tn77/b6kJkS36rlV7n4sOT6KP1/Un+VF5Tw2o8DrOCJyFDbu2Mt9H67g5F6pjDm+S6ufX+Xu587u15ELB3Xm0ekFLNus1TMigaCuzvHzKYswM+4dMwCzltli4HBU7gHg9xf0Iykuip9PWUxVjaZnRPzdc1+sZe7aHdx1QV+6Jsd5kkHlHgCS46O456L+5Gl6RsTvFWzbxf0freSMY9tzyZCunuVQuQeIs/p1ZPSgzjw2o0AXN4n4qeraOn722iLio8K5x6PpmP1U7gHk96P6kRwfxc+nLNL0jIgfenzGahYXlvHniwbQvk2Mp1lU7gEkKS6Key8awIotu3h0uu67KuJPlm4q4x/TV3HhoM6cN6CT13FU7oHmjL4dGDO4C4/NXM2SQk3PiPiDyupafvbaN7RLiOKPo/p7HQdQuQek313Qj9SEKG5/dSEVVbVexxEJeQ99kk/+1t385QfHkRgX6XUcQOUekBLjInnw0kGsKdnDn95f7nUckZD29bodPDV7DVd8rxun9W7vdZxvqdwD1IlZqYw/uQf/nruBqUu3eB1HJCTt2VfDhNcW0TU5ll+fd6zXcf6Lyj2ATTirN/27tGXim4vZUqYba4u0trvfz2Pjzr387ZJBxLfSVr4NpXIPYFERYTxy+WD2VdcxYco31NU5ryOJhIwPlhTx8rwNjD+lBzndU7yO8x1NLnczCzezhWb2nu9xdzOba2YFZvaqmUU1PaYcSs+0BH53QV++KNjO07PXeB1HJCRs3LGXX7yxmEHpSfz8rNa5s9LRao6R+21A3gGP/wI85JzLAnYC1zfDOeQwLhuazrn9O/LXj1fq6lWRFlZdW8dtrywEB/8YO5jIcP+cAGlSKjPrCnwfeMb32IARwOu+l0wGRjflHHJk+3eeS02I5taXF7K3qsbrSCJB66FP8lmwoZR7xgwgPcWbTcEaoqm/ch4G7gT2XwvfDih1zu1vl0LgoBsZm9l4M8s1s9zi4uImxpCkuCgevHQQa7fv4Y/vanmkSEv4fFUJT8xazdicdC4Y2NnrOIfV6HI3s/OBbc65+Y15v3PuKedctnMuOy0trbEx5AAn9GzHTaf25JWvN/LhkiKv44gEleJd+7jjtW/ISkvgrvP7eR3niJoycj8RGGVm64BXqJ+OeQRIMrP9a4K6ApualFCOyh1nHsPArolMfHMJm0srvI4jEhTq6hwTpiyivKKaf1wxmNiocK8jHVGjy90590vnXFfnXCZwOTDdOfdDYAZwse9l44C3m5xSGiwyvH55ZG2d4+Z/L9DukSLN4OnZa/gsv5i7LuhLn45tvY7TIC3xNe8vgJ+ZWQH1c/DPtsA55DAyU+O5/+LjWLihlHs+yDvyG0TkkL7ZWMoDH63k3P4duSKnm9dxGqxZLqlyzs0EZvp+XgPkNMe/K4133oBOXH9Sd579fC3HZyQzys+//BHxR+WV1dzy8gI6tI3hvjHHeXrzjaPlnws0pVlMPLcP2RnJTHxjMQXbdnkdRySgOOe4c8piNpdW8vexg/1mt8eGUrkHscjwMB694njiosK58cUF7Nmn9e8iDfXErNVMXbaFX57bhyEZyV7HOWoq9yDXMTGGv18+mDXFu/nlm0twTvvPiBzJZ/nF/PWjlVwwsDPXn9Td6ziNonIPAcOzUplwVm/eWbSZF75a73UcEb+2ccdebn1lIb3at+EvP/D2JtdNoXIPETed2pORfdpz9/vLWbBhp9dxRPxSRVUtN/xrPnV1jievGkJclH9t43s0VO4hIizMePDSQXRMjOGnLy1gx54qryOJ+BXnHL9+awl5W8p55PLBZKbGex2pSVTuISQxLpInfjiEkj1V3PbKQmq1/7vIt174aj1vLtzE7SOP4fQ+/nO7vMZSuYeY/l0S+eOofsxeVcK9usBJBKi/D+qf3lvOyD7tuWVEltdxmkXgTihJo12e0428onKe+XwtvTokcNnQwLnqTqS5bS2v5CcvLSA9JY4HLxtEWFhgfoH6vzRyD1G/Pb8vJ/dK5Tf/WcrcNdu9jiPiiaqaOn7yUv01IP+8cgiJsYF1odLhqNxDVITvAqf0lDhufHE+G7bv9TqSSKva/wXq/PU7uf/i4+jdsY3XkZqVyj2EJcZG8uy4odQ5uH7y1+yqrPY6kkireXzmaqbML+TWkb04/7jg23tJ5R7iuqfG88QPj2dNyR5ufVkraCQ0vLtoMw98tJILB3XmjjN6eR2nRajcheFZqfxhVD9mrCzWChoJevPX72TClEUMzUzmLz8IrJ0ej4ZWywgAVw7LYNXWXVpBI0Ftw/a9jH8hl06JMTx5VTYxkf5/R6XG0shdvqUVNBLMyvZWc+2kedTUOZ6/Zigp8VFeR2pRKnf5VkR4GI+OrV9Bc8OL87UHvASNqpo6bnppPht27OXJq4bQIy3B60gtTuUu/yUxLpLnrxlKRFgYVz87j6Iy3WRbAptzjt/8Zwlfrt7OfWOOY1iPdl5HahUqd/mOjHbxTLp2KOWVNVz97DxK92qTMQlcT8xazWu5hdw6IosfDOnqdZxWo3KXg+rfJZGnrh7C+u17uX5yLhVVtV5HEjlqr88v5P6pviWPZx7jdZxWpXKXQxreM5WHLx/Egg07+em/F1BdW+d1JJEGm7p0C3e+voiTslK5/+LgXfJ4KCp3OazzBnTijxf259MV23SbPgkYn68q4daXFzIwPYknrxpCdETwLnk8FK1zlyO6algGxbv28fdPV5GaEM3Ec/t4HUnkkOav38n4f+XSIy2eSdfkEB8dmjUXmp9ajtodZ/SiZPc+/jlrNakJUfzo5B5eRxL5jryicq59fh7t20TzwvU5JMYFzy6PR0vlLg1iZvzpwv7s2F3F3e/n0S4hiosGh87KA/F/60r2cNWz84iLiuBf13+P9m1ivI7kKc25S4OFhxkPXz6IE3q0Y8Jri3hn0WavI4kAUFRWwQ+fmUudc7z4oxzSU+K8juQ5lbsclZjIcJ4Zl012Zgq3v7KQd1Xw4rHtu/dx5TNzKauoZvK1OWS1D6592RtL5S5HLT46guevGUp2Rgq3qeDFQzv3VDHu+XkU7qzg2XHZDOia6HUkv6Fyl0aJj47g+WvrC/72V7/hvcUqeGldJbv3MfbpOeRv3c0/rxrC90JkW4GGUrlLo+0v+OO7JXHbK9/w/uIiryNJiNhWXsnlT81h3fY9PDsum9N7t/c6kt9RuUuTxEdHMOnaHI7vlsStryxUwUuL21xawaVPfsXm0gomXZvDyb3SvI7kl1Tu0mT1I/gcBqer4KVlbdyxl0uf/Irtu6v41/U5IbPDY2M0utzNLN3MZpjZcjNbZma3+Y6nmNknZrbK93dy88UVf5UQHcGk6/5/wWuZpDS3tSV7uOzJr9hVWcNLP/4eQzJSvI7k15oycq8BJjjn+gLDgJvNrC8wEfjUOdcL+NT3WELA/oIfkpHMba8sZNIXa72OJEGiYNsuLnvyKypr6nj5x8M4rmuS15H8XqPL3TlX5Jxb4Pt5F5AHdAEuBCb7XjYZGN3EjBJAEqIjeOG6HM48tgO/f3c5909doc3GpEnyisq57Mk5OODV8cPo27mt15ECQrPMuZtZJjAYmAt0cM7tn3TdAnQ4xHvGm1mumeUWFxc3RwzxEzGR4Txx5RDG5nTj8ZmrufP1xdRou2BphC8LSrj0ya+IDA/j1fHD6NVBFyg1VJPL3cwSgDeA251z5Qc+5+qHbAcdtjnnnnLOZTvnstPS9G13sAkPM+65qD+3jezFlPmF3PCv+brhhxyVN+YXMu75eXRKjOGNnwwPifueNqcmlbuZRVJf7C855970Hd5qZp18z3cCtjUtogQqM+OOM4/h7tH9mbFyGz98Zg479+iWfXJ4zjkembaKCVMWMTQzhSk3DqdLUqzXsQJOU1bLGPAskOece/CAp94Bxvl+Hge83fh4EgyuHJbB4z88nqWby7nEtz5Z5GCqa+u48/XFPDQtnzHHd2HStTkkxobutr1N0ZSR+4nAVcAIM/vG9+c84D7gTDNbBZzheywh7pz+nXjhuhy2llUy5vEvWba5zOtI4mfKK6u59vmvmTK/kFtH9uJvlwwkKkKX4jSW+cNKhuzsbJebm+t1DGkFeUXlXDfpa3bureL+iwcyamBnryOJH9hcWsF1k76mYNtu7hkzgEuz072OFBDMbL5zLvtgz+nXorSqYzu15Z2fnsSALonc+vJC7v0wj9o67wcY4p2lm8q46PEv2LSzfjsBFXvzULlLq0trE81LPxrGlcO68eSsNVw76WvK9lZ7HUs88OrXGxjzxJeEmTHlphM4qVeq15GChspdPBEVEcbdowdw75gBfLW6hFGPfU7+1l1ex5JWUlldy52vL+IXbywhJzOF9245iT4ddXFSc1K5i6fG5nTjlfHD2FtVy+jHvmDqUm06Fuw2bN/LmMe/5LXcQm4ZkcXk63JolxDtdaygo3IXzw3JqB+5HdOhDTe+uIC/fbxS8/BBatryrZz/j9lsKq3guWuymXBWb8LDzOtYQUnlLn6hQ9sYXr1hGJdmd+Uf0wu4/Kmv2Lhjr9expJnU1NZx/9QV/OiFXLq1i+O9W05iRJ+D7kwizUTlLn4jOiKcv/zgOB68dCB5Rbs475HZvLWwUBuPBbht5ZVc/dw8Hp+5mrE56bx+43DSU+K8jhX0IrwOIHIgM2PM8V0ZmpnCz177hjteXcT0FcXcPbq/rlQMMM453lm0mbveXkZldS0PXHwcl2iZY6tRuYtfSk+J45XxJ/DEzAIenraK+et28OBlg3TnnQBRsnsfv3lrKVOXbWFwtyT+eslAemrjr1alaRnxW+Fhxk9H9OKNm4YTHRnO2KfncN+HK6iq0fbB/uyDJUWc9dBnTF+xjYnn9uH1G4er2D2gkbv4vYHpSbx3y0nc/f5y/jlrNbPyi/nzRf05vpvu4OhPdu6p4rdvL+W9xUUc1zWRv10yUPuve0h7y0hA+XjZFu56exlbyisZm5POnWf3ITk+yutYIe/jZVv41VtLKauo4raRvbjx1J5EhGtioKUdbm8ZjdwloJzVryPDs1J5ZFo+z32xjqlLtzDx3D5cMiSdMK2XbnWri3fz5/fzmL5iG307teVf1+dwbCddaeoPNHKXgLViSzm/eWspuet3cny3JO4ePUD312wlZRXV/P3TVUz+ch2xkeHcMjKLa4Z31xa9rexwI3eVuwS0ujrHGwsKuffDFZTurWLc8EzuOPMY2sZo2WRLqK1zvDxvAw9+ks/OvVVclp3OhLN6k9ZG2wd4QdMyErTCwoxLstM5s28HHvhoJZO+XMd/Fm7ihlN7cvUJGcRF6f/izeXLghL++N5yVmzZRU73FO46vy/9uyR6HUsOQSN3CSqLC0v568f5fJZfTGpCFDee2pMrh2UQExnudbSAtaSwjL9PX8Uny7fSNTmWX513LOf270j9nTbFS5qWkZCTu24HD03L54uC7bRvE83Np2dxeU460REq+YZwzjFv7Q4enVHA7FUltImJ4IZTevCjk3voF6UfUblLyJqzZjsPfpzPvHU76JQYw09HZHHxkK4q+UNwzjFzZTGPzSggd/1OUhOiuP6kHlw5rBtt9D2G31G5S0hzzvFFwXb+9slKFm4opV18FJcNTWdsTjdtYOVTW+f4cGkRj81YTV5ROV2SYrnh1B5cmp2ukbofU7mLUF/ynxeUMPnL9UxfsRUHnN67PVcO68apx7QPyX3FN+7Yy5T5hbwxv5BNpRX0SIvnJ6dlceGgzkTqIiS/p3IX+R+bSit4Zd4GXp63kZLd++iaHMsV3+vGpdnppAb5XYH2VtXwwZItvD5/I3PW7MAMTspK5YqcbpzVr2NI/pILVCp3kUOoqqnj4+VbeHHOeuas2UFkuDG8Zypn9evAmcd2oH3bGK8jNgvnHLnrdzIldyPvLy5iT1Utme3iuHhIV8Yc35XOSbFeR5RGULmLNEDBtl28+vVGPlq2lQ2+u0AN7pbEWX07cla/DgG3s+HOPVXMLihh1spiZq8qZtuufcRFhfP9AZ24JDudoZnJWs4Y4FTuIkfBOUf+1t18vGwLHy/fypJNZQD0TIvnjGM7MCQjmUHdkmjfxr9G9TW1dSwqLGVWfgmz8otZXFiKc5AYG8nJvVIZ0ac9Z/frSHy0LuwKFip3kSbYVFrBtOVb+Xj5Fuau2UGN7+bdXZJiGZieyKD0JAalJ9O/S9tWuyK2srqWgm27ySsqJ69oFyu2lLN0UxnllTWEGQxKT+KUY9I49Zg0juuapHn0IKVyF2kmFVW1LNtcxjcbS7/9U7izAqi/uUj31Hi6JMXSOSmGzomxdEqKpXNiDJ2TYumYGNOgZYXOOcora9ixp4ode/axfXcVO/ZUUbxrH/nbdrOiqJw1JXuo9f2SiYkMo3fHtvTt1JaTslI5MasdSXHaBjkUaG8ZkWYSGxVOdmYK2Zkp3x4r3rWPRRtLWVRYysotuygqq2TZ5jJKdld99/2R4USEGRHhRnhYGBFhRvj+x2bs3lfDzr1VVNcefNDVNTmWYzu15dz+HenTqS19OrYho128RubyHSp3kSZKaxPNGX07cEbfDv91vLK6li1llWwuq2BzaSVFpRWUV1ZTU+eorXNU1zpq6+q+fVxT50iIiiAlIYp28VGk+P60i4/+9pguKJKGUrmLtJCYyHAyU+PJTI33OoqEIF2CJiIShFTuIiJBqEXK3czOMbOVZlZgZhNb4hwiInJozV7uZhYOPAacC/QFxppZ3+Y+j4iIHFpLjNxzgALn3BrnXBXwCnBhC5xHREQOoSXKvQuw8YDHhb5j/8XMxptZrpnlFhcXt0AMEZHQ5dkXqs65p5xz2c657LS0NK9iiIgEpZYo901A+gGPu/qOiYhIK2n2vWXMLALIB0ZSX+pfA1c455Yd5j3FwPpmDdJ6UoESr0N4QJ879ITqZ/fnz53hnDvo1EezX6HqnKsxs58CHwHhwHOHK3bfewJ2XsbMcg+1cU8w0+cOPaH62QP1c7fI9gPOuQ+AD1ri3xYRkSPTFaoiIkFI5d50T3kdwCP63KEnVD97QH5uv7hZh4iINC+N3EVEgpDKXUQkCKncm4mZTTAzZ2apXmdpLWb2gJmtMLPFZvaWmSV5naklheJup2aWbmYzzGy5mS0zs9u8ztTazCzczBaa2XteZzkaKvdmYGbpwFnABq+ztLJPgP7OueOov3Dtlx7naTEhvNtpDTDBOdcXGAbcHCKf+0C3AXlehzhaKvfm8RBwJxBS30475z52ztX4Hs6hfquJYBWSu50654qccwt8P++ivuS+sxFgsDKzrsD3gWe8znK0VO5NZGYXApucc4u8zuKx64APvQ7Rghq022kwM7NMYDAw1+Morelh6gdudR7nOGq6QXYDmNk0oONBnvo18Cvqp2SC0uE+u3Pubd9rfk39f76/1JrZpPWYWQLwBnC7c67c6zytwczOB7Y55+ab2WkexzlqKvcGcM6dcbDjZjYA6A4sMjOon5ZYYGY5zrktrRixxRzqs+9nZtcA5wMjXXBfNBGyu52aWST1xf6Sc+5Nr/O0ohOBUWZ2HhADtDWzF51zV3qcq0F0EVMzMrN1QLZzzl93kGtWZnYO8CBwqnMuqO+40pjdToOB1Y9aJgM7nHO3exzHM76R+8+dc+d7HKXBNOcuTfEo0Ab4xMy+MbN/eh2opfi+ON6/22ke8FqwF7vPicBVwAjf/8bf+Eay4uc0chcRCUIauYuIBCGVu4hIEFK5i4gEIZW7iEgQUrmLiAQhlbuISBBSuYuIBKH/BxzoKNU5ZGCzAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "xs = np.arange(-5, 5, 0.25)\n",
    "ys = f(xs)\n",
    "plt.plot(xs, ys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0029999999995311555"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# derivative at 2/3\n",
    "\n",
    "h = 0.001\n",
    "x = 2/3\n",
    "(f(x+h) - f(x))/h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x23369ffe820>]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD4CAYAAAAJmJb0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAAsTAAALEwEAmpwYAAAf8ElEQVR4nO3deXhU9f328feXnQQIS1gCSQg7BAIKAUTcocqmiKi1Wje0qI+22l8rCYKKIgriUltXrPpgpW4JCLLIouKKiqhMFhIIe0JIgEASyJ75Pn8kvz60BQVmJmdmcr+ui4uZc4b53nOF3BzOnPnEWGsREZHg1MDpACIi4jsqeRGRIKaSFxEJYip5EZEgppIXEQlijZwOcLzw8HAbExPjdAwRkYCyadOmg9ba9ifa51clHxMTw/fff+90DBGRgGKM2X2yfTpdIyISxFTyIiJBTCUvIhLEVPIiIkFMJS8iEsRU8iIiQUwlLyISxFTyIiIOqqx28+L6LDbvPeKT5/erD0OJiNQnqTmFJCS7SNtXxJ0XVjEoqrXX11DJi4jUsbLKav768TZe+XwHbUKa8NINgxkbF+GTtVTyIiJ16PtdBUxLdrHjwDGuGRLJzPGxhIU09tl6KnkRkTpwtLyK+R9l8OY3u+kc1pw3pwzjgt4nnCnmVSp5EREf+2zrAR5YnMK+wlJuHhHD/Zf1IbRp3dSvSl5ExEeOlFTw6PJ0Fv+QQ4/2obx/xwjiY9rWaQaVvIiIl1lrWZW6n4eWpnKkpJJ7Lu7JPZf0pFnjhnWeRSUvIuJF+UVlPLg0ldVpeQzo0oqFU4bRv3OYY3lU8iIiXmCt5f1N2Ty2PJ3yKjeJY/ty+3ndaNTQ2c+cquRFRDy0t6CE6YtT+DLrIMNi2jJ3chzd27dwOhagkhcROWPVbsubG3bx5EeZNDAw+8oB3DAsmgYNjNPR/kUlLyJyBrLyi5mW5OKHPUe4qE975kyKo0vr5k7H+i8el7wxJgp4E+gIWGCBtfY5Y0xb4F0gBtgFXGutPezpeiIiTqqsdvPKZ9v568dZhDZtyLO/HsSVZ3XBGP85ej+eN47kq4A/WWt/MMa0BDYZY9YCtwAfW2vnGmMSgUQgwQvriYg4IiW7kPuTNpOxv5gJAyOYdUV/wls0dTrWz/K45K21uUBu7e1iY8wWoAswEbio9mELgfWo5EUkAJVVVvOXddt49YsdtAttwoIbh3Bp/05OxzolXj0nb4yJAc4GvgU61v4DALCfmtM5J/ozU4GpANHR0d6MIyLisW93HCJxcQo7Dx7juqFRTB/Xj7Dmvhso5m1eK3ljTAsgGbjPWlt0/Pkpa601xtgT/Tlr7QJgAUB8fPwJHyMiUteKyyqZ91EGb32zh6i2zVl0+3BG9gx3OtZp80rJG2MaU1Pwi6y1i2s35xljIqy1ucaYCCDfG2uJiPjapxn5zFiSQm5RGbed140/XdqbkCaBeTGiN66uMcBrwBZr7TPH7VoG3AzMrf19qadriYj4UsGxCmYvT2fJjzn06tCC5LvOZXB0G6djecQb/zSNBG4EUowxP9Vue4Cacn/PGHMbsBu41gtriYh4nbWW5a5cZi1Lo7C0kj+M6sXdF/egaaO6Hyjmbd64uuZL4GQXiI7y9PlFRHwpr6iMGUtSWbclj4GRYbx1+3D6RbRyOpbXBOZJJhERD1lreXfjXuas3EJFlZvpY/tymx8MFPM2lbyI1Dt7DpWQuNjF19sPMbxbW+ZNHkhMeKjTsXxCJS8i9Ua12/LGVzt5ak0mjRo04PFJcVw3NMqvBop5m0peROqFrXk1A8V+2nuES/p2YM6kAUSE+d9AMW9TyYtIUKuocvPS+u08/+k2WjZrzHPXncUVgzr77UAxb1PJi0jQ2rz3CNOSXGTmFTPxrM48NCGWdn4+UMzbVPIiEnRKK6p5Zm0mr325kw4tm/H3m+IZHXvC8VlBTyUvIkFlw/ZDJC52sftQCdcPjyZxbF9aNQucgWLeppIXkaBQVFbJEyszePu7PXRtF8I/fzecc3sE3kAxb1PJi0jA+3hLHjOWpJJfXMbUC7rzx9G9ad4k8EcSeINKXkQC1qGj5TzyYTrLNu+jb6eWvHLjEAZFtXY6ll9RyYtIwLHWsmzzPh75MJ3iskr+OLo3d13UgyaNgmskgTeo5EUkoOQWljJzSSofZ+RzVlRrnrx6IL07tnQ6lt9SyYtIQHC7Le9s3MsTK7dQ6XYzc3w/bh3ZjYZBPJLAG1TyIuL3dh08RuJiF9/sKODcHu2Ye9VAotuFOB0rIKjkRcRvVVW7eeOrXTy9NpPGDRswb3Ic18ZH1ZuRBN6gkhcRv5Sxv4iEJBebswsZ3a8jj105gE5hzZyOFXBU8iLiV8qrqnnh0+28+GkWYc0b8/z1ZzM+LkJH72dIJS8ifuPHPYdJSHaxNe8ok87uwkMTYmkT2sTpWAFNJS8ijiupqOLpNVt5/audRLRqxhu3DOXivh2cjhUUVPIi4qivsg6SuNjF3oJSfntONAlj+tKyHg8U8zaVvIg4orC0kidWbuGdjXvpFh7Ku1PPYXj3dk7HCjoqeRGpc2vS9jPzg1QOHavgzgt7cN/oXjRrrIFivqCSF5E6c/BoObOWpbHclUu/iFa8dvNQ4iLDnI4V1FTyIuJz1lo++CmHRz5Mp6S8mj9f2ps7LuxB44YaKOZrXil5Y8zrwAQg31o7oHZbW+BdIAbYBVxrrT3sjfVEJHDkHCllxpIU1mceYHB0a+ZNHkgvDRSrM976Z/T/AmP+Y1si8LG1thfwce19Eakn3G7LP77ZzaXPfMa3Owp4+PJY3r/zXBV8HfPKkby19nNjTMx/bJ4IXFR7eyGwHkjwxnoi4t92HDhKYnIK3+0q4Lye4TxxVRxRbTVQzAm+PCff0VqbW3t7P3DCH5VujJkKTAWIjo72YRwR8bWqajd//3Inz67dStNGDXjy6oFcMyRSIwkcVCdvvFprrTHGnmTfAmABQHx8/AkfIyL+L31fEQnJLlJyCrmsf0dmTxxAh1YaKOY0X5Z8njEmwlqba4yJAPJ9uJaIOKS8qprnP8nipfXbaR3SmBeuH8y4uE46evcTviz5ZcDNwNza35f6cC0RccCm3QUkJKeQlX+UyYMjmTm+nwaK+RlvXUL5NjVvsoYbY7KBh6kp9/eMMbcBu4FrvbGWiDjvWHkV81dnsnDDLjqHNWfhlGFc2Lu907HkBLx1dc1vTrJrlDeeX0T8xxfbDjB9cQrZh0u5eURX7h/TlxZN9blKf6WvjIicksKSSh5bkc77m7Lp3j6U9+8cwdCYtk7Hkl+gkheRX/RRai4PLk2j4FgFd13Ug3tHaaBYoFDJi8hJ5ReX8fDSNFal7ic2ohVv3DKUAV00UCyQqORF5L9Ya0n+IYfZy9Mprazm/sv6MPWC7hooFoBU8iLyb7IPl/DAklQ+33qAIV3bMG/yQHp2aOF0LDlDKnkRAf7/QLF5H2UA8MgV/bnxnK40aKAPNQUylbyIkJV/lMRkF9/vPsz5vWoGikW20UCxYKCSF6nHKqvdLPh8B8+t20bzJg156ppBTB7cRSMJgohKXqSeSs0pZFqSi/TcIsbFdWLWFf3p0FIDxYKNSl6knimrrOa5j7ex4PMdtA1twsu/HcyYARFOxxIfUcmL1CMbdxWQkORix8FjXDMkkpnjYwkLaex0LPEhlbxIPXC0vIonP8rgzQ276dK6Of+4bRjn99JAsfpAJS8S5NZn5jNjSSr7Cku5dWQMf760D6EaKFZv6CstEqQOH6tg9op0Fv+QQ88OLUi681yGdG3jdCypYyp5kSBjrWVV6n4eWprKkZJKfn9JT+65pCdNG2mgWH2kkhcJIvlFZTy4NJXVaXnEdQnjzSnDie3cyulY4iCVvEgQsNby/vfZPLYinbIqN4lj+3L7ed1opIFi9Z5KXiTA7S0oYfriFL7MOsiwmLbMnRxH9/YaKCY1VPIiAarabVn49S7mr86kgYHZVw7ghmHRGigm/0YlLxKAtuUVk5Ds4oc9R7ioT3vmTIqjS+vmTscSP6SSFwkgFVVuXvlsO3/7JIuQpg159teDuPIsDRSTk1PJiwQIV/YRpiW5yNhfzPiBETxyRX/CWzR1Opb4OZW8iJ8rq6zm2bVbefWLHYS3aMorNw7hsv6dnI4lAUIlL+LHvtlxiMRkF7sOlXDd0Cimj+tHWHMNFJNTp5IX8UPFZZXMXZXBom/3EN02hEW3D2dkz3CnY0kA8nnJG2PGAM8BDYG/W2vn+npNkUD2aUY+DyxJIa+ojNvP68b/XNqbkCY6HpMz49O/OcaYhsALwK+AbGCjMWaZtTbdl+uKBKKCYxU8+mEaH/y0j14dWvDiXedydrQGiolnfH14MAzIstbuADDGvANMBFTyIrWstSx35TJrWRqFpZXcO6oX/+fiHhooJl7h65LvAuw97n42MPz4BxhjpgJTAaKjo30cR8S/5BWVMWNJKuu25DEwMoxFvxtO304aKCbe4/iJPmvtAmABQHx8vHU4jkidsNby7sa9zFm5hYoqNzPG9ePWkTEaKCZe5+uSzwGijrsfWbtNpN7ac6iExMUuvt5+iOHd2jJv8kBiwkOdjiVBytclvxHoZYzpRk25Xwdc7+M1RfxStdvyxlc7eWpNJo0aNODxSXFcNzRKA8XEp3xa8tbaKmPMPcBqai6hfN1am+bLNUX80da8YqYlufhp7xEu6duBOZMGEBGmgWLiez4/J2+tXQms9PU6Iv6oosrNS+u38/yn22jZrDHPXXcWVwzqrIFiUmccf+NVJFht3lszUCwzr5iJZ3XmoQmxtNNAMaljKnkRLyutqOaZtZm89uVOOrRsxms3xzOqX0enY0k9pZIX8aKvtx8kMTmFPQUlXD88msSxfWnVTAPFxDkqeREvKCqr5ImVGbz93R66tgvh7d+dw4ge7ZyOJaKSF/HUuvQ8ZnyQwoHicqZe0J0/ju5N8yYaSSD+QSUvcoYOHS3nkQ/TWbZ5H307tWTBjfEMimrtdCyRf6OSFzlN1lqWbd7HrGVpHC2v4o+je3PXRT1o0kgjCcT/qORFTkNuYSkzl6TycUY+Z0W15smrB9K7Y0unY4mclEpe5BS43Za3N+7hiZUZVLstM8f349aR3WiokQTi51TyIr9g58FjJCa7+HZnASN7tuOJSQOJbhfidCyRU6KSFzmJqmo3r3+1k6fXbKVJowbMmxzHtfFRGkkgAUUlL3ICW3KLSEh24cou5FexHXnsygF0bNXM6Vgip00lL3Kc8qpqXvgkixfXb6d1SGNeuH4w4+I66ehdApZKXqTWD3sOk5DkYlv+Ua46uwsPToilTWgTp2OJeEQlL/VeSUUVT63eyhtf7ySiVTPeuHUoF/fp4HQsEa9QyUu99lXWQRIXu9hbUMqN53Rl2pg+tNRAMQkiKnmplwpLK3l8xRbe/X4v3cJDeXfqOQzvroFiEnxU8lLvrEnbz8wPUjl0rII7L+zBfaN70ayxBopJcFLJS71xoLicWR+mscKVS7+IVrx281DiIsOcjiXiUyp5CXrWWpb8mMOjy9MpKa/m/sv6MPWC7jRuqIFiEvxU8hLUco6UMmNJCuszDzA4umagWM8OGigm9YdKXoKS221Z9O1u5q7KwAKzLo/lxhExGigm9Y5KXoLO9gNHSUx2sXHXYc7vFc7jk+KIaquBYlI/qeQlaFRVu1nwxQ7+sm4bzRo14MmrB3LNkEiNJJB6TSUvQSFtXyEJyS5Sc4oY078Tj17Znw4tNVBMxKPLC4wx1xhj0owxbmNM/H/sm26MyTLGZBpjLvMspsiJlVVWM391Blc8/xX7C8t56YbBvHzjEBW8SC1Pj+RTgauAV47faIyJBa4D+gOdgXXGmN7W2moP1xP5l027C5iW5GL7gWNMHhzJgxP60TpEA8VEjudRyVtrtwAnOuc5EXjHWlsO7DTGZAHDgA2erCcCcKy8ivmrM1m4YRedw5qzcMowLuzd3ulYIn7JV+fkuwDfHHc/u3bbfzHGTAWmAkRHR/sojgSLz7ceYPriFPYVlnLTOV25f0xfWjTVW0siJ/OL3x3GmHVApxPsmmGtXeppAGvtAmABQHx8vPX0+SQ4HSmp4LEVW0jalE339qG8f8cI4mPaOh1LxO/9Yslba0efwfPmAFHH3Y+s3SZy2lal5PLg0jQOl1Rw98U9+P0lGigmcqp89f/cZcA/jTHPUPPGay/gOx+tJUEqv7iMh5emsSp1P/07t2LhlKH076yBYiKnw6OSN8ZMAv4GtAdWGGN+stZeZq1NM8a8B6QDVcDdurJGTpW1lqRN2Ty2YgulldUkjOnL7ed300AxkTPg6dU1S4AlJ9k3B5jjyfNL/bO3oIQHlqTwxbaDDI1pw9zJA+nRvoXTsUQCli5LEL/gdlve3LCLJ1dnYoDZE/tzw/CuNNBAMRGPqOTFcVn5NQPFvt99mAt7t2fOpAFEttFAMRFvUMmLYyqr3Sz4fAfPrdtGSNOGPHPtICad3UUDxUS8SCUvjkjNKWRakov03CLGx0Uw64r+tG/Z1OlYIkFHJS91qqyymuc+3saCz3fQNrQJL/92CGMGnOizdiLiDSp5qTMbdxWQkORix8FjXBsfyYxxsYSFNHY6lkhQU8mLzx0tr+LJjzJ4c8NuIts0563bhnNer3CnY4nUCyp58alPM/OZsTiF3KIybh0Zw58v7UOoBoqJ1Bl9t4lPHD5Wwezl6Sz+MYeeHVqQdOe5DOnaxulYIvWOSl68ylrLypT9PLwslSMlldxzcU9+P6onTRtpoJiIE1Ty4jX5RWXM/CCVNel5xHUJ480pw4nt3MrpWCL1mkpePGat5f3vs5m9Ip2KKjfTx/bltvO60UgDxUQcp5IXj+wtKGH64hS+zDrIsG5tmXtVHN01UEzEb6jk5YxUuy0Lv97F/NWZNGxgeOzKAVw/LFoDxUT8jEpeTtu2vGISkl38sOcIF/dpz5xJcXRu3dzpWCJyAip5OWWV1W5eXr+dv32SRWjThvzl12cx8azOGigm4sdU8nJKXNlHmJbkImN/MZcP6szDl8cS3kIDxUT8nUpeflZZZTXPrt3Kq1/soH3Lprx6Uzy/iu3odCwROUUqeTmpb3ccIiHZxa5DJfxmWBSJY/sR1lwDxUQCiUpe/ktxWSVzV2Ww6Ns9RLVtzj9vH865PTVQTCQQqeTl33ySkceMJankFZVx+3nd+J9LexPSRH9NRAKVvnsFgIJjFTz6YRof/LSP3h1b8OIN53J2tAaKiQQ6lXw9Z63lQ1cus5alUVxWyb2jenH3xT1p0kgjCUSCgUq+HttfWDNQbN2WPAZFhjHv6uH07aSBYiLBxKOSN8bMBy4HKoDtwK3W2iO1+6YDtwHVwB+stas9iyreYq3lnY17eXzFFirdbmaO78etI7vRUCMJRIKOp0fya4Hp1toqY8w8YDqQYIyJBa4D+gOdgXXGmN7W2moP1xMP7T50jMTkFDbsOMSI7u2YOzmOru1CnY4lIj7iUclba9ccd/cb4Ora2xOBd6y15cBOY0wWMAzY4Ml6cuaq3ZY3vtrJU2syadygAU9cFcd1Q6M0kkAkyHnznPwU4N3a212oKf3/lV27TRyQub+YackuNu89wuh+HXjsyjg6hTVzOpaI1IFfLHljzDqg0wl2zbDWLq19zAygClh0ugGMMVOBqQDR0dGn+8flZ1RUuXlxfRYvfJpFy2aN+etvzubygRE6ehepR36x5K21o39uvzHmFmACMMpaa2s35wBRxz0ssnbbiZ5/AbAAID4+3p7oMXL6ftp7hIQkF5l5xUw8qzMPX96ftqFNnI4lInXM06trxgDTgAuttSXH7VoG/NMY8ww1b7z2Ar7zZC05NaUV1Ty9JpPXv9pJh5bNeO3meEb100AxkfrK03PyzwNNgbW1pwC+sdbeaa1NM8a8B6RTcxrnbl1Z43tfbz9IYnIKewpKuGF4NAlj+9KqmQaKidRnnl5d0/Nn9s0B5njy/HJqisoqeWLlFt7+bi8x7UJ4Z+o5nNO9ndOxRMQP6BOvAW5teh4zP0jhQHE5d1zQnftG96Z5k4ZOxxIRP6GSD1CHjpbz8LI0lrty6dupJQtujGdQVGunY4mIn1HJBxhrLcs272PWsjSOllfxP7/qzZ0X9tBAMRE5IZV8ANl3pJSZH6TySUY+Z0e3Zt7kgfTu2NLpWCLix1TyAcDttry9cQ9PrMyg2m15aEIsN58bo4FiIvKLVPJ+btfBYyQku/h2ZwEje7bjiUkDiW4X4nQsEQkQKnk/VVXt5vWvdvL0mq00adSAJycP5Jr4SI0kEJHTopL3Q1tyi0hIduHKLuRXsR157MoBdGylgWIicvpU8n6kvKqaFz7J4sX122kd0pgXrh/MuLhOOnoXkTOmkvcTP+w5TEKSi235R7lqcBceHB9LGw0UExEPqeQdVlJRxVOrt/LG1zuJaNWMN24dysV9OjgdS0SChEreQV9uO0jiYhfZh0u5aURXpo3pS4um+pKIiPeoURxQWFrJnBXpvPd9Nt3CQ3nvjhEM69bW6VgiEoRU8nVsddp+HvwglUPHKrjroh7cO6oXzRproJiI+IZKvo4cKC5n1rI0VqTk0i+iFa/dPJS4yDCnY4lIkFPJ+5i1liU/5vDo8nRKyqu5/7I+TL2gO40baqCYiPieSt6Hco6U8sDiFD7beoDB0a158uqB9OyggWIiUndU8j7gdlve+nY381ZlYIGHL4/lphEaKCYidU8l72XbDxwlMdnFxl2HOb9XOI9PiiOqrQaKiYgzVPJeUlXt5tUvdvLsuq00a9SA+VcP5OohGigmIs5SyXtB+r4ipiVvJjWniDH9O/Holf3p0FIDxUTEeSp5D5RVVvP8J1m8/Nl2Woc04aUbBjM2LsLpWCIi/6KSP0ObdhcwLcnF9gPHmDw4kgcn9KN1iAaKiYh/UcmfpmPlVcxfncnCDbvoHNachVOGcWHv9k7HEhE5IZX8afh86wGmL05hX2EpN4+I4f7L+hCqgWIi4sc8aihjzGxgIuAG8oFbrLX7TM0lJc8B44CS2u0/eBrWKUdKKnhsxRaSNmXTvX0o798xgvgYDRQTEf/n6WHofGvtgwDGmD8ADwF3AmOBXrW/hgMv1f4ecFal5PLg0jQOl1Rwz8U9ueeSnhooJiIBw6OSt9YWHXc3FLC1tycCb1prLfCNMaa1MSbCWpvryXp1Kb+ojIeWpvFR2n76d27FwilD6d9ZA8VEJLB4fELZGDMHuAkoBC6u3dwF2Hvcw7Jrt/1XyRtjpgJTAaKjoz2N4zFrLUmbspm9PJ2yKjcJY/ryu/O70UgDxUQkAP1icxlj1hljUk/wayKAtXaGtTYKWATcc7oBrLULrLXx1tr49u2dvUplb0EJN73+HfcnuejTqSWr7j2fuy7qoYIXkYD1i0fy1trRp/hci4CVwMNADhB13L7I2m1+qdpteXPDLuavzsQAsyf254bhXWmggWIiEuA8vbqml7V2W+3diUBG7e1lwD3GmHeoecO10F/Px2flF5OQnMKm3Ye5sHd75kwaQGQbDRQTkeDg6Tn5ucaYPtRcQrmbmitroOaIfhyQRc0llLd6uI7XVVa7eeWz7fz14yxCmjbk6WsGcdXgLhooJiJBxdOrayafZLsF7vbkuX0pNaeQ+5NcbMktYnxcBLOu6E/7lk2djiUi4nX16uOaZZXV/GXdNl79YgdtQ5vw8m+HMGZAJ6djiYj4TL0p+e92FpCY7GLHwWNcGx/JjHGxhIU0djqWiIhPBX3JHy2vYt6qDP7xzW4i2zTnrduGc16vcKdjiYjUiaAu+U8z85mxOIXcojJuHVkzUCykSVC/ZBGRfxOUjXf4WAWzl6ez+MccenZoQdKd5zKkaxunY4mI1LmgKnlrLStScnl4aRqFpZX84ZKe3H1JT5o20kAxEamfgqbk84rKePCDVNak5xHXJYy3bh9Ov4hWTscSEXFUUJT8pxn5/OGdH6mocjN9bF9uO08DxUREIEhKvlt4KIOj2zDriv50Cw91Oo6IiN8IipKPCQ9l4ZRhTscQEfE7OqchIhLEVPIiIkFMJS8iEsRU8iIiQUwlLyISxFTyIiJBTCUvIhLEVPIiIkHM1PykPv9gjDlAzc+KDUThwEGnQzigvr5uqL+vXa/b/3S11rY/0Q6/KvlAZoz53lob73SOulZfXzfU39eu1x1YdLpGRCSIqeRFRIKYSt57FjgdwCH19XVD/X3tet0BROfkRUSCmI7kRUSCmEpeRCSIqeS9zBjzJ2OMNcaEO52lrhhj5htjMowxLmPMEmNMa6cz+ZIxZowxJtMYk2WMSXQ6T10wxkQZYz41xqQbY9KMMfc6namuGWMaGmN+NMYsdzrL6VDJe5ExJgq4FNjjdJY6thYYYK0dCGwFpjucx2eMMQ2BF4CxQCzwG2NMrLOp6kQV8CdrbSxwDnB3PXndx7sX2OJ0iNOlkveuZ4FpQL16N9tau8ZaW1V79xsg0sk8PjYMyLLW7rDWVgDvABMdzuRz1tpca+0PtbeLqSm7Ls6mqjvGmEhgPPB3p7OcLpW8lxhjJgI51trNTmdx2BRgldMhfKgLsPe4+9nUo7IDMMbEAGcD3zocpS79hZoDOLfDOU5bUPwg77pijFkHdDrBrhnAA9ScqglKP/farbVLax8zg5r/1i+qy2xSd4wxLYBk4D5rbZHTeeqCMWYCkG+t3WSMucjhOKdNJX8arLWjT7TdGBMHdAM2G2Og5nTFD8aYYdba/XUY0WdO9tr/lzHmFmACMMoG94cvcoCo4+5H1m4LesaYxtQU/CJr7WKn89ShkcAVxphxQDOglTHmLWvtbx3OdUr0YSgfMMbsAuKttf46sc6rjDFjgGeAC621B5zO40vGmEbUvLk8ippy3whcb61NczSYj5mao5eFQIG19j6H4zim9kj+z9baCQ5HOWU6Jy/e8DzQElhrjPnJGPOy04F8pfYN5nuA1dS8+fhesBd8rZHAjcAltV/jn2qPbMXP6UheRCSI6UheRCSIqeRFRIKYSl5EJIip5EVEgphKXkQkiKnkRUSCmEpeRCSI/T8K0BJulhUOnAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# derivative of f at all points\n",
    "\n",
    "def d():\n",
    "    ds = []\n",
    "    h = 0.001\n",
    "    for i in range(len(ys)):\n",
    "        ds.append((f(xs[i]+h) - f(xs[i]))/h)\n",
    "    return ds\n",
    "ds=d()\n",
    "plt.plot(xs, ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "d1 4.0\n",
      "d2 3.997\n",
      "slope -3.0000000000001137\n"
     ]
    }
   ],
   "source": [
    "# More complex \n",
    "\n",
    "h = 0.001\n",
    "\n",
    "a = 2.0\n",
    "b = -3.0\n",
    "c = 10.0\n",
    "\n",
    "d1 = a * b + c\n",
    "a += h\n",
    "d2 = a * b + c\n",
    "\n",
    "print(f\"d1 {d1}\")\n",
    "print(f\"d2 {d2}\")\n",
    "print(f\"slope\", (d2 - d1)/h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Value:\n",
    "\n",
    "    def __init__(self, data, _children=(), _op='', label=\"\"):\n",
    "        self.data = data\n",
    "        self.grad = 0.0\n",
    "        self._backward = lambda:None\n",
    "        self._prev = set(_children)\n",
    "        self._op = _op\n",
    "        self.label = label\n",
    "\n",
    "    def __repr__(self):\n",
    "        return (f\"Value(data) = {self.data}\")\n",
    "\n",
    "\n",
    "    def __add__(self, other):\n",
    "        other = other if isinstance(other, Value) else Value(other)\n",
    "        out = Value(self.data+ other.data, (self,other), _op=\"+\")\n",
    "        \n",
    "        def _backward():\n",
    "            self.grad += 1.0*out.grad\n",
    "            other.grad += 1.0*out.grad\n",
    "        out._backward = _backward\n",
    "        \n",
    "        return out\n",
    "\n",
    "    def __mul__(self, other):\n",
    "        other = other if isinstance(other, Value) else Value(other)\n",
    "        out = Value(self.data * other.data,(self, other), _op=\"*\")\n",
    "        \n",
    "        def _backward():\n",
    "            self.grad = other.data * out.grad\n",
    "            other.grad = self.data * out.grad\n",
    "        out._backward = _backward\n",
    "        \n",
    "        return out\n",
    "    \n",
    "    def __pow__(self, other):\n",
    "        assert isinstance(other, (int, float)), \"only supporting int/float powers for now\"\n",
    "        out = Value(self.data**other, (self,), f'**{other}')\n",
    "\n",
    "        def _backward():\n",
    "            self.grad += other * (self.data ** (other - 1)) * out.grad\n",
    "        out._backward = _backward\n",
    "\n",
    "        return out\n",
    "\n",
    "\n",
    "    def __rmul__(self, other):   # other * self\n",
    "        return self * other\n",
    "\n",
    "    def __truediv__(self, other): # self / other\n",
    "        return self * other**-1\n",
    "\n",
    "    def __neg__(self):            # -self\n",
    "        return self * -1\n",
    "\n",
    "    def __sub__(self, other):     # self - other\n",
    "        return self + (-other)\n",
    "\n",
    "    def __radd__(self, other):     # other + self\n",
    "        return self + other\n",
    "\n",
    "\n",
    "    def tanh(self):\n",
    "        x = self.data\n",
    "        t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1)\n",
    "        out=Value(t, (self,), 'tanh')\n",
    "        \n",
    "        def _backward():\n",
    "            self.grad = (1 - t**2) * out.grad\n",
    "        out._backward = _backward\n",
    "        \n",
    "        return out\n",
    "\n",
    "    def exp(self):\n",
    "        x = self.data\n",
    "        out = Value(math.exp(x), (self, ), 'exp')\n",
    "        \n",
    "        def _backward():\n",
    "            self.grad += out.data * out.grad \n",
    "        out._backward = _backward\n",
    "        \n",
    "        return out\n",
    "\n",
    "\n",
    "    def backward(self):\n",
    "        \"https://bit.ly/sortTopological\"\n",
    "        # \"topological sort\" to let every neuron get its gradient \n",
    "        #set as a sum of all dependencies\n",
    "        topo = []\n",
    "        visited = set()\n",
    "        def build_topo(v):\n",
    "            if v not in visited:\n",
    "                visited.add(v)\n",
    "                for child in v._prev:\n",
    "                    build_topo(child)\n",
    "                    topo.append(v)\n",
    "        build_topo(self)\n",
    "        \n",
    "        self.grad = 1.0\n",
    "        for node in reversed(topo):\n",
    "            node._backward()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# download link for graphwiz on windows: https://graphviz.org/download\n",
    "import os\n",
    "os.environ[\"PATH\"] += os.pathsep + 'F:\\\\softwares\\\\Graphviz\\\\bin'\n",
    "from graphviz import Digraph\n",
    "\n",
    "def trace(root):\n",
    "  # builds a set of all nodes and edges in a graph\n",
    "    nodes, edges = set(), set()\n",
    "    def build(v):\n",
    "      if v not in nodes:\n",
    "          nodes.add(v)\n",
    "          for child in v._prev:\n",
    "              edges.add((child, v))\n",
    "              build(child)\n",
    "    build(root)\n",
    "    return nodes, edges\n",
    "\n",
    "def draw_dot(root):\n",
    "    dot = Digraph(format='svg', graph_attr={'rankdir': 'LR'}) # LR = left to right\n",
    "  \n",
    "    nodes, edges = trace(root)\n",
    "    for n in nodes:\n",
    "        uid = str(id(n))\n",
    "    # for any value in the graph, create a rectangular ('record') node for it\n",
    "    dot.node(name = uid, label = \"{ %s | data %.4f | grad %.4f }\" % (n.label, n.data, n.grad), shape='record')\n",
    "    if n._op:\n",
    "      # if this value is a result of some operation, create an op node for it\n",
    "        dot.node(name = uid + n._op, label = n._op)\n",
    "      # and connect this node to it\n",
    "        dot.edge(uid + n._op, uid)\n",
    "\n",
    "    for n1, n2 in edges:\n",
    "    # connect n1 to the op node of n2\n",
    "        dot.edge(str(id(n1)), str(id(n2)) + n2._op)\n",
    "\n",
    "    return dot\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{Value(data) = 10, Value(data) = 10.5}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = Value(2.1, label=\"a\")\n",
    "b = Value(5, label=\"b\")\n",
    "c = Value(10, label=\"c\")\n",
    "d = a * b + c\n",
    "d._prev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#inputs\n",
    "x1 = Value(2.0, label='x1')\n",
    "x2 = Value(0.0, label='x2')\n",
    "\n",
    "#weights\n",
    "w1 = Value(-3.0, label='w1')\n",
    "w2 = Value(1.0, label='w1')\n",
    "\n",
    "#bias of the neuron this specific number for numbers comming out nicely\n",
    "b = Value(6.8813735870195432, label='b')\n",
    "\n",
    "x1w1 = x1*w1 ; x1w1.label = 'x1w1'\n",
    "x2w2 = x2*w2 ; x2w2.label = 'x2w2'\n",
    "\n",
    "x1w1x2w2 = x1w1 + x2w2 ; x1w1x2w2.label = \"x1w1+x2w2\"\n",
    "\n",
    "n = x1w1x2w2+b ; n.label = 'n'\n",
    "o = n.tanh() ; o.label = 'o'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 7.0.2 (20221119.0110)\n -->\n<!-- Pages: 1 -->\n<svg width=\"1559pt\" height=\"210pt\"\n viewBox=\"0.00 0.00 1559.00 210.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 206)\">\n<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-206 1555,-206 1555,4 -4,4\"/>\n<!-- 2419848878112 -->\n<g id=\"node1\" class=\"node\">\n<title>2419848878112</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"703,-137.5 703,-173.5 891,-173.5 891,-137.5 703,-137.5\"/>\n<text text-anchor=\"middle\" x=\"715\" y=\"-151.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">b</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"727,-137.5 727,-173.5\"/>\n<text text-anchor=\"middle\" x=\"767.5\" y=\"-151.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 6.8814</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"808,-137.5 808,-173.5\"/>\n<text text-anchor=\"middle\" x=\"849.5\" y=\"-151.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 0.0000</text>\n</g>\n<!-- 2419847257440+ -->\n<g id=\"node10\" class=\"node\">\n<title>2419847257440+</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"987\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"987\" y=\"-123.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">+</text>\n</g>\n<!-- 2419848878112&#45;&gt;2419847257440+ -->\n<g id=\"edge11\" class=\"edge\">\n<title>2419848878112&#45;&gt;2419847257440+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M890.9,-141.66C911.64,-138.57 932.46,-135.47 949.31,-132.97\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"949.41,-136.49 958.79,-131.55 948.38,-129.56 949.41,-136.49\"/>\n</g>\n<!-- 2419847256624 -->\n<g id=\"node2\" class=\"node\">\n<title>2419847256624</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"670,-82.5 670,-118.5 924,-118.5 924,-82.5 670,-82.5\"/>\n<text text-anchor=\"middle\" x=\"712.5\" y=\"-96.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">x1w1+x2w2</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"755,-82.5 755,-118.5\"/>\n<text text-anchor=\"middle\" x=\"798\" y=\"-96.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"841,-82.5 841,-118.5\"/>\n<text text-anchor=\"middle\" x=\"882.5\" y=\"-96.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 0.0000</text>\n</g>\n<!-- 2419847256624&#45;&gt;2419847257440+ -->\n<g id=\"edge14\" class=\"edge\">\n<title>2419847256624&#45;&gt;2419847257440+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M923.14,-118.47C932.29,-119.79 941.01,-121.04 948.87,-122.17\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"948.2,-125.61 958.59,-123.56 949.19,-118.68 948.2,-125.61\"/>\n</g>\n<!-- 2419847256624+ -->\n<g id=\"node3\" class=\"node\">\n<title>2419847256624+</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"607\" cy=\"-100.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"607\" y=\"-96.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">+</text>\n</g>\n<!-- 2419847256624+&#45;&gt;2419847256624 -->\n<g id=\"edge1\" class=\"edge\">\n<title>2419847256624+&#45;&gt;2419847256624</title>\n<path fill=\"none\" stroke=\"black\" d=\"M634.14,-100.5C641.39,-100.5 649.78,-100.5 658.86,-100.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"658.74,-104 668.74,-100.5 658.74,-97 658.74,-104\"/>\n</g>\n<!-- 2419847317584 -->\n<g id=\"node4\" class=\"node\">\n<title>2419847317584</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"1363,-109.5 1363,-145.5 1551,-145.5 1551,-109.5 1363,-109.5\"/>\n<text text-anchor=\"middle\" x=\"1375\" y=\"-123.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">o</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1387,-109.5 1387,-145.5\"/>\n<text text-anchor=\"middle\" x=\"1427.5\" y=\"-123.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 0.7071</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1468,-109.5 1468,-145.5\"/>\n<text text-anchor=\"middle\" x=\"1509.5\" y=\"-123.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 0.0000</text>\n</g>\n<!-- 2419847317584tanh -->\n<g id=\"node5\" class=\"node\">\n<title>2419847317584tanh</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"1300\" cy=\"-127.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"1300\" y=\"-123.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">tanh</text>\n</g>\n<!-- 2419847317584tanh&#45;&gt;2419847317584 -->\n<g id=\"edge2\" class=\"edge\">\n<title>2419847317584tanh&#45;&gt;2419847317584</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1327.29,-127.5C1334.52,-127.5 1342.82,-127.5 1351.66,-127.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1351.57,-131 1361.57,-127.5 1351.57,-124 1351.57,-131\"/>\n</g>\n<!-- 2419848877728 -->\n<g id=\"node6\" class=\"node\">\n<title>2419848877728</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"4,-55.5 4,-91.5 198,-91.5 198,-55.5 4,-55.5\"/>\n<text text-anchor=\"middle\" x=\"19\" y=\"-69.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">x2</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"34,-55.5 34,-91.5\"/>\n<text text-anchor=\"middle\" x=\"74.5\" y=\"-69.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 0.0000</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"115,-55.5 115,-91.5\"/>\n<text text-anchor=\"middle\" x=\"156.5\" y=\"-69.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 0.0000</text>\n</g>\n<!-- 2419847254992* -->\n<g id=\"node15\" class=\"node\">\n<title>2419847254992*</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"265\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"265\" y=\"-69.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">*</text>\n</g>\n<!-- 2419848877728&#45;&gt;2419847254992* -->\n<g id=\"edge12\" class=\"edge\">\n<title>2419848877728&#45;&gt;2419847254992*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M197.64,-73.5C207.72,-73.5 217.49,-73.5 226.29,-73.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"226.11,-77 236.11,-73.5 226.11,-70 226.11,-77\"/>\n</g>\n<!-- 2419847254272 -->\n<g id=\"node7\" class=\"node\">\n<title>2419847254272</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"328,-110.5 328,-146.5 544,-146.5 544,-110.5 328,-110.5\"/>\n<text text-anchor=\"middle\" x=\"351.5\" y=\"-124.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">x1w1</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"375,-110.5 375,-146.5\"/>\n<text text-anchor=\"middle\" x=\"418\" y=\"-124.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"461,-110.5 461,-146.5\"/>\n<text text-anchor=\"middle\" x=\"502.5\" y=\"-124.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 0.0000</text>\n</g>\n<!-- 2419847254272&#45;&gt;2419847256624+ -->\n<g id=\"edge13\" class=\"edge\">\n<title>2419847254272&#45;&gt;2419847256624+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M543.55,-110.85C552.57,-109.35 561.22,-107.92 569.07,-106.62\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"569.51,-110.09 578.8,-105.01 568.36,-103.19 569.51,-110.09\"/>\n</g>\n<!-- 2419847254272* -->\n<g id=\"node8\" class=\"node\">\n<title>2419847254272*</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"265\" cy=\"-128.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"265\" y=\"-124.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">*</text>\n</g>\n<!-- 2419847254272*&#45;&gt;2419847254272 -->\n<g id=\"edge3\" class=\"edge\">\n<title>2419847254272*&#45;&gt;2419847254272</title>\n<path fill=\"none\" stroke=\"black\" d=\"M292.41,-128.5C299.59,-128.5 307.85,-128.5 316.71,-128.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"316.7,-132 326.7,-128.5 316.7,-125 316.7,-132\"/>\n</g>\n<!-- 2419847257440 -->\n<g id=\"node9\" class=\"node\">\n<title>2419847257440</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"1050,-109.5 1050,-145.5 1237,-145.5 1237,-109.5 1050,-109.5\"/>\n<text text-anchor=\"middle\" x=\"1061.5\" y=\"-123.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">n</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1073,-109.5 1073,-145.5\"/>\n<text text-anchor=\"middle\" x=\"1113.5\" y=\"-123.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 0.8814</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"1154,-109.5 1154,-145.5\"/>\n<text text-anchor=\"middle\" x=\"1195.5\" y=\"-123.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 0.0000</text>\n</g>\n<!-- 2419847257440&#45;&gt;2419847317584tanh -->\n<g id=\"edge10\" class=\"edge\">\n<title>2419847257440&#45;&gt;2419847317584tanh</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1236.6,-127.5C1245.26,-127.5 1253.65,-127.5 1261.33,-127.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1261.23,-131 1271.23,-127.5 1261.23,-124 1261.23,-131\"/>\n</g>\n<!-- 2419847257440+&#45;&gt;2419847257440 -->\n<g id=\"edge4\" class=\"edge\">\n<title>2419847257440+&#45;&gt;2419847257440</title>\n<path fill=\"none\" stroke=\"black\" d=\"M1014.21,-127.5C1021.47,-127.5 1029.83,-127.5 1038.71,-127.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"1038.7,-131 1048.7,-127.5 1038.7,-124 1038.7,-131\"/>\n</g>\n<!-- 2419848879984 -->\n<g id=\"node11\" class=\"node\">\n<title>2419848879984</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"0,-165.5 0,-201.5 202,-201.5 202,-165.5 0,-165.5\"/>\n<text text-anchor=\"middle\" x=\"16.5\" y=\"-179.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">w1</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"33,-165.5 33,-201.5\"/>\n<text text-anchor=\"middle\" x=\"76\" y=\"-179.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data &#45;3.0000</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"119,-165.5 119,-201.5\"/>\n<text text-anchor=\"middle\" x=\"160.5\" y=\"-179.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 0.0000</text>\n</g>\n<!-- 2419848879984&#45;&gt;2419847254272* -->\n<g id=\"edge9\" class=\"edge\">\n<title>2419848879984&#45;&gt;2419847254272*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M173.99,-165.56C183.48,-162.75 193.03,-159.71 202,-156.5 212.1,-152.88 222.83,-148.31 232.49,-143.91\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"233.88,-147.12 241.46,-139.72 230.92,-140.78 233.88,-147.12\"/>\n</g>\n<!-- 2419848877440 -->\n<g id=\"node12\" class=\"node\">\n<title>2419848877440</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"2.5,-0.5 2.5,-36.5 199.5,-36.5 199.5,-0.5 2.5,-0.5\"/>\n<text text-anchor=\"middle\" x=\"19\" y=\"-14.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">w1</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"35.5,-0.5 35.5,-36.5\"/>\n<text text-anchor=\"middle\" x=\"76\" y=\"-14.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 1.0000</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"116.5,-0.5 116.5,-36.5\"/>\n<text text-anchor=\"middle\" x=\"158\" y=\"-14.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 0.0000</text>\n</g>\n<!-- 2419848877440&#45;&gt;2419847254992* -->\n<g id=\"edge7\" class=\"edge\">\n<title>2419848877440&#45;&gt;2419847254992*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M170.98,-36.47C181.45,-39.58 192.06,-42.96 202,-46.5 211.97,-50.05 222.6,-54.44 232.19,-58.65\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"230.55,-61.75 241.11,-62.64 233.41,-55.36 230.55,-61.75\"/>\n</g>\n<!-- 2419848879024 -->\n<g id=\"node13\" class=\"node\">\n<title>2419848879024</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"4,-110.5 4,-146.5 198,-146.5 198,-110.5 4,-110.5\"/>\n<text text-anchor=\"middle\" x=\"19\" y=\"-124.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">x1</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"34,-110.5 34,-146.5\"/>\n<text text-anchor=\"middle\" x=\"74.5\" y=\"-124.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 2.0000</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"115,-110.5 115,-146.5\"/>\n<text text-anchor=\"middle\" x=\"156.5\" y=\"-124.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 0.0000</text>\n</g>\n<!-- 2419848879024&#45;&gt;2419847254272* -->\n<g id=\"edge8\" class=\"edge\">\n<title>2419848879024&#45;&gt;2419847254272*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M197.64,-128.5C207.72,-128.5 217.49,-128.5 226.29,-128.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"226.11,-132 236.11,-128.5 226.11,-125 226.11,-132\"/>\n</g>\n<!-- 2419847254992 -->\n<g id=\"node14\" class=\"node\">\n<title>2419847254992</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"330.5,-55.5 330.5,-91.5 541.5,-91.5 541.5,-55.5 330.5,-55.5\"/>\n<text text-anchor=\"middle\" x=\"354\" y=\"-69.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">x2w2</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"377.5,-55.5 377.5,-91.5\"/>\n<text text-anchor=\"middle\" x=\"418\" y=\"-69.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 0.0000</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"458.5,-55.5 458.5,-91.5\"/>\n<text text-anchor=\"middle\" x=\"500\" y=\"-69.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 0.0000</text>\n</g>\n<!-- 2419847254992&#45;&gt;2419847256624+ -->\n<g id=\"edge6\" class=\"edge\">\n<title>2419847254992&#45;&gt;2419847256624+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M541.14,-90.14C551.14,-91.73 560.74,-93.27 569.36,-94.65\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"568.51,-98.06 578.94,-96.18 569.62,-91.14 568.51,-98.06\"/>\n</g>\n<!-- 2419847254992*&#45;&gt;2419847254992 -->\n<g id=\"edge5\" class=\"edge\">\n<title>2419847254992*&#45;&gt;2419847254992</title>\n<path fill=\"none\" stroke=\"black\" d=\"M292.41,-73.5C300.3,-73.5 309.5,-73.5 319.36,-73.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"319.22,-77 329.22,-73.5 319.22,-70 319.22,-77\"/>\n</g>\n</g>\n</svg>\n",
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x233674642e0>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "draw_dot(o)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 7.0.2 (20221119.0110)\n -->\n<!-- Pages: 1 -->\n<svg width=\"835pt\" height=\"100pt\"\n viewBox=\"0.00 0.00 835.00 100.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 96)\">\n<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-96 831,-96 831,4 -4,4\"/>\n<!-- 2419848879840 -->\n<g id=\"node1\" class=\"node\">\n<title>2419848879840</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"0,-55.5 0,-91.5 192,-91.5 192,-55.5 0,-55.5\"/>\n<text text-anchor=\"middle\" x=\"12\" y=\"-69.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">b</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"24,-55.5 24,-91.5\"/>\n<text text-anchor=\"middle\" x=\"64.5\" y=\"-69.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 3.0000</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"105,-55.5 105,-91.5\"/>\n<text text-anchor=\"middle\" x=\"148.5\" y=\"-69.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad &#45;2.0000</text>\n</g>\n<!-- 2419847318304* -->\n<g id=\"node5\" class=\"node\">\n<title>2419847318304*</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"255\" cy=\"-73.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"255\" y=\"-69.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">*</text>\n</g>\n<!-- 2419848879840&#45;&gt;2419847318304* -->\n<g id=\"edge4\" class=\"edge\">\n<title>2419848879840&#45;&gt;2419847318304*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M191.95,-73.5C200.51,-73.5 208.8,-73.5 216.38,-73.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"216.14,-77 226.14,-73.5 216.14,-70 216.14,-77\"/>\n</g>\n<!-- 2419847315760+ -->\n<g id=\"node7\" class=\"node\">\n<title>2419847315760+</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"255\" cy=\"-18.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"255\" y=\"-14.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">+</text>\n</g>\n<!-- 2419848879840&#45;&gt;2419847315760+ -->\n<g id=\"edge8\" class=\"edge\">\n<title>2419848879840&#45;&gt;2419847315760+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M164.86,-55.54C174.05,-52.72 183.3,-49.68 192,-46.5 201.99,-42.85 212.63,-38.3 222.22,-33.93\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"223.55,-37.17 231.15,-29.78 220.6,-30.83 223.55,-37.17\"/>\n</g>\n<!-- 2419847318256 -->\n<g id=\"node2\" class=\"node\">\n<title>2419847318256</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"637,-27.5 637,-63.5 827,-63.5 827,-27.5 637,-27.5\"/>\n<text text-anchor=\"middle\" x=\"647.5\" y=\"-41.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">f</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"658,-27.5 658,-63.5\"/>\n<text text-anchor=\"middle\" x=\"701\" y=\"-41.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"744,-27.5 744,-63.5\"/>\n<text text-anchor=\"middle\" x=\"785.5\" y=\"-41.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 1.0000</text>\n</g>\n<!-- 2419847318256* -->\n<g id=\"node3\" class=\"node\">\n<title>2419847318256*</title>\n<ellipse fill=\"none\" stroke=\"black\" cx=\"574\" cy=\"-45.5\" rx=\"27\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"574\" y=\"-41.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">*</text>\n</g>\n<!-- 2419847318256*&#45;&gt;2419847318256 -->\n<g id=\"edge1\" class=\"edge\">\n<title>2419847318256*&#45;&gt;2419847318256</title>\n<path fill=\"none\" stroke=\"black\" d=\"M601.46,-45.5C608.68,-45.5 616.95,-45.5 625.76,-45.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"625.64,-49 635.64,-45.5 625.64,-42 625.64,-49\"/>\n</g>\n<!-- 2419847318304 -->\n<g id=\"node4\" class=\"node\">\n<title>2419847318304</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"318,-55.5 318,-91.5 511,-91.5 511,-55.5 318,-55.5\"/>\n<text text-anchor=\"middle\" x=\"330\" y=\"-69.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">d</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"342,-55.5 342,-91.5\"/>\n<text text-anchor=\"middle\" x=\"385\" y=\"-69.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data &#45;6.0000</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"428,-55.5 428,-91.5\"/>\n<text text-anchor=\"middle\" x=\"469.5\" y=\"-69.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 1.0000</text>\n</g>\n<!-- 2419847318304&#45;&gt;2419847318256* -->\n<g id=\"edge7\" class=\"edge\">\n<title>2419847318304&#45;&gt;2419847318256*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M510.75,-56.57C519.7,-54.98 528.35,-53.44 536.21,-52.04\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"536.75,-55.5 545.99,-50.3 535.53,-48.61 536.75,-55.5\"/>\n</g>\n<!-- 2419847318304*&#45;&gt;2419847318304 -->\n<g id=\"edge2\" class=\"edge\">\n<title>2419847318304*&#45;&gt;2419847318304</title>\n<path fill=\"none\" stroke=\"black\" d=\"M282.36,-73.5C289.57,-73.5 297.84,-73.5 306.66,-73.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"306.56,-77 316.56,-73.5 306.56,-70 306.56,-77\"/>\n</g>\n<!-- 2419847315760 -->\n<g id=\"node6\" class=\"node\">\n<title>2419847315760</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"319,-0.5 319,-36.5 510,-36.5 510,-0.5 319,-0.5\"/>\n<text text-anchor=\"middle\" x=\"330.5\" y=\"-14.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">e</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"342,-0.5 342,-36.5\"/>\n<text text-anchor=\"middle\" x=\"382.5\" y=\"-14.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data 1.0000</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"423,-0.5 423,-36.5\"/>\n<text text-anchor=\"middle\" x=\"466.5\" y=\"-14.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad &#45;6.0000</text>\n</g>\n<!-- 2419847315760&#45;&gt;2419847318256* -->\n<g id=\"edge6\" class=\"edge\">\n<title>2419847315760&#45;&gt;2419847318256*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M509.85,-34.67C519.09,-36.26 528.03,-37.79 536.14,-39.18\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"535.42,-42.61 545.87,-40.85 536.61,-35.71 535.42,-42.61\"/>\n</g>\n<!-- 2419847315760+&#45;&gt;2419847315760 -->\n<g id=\"edge3\" class=\"edge\">\n<title>2419847315760+&#45;&gt;2419847315760</title>\n<path fill=\"none\" stroke=\"black\" d=\"M282.36,-18.5C289.83,-18.5 298.46,-18.5 307.64,-18.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"307.56,-22 317.56,-18.5 307.56,-15 307.56,-22\"/>\n</g>\n<!-- 2419847316768 -->\n<g id=\"node8\" class=\"node\">\n<title>2419847316768</title>\n<polygon fill=\"none\" stroke=\"black\" points=\"0,-0.5 0,-36.5 192,-36.5 192,-0.5 0,-0.5\"/>\n<text text-anchor=\"middle\" x=\"11.5\" y=\"-14.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">a</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"23,-0.5 23,-36.5\"/>\n<text text-anchor=\"middle\" x=\"66\" y=\"-14.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">data &#45;2.0000</text>\n<polyline fill=\"none\" stroke=\"black\" points=\"109,-0.5 109,-36.5\"/>\n<text text-anchor=\"middle\" x=\"150.5\" y=\"-14.8\" font-family=\"Times New Roman,serif\" font-size=\"14.00\">grad 3.0000</text>\n</g>\n<!-- 2419847316768&#45;&gt;2419847318304* -->\n<g id=\"edge9\" class=\"edge\">\n<title>2419847316768&#45;&gt;2419847318304*</title>\n<path fill=\"none\" stroke=\"black\" d=\"M161.99,-36.49C172.11,-39.62 182.38,-43 192,-46.5 201.86,-50.08 212.39,-54.46 221.92,-58.63\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"220.23,-61.71 230.79,-62.59 223.09,-55.32 220.23,-61.71\"/>\n</g>\n<!-- 2419847316768&#45;&gt;2419847315760+ -->\n<g id=\"edge5\" class=\"edge\">\n<title>2419847316768&#45;&gt;2419847315760+</title>\n<path fill=\"none\" stroke=\"black\" d=\"M191.95,-18.5C200.51,-18.5 208.8,-18.5 216.38,-18.5\"/>\n<polygon fill=\"black\" stroke=\"black\" points=\"216.14,-22 226.14,-18.5 216.14,-15 216.14,-22\"/>\n</g>\n</g>\n</svg>\n",
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x2336a23cb80>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "a = Value(-2.0, label='a')\n",
    "b = Value(3.0, label='b')\n",
    "\n",
    "d = a * b    ; d.label = 'd'\n",
    "e = a + b    ; e.label = 'e'\n",
    "f = d * e    ; f.label = 'f'\n",
    "\n",
    "f.backward()\n",
    "\n",
    "draw_dot(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Neuron:\n",
    "    \"\"\"\n",
    "    Takes in nin inputs (len n) takes their dot product with weights  (len n) adds bias (1) and applies activation function to the result and returns that Value\n",
    "\n",
    "    ouput = activationF(inputs.weights + bias)\n",
    "\n",
    "    \"\"\"\n",
    "    def __init__(self, nin):\n",
    "        self.w = [Value(random.uniform(-1,1)) for _ in range(nin)]\n",
    "        self.b = Value(random.uniform(-1,1))\n",
    "    \n",
    "    def __call__(self, x):\n",
    "        # w * x + b\n",
    "        act = sum((wi*xi for wi, xi in zip(self.w, x)), self.b)\n",
    "        out = act.tanh()\n",
    "        return out\n",
    "        \n",
    "    def parameters(self):\n",
    "        return self.w + [self.b]\n",
    "\n",
    "class Layer:\n",
    "    \"\"\"\n",
    "      nin is nin for the neurons and nout is the number of neurons    \n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, nin, nout):\n",
    "        self.neurons = [Neuron(nin) for _ in range(nout)]\n",
    "  \n",
    "    def __call__(self, x):\n",
    "        outs = [n(x) for n in self.neurons]\n",
    "        return outs[0] if len(outs) == 1 else outs\n",
    "  \n",
    "    def parameters(self):\n",
    "        return [p for neuron in self.neurons for p in neuron.parameters()]\n",
    "\n",
    "class MLP:\n",
    "    \"\"\"\n",
    "    nin specifies the number of inputs of the models\n",
    "\n",
    "    nouts is an list that specifies the number of outputs of each layer\n",
    "    as number of outputs equal the number of nerons in each layer \n",
    "    nouts also specifies the number of nerons in each layer\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, nin, nouts):\n",
    "        sz = [nin] + nouts\n",
    "        self.layers = [Layer(sz[i], sz[i+1]) for i in range(len(nouts))]\n",
    "      \n",
    "    def __call__(self, x):\n",
    "        for layer in self.layers:\n",
    "          x = layer(x)\n",
    "        return x\n",
    "      \n",
    "    def parameters(self):\n",
    "        return [p for layer in self.layers for p in layer.parameters()]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Value(data) = -0.9753297822318988"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# three layer model with each layer containing 4, 4, 1 neurons \n",
    "x = [2.0, 3.0, -1.0]\n",
    "n = MLP(3, [4, 4, 1])\n",
    "n(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "xs = [\n",
    "  [2.0, 3.0, -1.0],\n",
    "  [3.0, -1.0, 0.5],\n",
    "  [0.5, 1.0, 1.0],\n",
    "  [1.0, 1.0, -1.0],\n",
    "]\n",
    "ys = [1.0, -1.0, -1.0, 1.0] # desired targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 7.83843754118174\n",
      "1 6.86441025150103\n",
      "2 7.998138777426328\n",
      "3 7.998045078882516\n",
      "4 7.997959497517847\n",
      "5 7.997944463193285\n",
      "6 7.997898654121585\n",
      "7 7.997781056493949\n",
      "8 7.997631692943746\n",
      "9 7.997472253124304\n",
      "10 7.997383883327377\n",
      "11 7.997288033259455\n",
      "12 7.997134673804023\n",
      "13 7.997101669424174\n",
      "14 7.9965377197818555\n",
      "15 7.994841590018066\n",
      "16 7.994712478242436\n",
      "17 7.993966325579161\n",
      "18 7.99372912994776\n",
      "19 7.992087181474355\n"
     ]
    }
   ],
   "source": [
    "\n",
    "for k in range(20):\n",
    "  \n",
    "  # forward pass\n",
    "  ypred = [n(x) for x in xs]\n",
    "  loss = sum((yout - ygt)**2 for ygt, yout in zip(ys, ypred))\n",
    "  \n",
    "  # backward pass\n",
    "  for p in n.parameters():\n",
    "    p.grad = 0.0\n",
    "  loss.backward()\n",
    "  \n",
    "  # update\n",
    "  for p in n.parameters():\n",
    "    p.data += -0.1 * p.grad\n",
    "  \n",
    "  print(k, loss.data)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "81794d4967e6c3204c66dcd87b604927b115b27c00565d3d43f05ba2f3a2cb0d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}