Chapter 2 - Machine Learning Basics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 2  Scripts for running basic experiments on Theory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import packages and verify versions\n",
    "\n",
    "# numpy\n",
    "import numpy\n",
    "print('numpy: %s' % numpy.__version__)\n",
    "# scipy\n",
    "import scipy\n",
    "print('scipy: %s' % scipy.__version__)\n",
    "# matplotlib\n",
    "import matplotlib\n",
    "print('matplotlib: %s' % matplotlib.__version__)\n",
    "# pandas\n",
    "import pandas\n",
    "print('pandas: %s' % pandas.__version__)\n",
    "# scikit-learn\n",
    "import sklearn\n",
    "print('sklearn: %s' % sklearn.__version__)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.pipeline import Pipeline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "print(__doc__)\n",
    "\n",
    "\n",
    "def true_fun(X):\n",
    "    return np.sin(1.5 * np.pi * X)\n",
    "\n",
    "\n",
    "np.random.seed(0)\n",
    "\n",
    "n_samples = 30\n",
    "degrees = [1, 3, 15]\n",
    "\n",
    "X = np.sort(np.random.rand(n_samples))\n",
    "y = true_fun(X) + np.random.randn(n_samples) * 0.1\n",
    "\n",
    "plt.figure(figsize=(14, 5))\n",
    "for i in range(len(degrees)):\n",
    "    ax = plt.subplot(1, len(degrees), i + 1)\n",
    "    plt.setp(ax, xticks=(), yticks=())\n",
    "\n",
    "    polynomial_features = PolynomialFeatures(degree=degrees[i],\n",
    "                                             include_bias=False)\n",
    "    linear_regression = LinearRegression()\n",
    "    pipeline = Pipeline([(\"polynomial_features\", polynomial_features),\n",
    "                         (\"linear_regression\", linear_regression)])\n",
    "    pipeline.fit(X[:, np.newaxis], y)\n",
    "\n",
    "    # Evaluate the models using crossvalidation\n",
    "    scores = cross_val_score(pipeline, X[:, np.newaxis], y,\n",
    "                             scoring=\"neg_mean_squared_error\", cv=10)\n",
    "\n",
    "    X_test = np.linspace(0, 1, 100)\n",
    "    plt.plot(X_test, pipeline.predict(\n",
    "        X_test[:, np.newaxis]), label=\"Hypothesis\")\n",
    "    plt.plot(X_test, true_fun(X_test), label=\"Target function\")\n",
    "    plt.scatter(X, y, edgecolor='b', s=20, label=\"Training Samples\")\n",
    "    plt.xlabel(\"x\")\n",
    "    plt.ylabel(\"y\")\n",
    "    plt.xlim((0, 1))\n",
    "    plt.ylim((-2, 2))\n",
    "    plt.legend(loc=\"best\")\n",
    "    plt.title(\"Degree {}\\nTraining Error = {:.2e}(+/- {:.2e})\".format(\n",
    "        degrees[i], -scores.mean(), scores.std()))\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bias Variance Error \n",
    "Polynomial Regression with degree 1 and degree 12 both trying to fit a target function dawn from sine function in one dimension to illustrate bias, variance and total error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(__doc__)\n",
    "\n",
    "# Author: Gilles Louppe <g.louppe@gmail.com>, Uday Kamath<kamathuday@gmail.com>\n",
    "# License: BSD 3 clause\n",
    "# extended this code and changed it to reflect bias, variance and error\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "\n",
    "# Settings\n",
    "n_repeat = 50       # Number of iterations for computing expectations\n",
    "n_train = 50        # Size of the training set\n",
    "n_test = 1000       # Size of the test set\n",
    "noise = 0.1         # Standard deviation of the noise\n",
    "np.random.seed(0)\n",
    "\n",
    "# three polynomials with degree 1, \n",
    "# and degree 12 polynomial. The impact on bias, variance and error on all three\n",
    "# can be seen through this example.\n",
    "polynomial_features1 = PolynomialFeatures(degree=1,\n",
    "                                             include_bias=False)\n",
    "linear_regression1= LinearRegression()\n",
    "pipeline1 = Pipeline([(\"polynomial_features\", polynomial_features1),\n",
    "                        (\"linear_regression\", linear_regression1)])\n",
    "\n",
    "polynomial_features2 = PolynomialFeatures(degree=1,\n",
    "                                             include_bias=False)\n",
    "\n",
    "polynomial_features3 = PolynomialFeatures(degree=12,\n",
    "                                             include_bias=False)\n",
    "linear_regression3= LinearRegression()\n",
    "pipeline3 = Pipeline([(\"polynomial_features\", polynomial_features3),\n",
    "                        (\"linear_regression\", linear_regression3)])\n",
    "estimators = [(\"Poly deg=1\", pipeline1), (\"Poly deg=12\", pipeline3)]\n",
    "\n",
    "n_estimators = len(estimators)\n",
    "\n",
    "\n",
    "# Generate data with sine function\n",
    "def f(x):\n",
    "    x = x.ravel()\n",
    "\n",
    "    return np.sin(1.5 * np.pi * x)\n",
    "\n",
    "\n",
    "def generate(n_samples, noise, n_repeat=1):\n",
    "    X = np.random.rand(n_samples) \n",
    "    X = np.sort(X)\n",
    "\n",
    "    if n_repeat == 1:\n",
    "        y = f(X) + np.random.normal(0.0, noise, n_samples)\n",
    "    else:\n",
    "        y = np.zeros((n_samples, n_repeat))\n",
    "\n",
    "        for i in range(n_repeat):\n",
    "            y[:, i] = f(X) + np.random.normal(0.0, noise, n_samples)\n",
    "\n",
    "    X = X.reshape((n_samples, 1))\n",
    "\n",
    "    return X, y\n",
    "\n",
    "\n",
    "X_train = []\n",
    "y_train = []\n",
    "\n",
    "for i in range(n_repeat):\n",
    "    X, y = generate(n_samples=n_train, noise=noise)\n",
    "    X_train.append(X)\n",
    "    y_train.append(y)\n",
    "\n",
    "X_test, y_test = generate(n_samples=n_test, noise=noise, n_repeat=n_repeat)\n",
    "\n",
    "plt.figure(figsize=(30, 14))\n",
    "\n",
    "# Loop over estimators to compare\n",
    "for n, (name, estimator) in enumerate(estimators):\n",
    "    # Compute predictions\n",
    "    y_predict = np.zeros((n_test, n_repeat))\n",
    "\n",
    "    for i in range(n_repeat):\n",
    "        estimator.fit(X_train[i], y_train[i])\n",
    "        y_predict[:, i] = estimator.predict(X_test)\n",
    "\n",
    "    # Bias^2 + Variance + Noise decomposition of the mean squared error\n",
    "    y_error = np.zeros(n_test)\n",
    "\n",
    "    for i in range(n_repeat):\n",
    "        for j in range(n_repeat):\n",
    "            y_error += (y_test[:, j] - y_predict[:, i]) ** 2\n",
    "\n",
    "    y_error /= (n_repeat * n_repeat)\n",
    "\n",
    "    y_noise = np.var(y_test, axis=1)\n",
    "    y_bias = (f(X_test) - np.mean(y_predict, axis=1)) ** 2\n",
    "    y_var = np.var(y_predict, axis=1)\n",
    "\n",
    "    print(\"{0}: {1:.4f} (error) = {2:.4f} (bias^2) \"\n",
    "          \" + {3:.4f} (var) + {4:.4f} (noise)\".format(name,\n",
    "                                                      np.mean(y_error),\n",
    "                                                      np.mean(y_bias),\n",
    "                                                      np.mean(y_var),\n",
    "                                                      np.mean(y_noise)))\n",
    "\n",
    "    # Plot figures\n",
    "    plt.subplot(2, n_estimators, n + 1)\n",
    "    plt.plot(X_test, f(X_test), \"b\", label=\"target $f(x)$\")\n",
    "    plt.plot(X_train[0], y_train[0], \".b\", label=\"Training Data $y = f(x)+noise$\")\n",
    "\n",
    "    for i in range(n_repeat):\n",
    "        if i == 0:\n",
    "            plt.plot(X_test, y_predict[:, i], \"r\", label=\"Hypothesis $\\^y(x)$\")\n",
    "        else:\n",
    "            plt.plot(X_test, y_predict[:, i], \"r\", alpha=0.05)\n",
    "\n",
    "    plt.plot(X_test, np.mean(y_predict, axis=1), \"c\",\n",
    "             label=\"$\\mathbb{E}_{Train} \\^y(x)$\")\n",
    "\n",
    "    plt.xlim([0, 1])\n",
    "    plt.title(name)\n",
    "\n",
    "    if n == n_estimators - 1:\n",
    "        plt.ylim([-1.5,2])\n",
    "        plt.legend(loc=(1.1, .5))\n",
    "\n",
    "    plt.subplot(2, n_estimators, n_estimators + n + 1)\n",
    "    plt.plot(X_test, y_error, \"r\", label=\"$error(x)$\")\n",
    "    plt.plot(X_test, y_bias, \"b\", label=\"$bias^2(x)$\"),\n",
    "    plt.plot(X_test, y_var, \"g\", label=\"$variance(x)$\"),\n",
    "    plt.plot(X_test, y_noise, \"c\", label=\"$noise(x)$\")\n",
    "\n",
    "    plt.xlim([0, 1])\n",
    "    plt.ylim([0, 0.06])\n",
    "\n",
    "    if n == n_estimators - 1:\n",
    "\n",
    "        plt.legend(loc=(1.1, .5))\n",
    "\n",
    "plt.subplots_adjust(right=.75)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Transformation and Linear Model in transformed Z space showing Kernels in action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(__doc__)\n",
    "\n",
    "import numpy as np\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "from math import sqrt\n",
    "from sklearn.datasets import make_circles\n",
    "\n",
    "\n",
    "\n",
    "def transform(x1, x2):\n",
    "    # Transforms a kernel phi(x1,x2) = [x1, x2, x1^2 + x2^2] \"\"\"\n",
    "    return np.array([x1, x2, x1**2.0 + x2**2.0])\n",
    "\n",
    "    \n",
    "n = 200\n",
    "X, Y = make_circles(n_samples=n, noise=0.07, factor=0.4)\n",
    "\n",
    "A = X[np.where(Y == 0)]\n",
    "B = X[np.where(Y == 1)]\n",
    "\n",
    "X0_orig = A[:, 0]\n",
    "Y0_orig = A[:, 1]\n",
    "\n",
    "X1_orig = B[:, 0]\n",
    "Y1_orig = B[:, 1]\n",
    "\n",
    "\n",
    "\n",
    "A = np.array([transform(x,y) for x,y in zip(np.ravel(X0_orig), np.ravel(Y0_orig))])\n",
    "X0 = A[:, 0]\n",
    "Y0 = A[:, 1]\n",
    "Z0 = A[:, 2]\n",
    "\n",
    "A = np.array([transform(x,y) for x,y in zip(np.ravel(X1_orig), np.ravel(Y1_orig))])\n",
    "X1 = A[:, 0]\n",
    "Y1 = A[:, 1]\n",
    "Z1 = A[:, 2]\n",
    "\n",
    "x = np.arange(-1.25, 1.25, 0.1)\n",
    "y = np.arange(-1.25, 1.25, 0.1)\n",
    "X, Y = np.meshgrid(x, y)\n",
    "\n",
    "Z = np.zeros(X.shape)\n",
    "Z[:,:] = 0.5\n",
    "\n",
    "#Plot without Decision Boundary\n",
    "fig = plt.figure(figsize=(20,8))\n",
    "ax = fig.add_subplot(122, projection='3d')\n",
    "\n",
    "ax.scatter(X0, Y0, Z0,s=60, c='#ff7f00', marker='o')\n",
    "ax.scatter(X1, Y1, Z1,s=60, c='#377eb8', marker='^')\n",
    "\n",
    "ax.set_xlabel('X1')\n",
    "ax.set_ylabel('X2')\n",
    "ax.set_zlabel('X1^2 + X2^2')\n",
    "#ax.set_title(\"Feature space in R^3\")\n",
    "\n",
    "# Project data to X/Y plane\n",
    "#plt.figure(figsize=(10,4))\n",
    "ax2d = fig.add_subplot(121)\n",
    "ax2d.scatter(X0, Y0, c='#ff7f00', s=60,marker='o')\n",
    "ax2d.scatter(X1, Y1, c='#377eb8',s=60, marker='^')\n",
    "\n",
    "ax2d.set_xlabel('X1')\n",
    "ax2d.set_ylabel('X2')\n",
    "#ax2d.set_title(\"Input space in R^2\")\n",
    "\n",
    "plt.show()\n",
    "\n",
    "\n",
    "# Plot with Decision Boundary\n",
    "fig = plt.figure(figsize=(20,8))\n",
    "# Project data to X/Y plane\n",
    "ax2d = fig.add_subplot(121)\n",
    "ax2d.scatter(X0, Y0, c='#ff7f00', s=60,marker='o')\n",
    "ax2d.scatter(X1, Y1, c='#377eb8',s=60, marker='^')\n",
    "\n",
    "\n",
    "ax2d.add_patch(plt.Circle((0,0), radius=sqrt(0.5),\n",
    "               fill=False, linestyle='dashed', linewidth=2.5,\n",
    "               color='#dede00'))\n",
    "\n",
    "ax2d.set_xlabel('X1')\n",
    "ax2d.set_ylabel('X2')\n",
    "#ax2d.set_title(\"Input space in R^2\")\n",
    "#plt.figure(figsize=(10,4))\n",
    "ax = fig.add_subplot(122, projection='3d')\n",
    "ax.set_xlabel('X1')\n",
    "ax.set_ylabel('X2')\n",
    "ax.set_zlabel('X1^2 + X2^2')\n",
    "#ax.set_title(\"Feature space in R^3\")\n",
    "ax.scatter(X0, Y0, Z0,s=60, c='#ff7f00', marker='o')\n",
    "ax.scatter(X1, Y1, Z1,s=60, c='#377eb8', marker='^')\n",
    "ax.plot_surface(X, Y, Z, color='#dede00', alpha=0.2, antialiased=True)\n",
    "\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Optimization using Gradient Descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import stats \n",
    "\n",
    "from sklearn.datasets.samples_generator import make_regression\n",
    "\n",
    "#code adapted from http://tillbergmann.com/blog/python-gradient-descent.html\n",
    "x, y = make_regression(n_samples = 100, \n",
    "                       n_features=1, \n",
    "                       n_informative=1, \n",
    "                       noise=20,\n",
    "                       random_state=2017)\n",
    "x = x.flatten()\n",
    "slope, intercept, _,_,_ = stats.linregress(x,y)\n",
    "best_fit = np.vectorize(lambda x: x * slope + intercept)\n",
    "plt.plot(x,y, 'o', alpha=0.5)\n",
    "grid = np.arange(-3,3,0.1)\n",
    "plt.plot(grid,best_fit(grid), '.')\n",
    "\n",
    "#code adapted from https://am207.github.io/\n",
    "def gradient_descent(x, y, theta_init, step=0.001, maxsteps=0, precision=0.001, ):\n",
    "    costs = []\n",
    "    m = y.size # number of data points\n",
    "    theta = theta_init\n",
    "    history = [] # to store all thetas\n",
    "    preds = []\n",
    "    counter = 0\n",
    "    old_cost = 0\n",
    "    pred = np.dot(x, theta)\n",
    "    error = pred - y \n",
    "    current_cost = np.sum(error ** 2) / (2 * m)\n",
    "    preds.append(pred)\n",
    "    costs.append(current_cost)\n",
    "    history.append(theta)\n",
    "    counter+=1\n",
    "    while abs(current_cost - old_cost) > precision:\n",
    "        old_cost=current_cost\n",
    "        gradient = x.T.dot(error)/m \n",
    "        theta = theta - step * gradient  # update\n",
    "        history.append(theta)\n",
    "        \n",
    "        pred = np.dot(x, theta)\n",
    "        error = pred - y \n",
    "        current_cost = np.sum(error ** 2) / (2 * m)\n",
    "        costs.append(current_cost)\n",
    "        \n",
    "        if counter % 25 == 0: preds.append(pred)\n",
    "        counter+=1\n",
    "        if maxsteps:\n",
    "            if counter == maxsteps:\n",
    "                break\n",
    "        \n",
    "    return history, costs, preds, counter\n",
    "\n",
    "xaug = np.c_[np.ones(x.shape[0]), x]\n",
    "theta_i = [-15, 40] + np.random.rand(2)\n",
    "history, cost, preds, iters = gradient_descent(xaug, y, theta_i, step=0.1)\n",
    "theta = history[-1]\n",
    "\n",
    "print(\"Gradient Descent: {:.2f}, {:.2f} {:d}\".format(theta[0], theta[1], iters))\n",
    "print(\"Least Squares: {:.2f}, {:.2f}\".format(intercept, slope))\n",
    "\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "\n",
    "def error(x, y, THETA):\n",
    "    return np.sum((x.dot(THETA) - y)**2)/(2*y.size)\n",
    "\n",
    "ms = np.linspace(theta[0] - 20 , theta[0] + 20, 20)\n",
    "bs = np.linspace(theta[1] - 40 , theta[1] + 40, 40)\n",
    "\n",
    "M, B = np.meshgrid(ms, bs)\n",
    "\n",
    "zs = np.array([error(xaug, y, theta) \n",
    "               for theta in zip(np.ravel(M), np.ravel(B))])\n",
    "Z = zs.reshape(M.shape)\n",
    "\n",
    "fig = plt.figure(figsize=(10, 6))\n",
    "ax = fig.add_subplot(111, projection='3d')\n",
    "\n",
    "ax.plot_surface(M, B, Z, rstride=1, cstride=1, color='b', alpha=0.2)\n",
    "ax.contour(M, B, Z, 20, color='b', alpha=0.5, offset=0, stride=30)\n",
    "\n",
    "\n",
    "ax.set_xlabel('w0')\n",
    "ax.set_ylabel('w1')\n",
    "ax.set_zlabel('Cost')\n",
    "ax.view_init(elev=30., azim=10)\n",
    "ax.plot([history[0][0]], [history[0][1]], [cost[0]] , markerfacecolor='r', markeredgecolor='r', marker='o', markersize=7);\n",
    "\n",
    "\n",
    "ax.plot([t[0] for t in history], [t[1] for t in history], cost , markerfacecolor='r', markeredgecolor='r', marker='.', markersize=2);\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Optimization using Stochastic Gradient Descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import stats\n",
    "\n",
    "from sklearn.datasets.samples_generator import make_regression\n",
    "\n",
    "\n",
    "def sgd(x, y, theta_init, step=0.001, maxsteps=0, precision=0.001, ):\n",
    "    costs = []\n",
    "    m = y.size  # number of data points\n",
    "    old_theta = 0\n",
    "    theta = theta_init\n",
    "    history = []  # to store all thetas\n",
    "    preds = []\n",
    "    grads = []\n",
    "    counter = 0\n",
    "    old_cost = 0\n",
    "    epoch = 0\n",
    "    i = 0  # index\n",
    "    pred = np.dot(x[i, :], theta)\n",
    "    error = pred - y[i]\n",
    "    gradient = x[i, :].T * error\n",
    "    grads.append(gradient)\n",
    "    print(gradient, x[i], y[i], pred, error, np.sum(error ** 2) / 2)\n",
    "    current_cost = np.sum(error ** 2) / 2\n",
    "    counter += 1\n",
    "    preds.append(pred)\n",
    "    cost_sum = current_cost\n",
    "    costs.append(cost_sum / counter)\n",
    "    history.append(theta)\n",
    "    print(\"start\", counter, costs, old_cost)\n",
    "    while True:\n",
    "        gradient = x[i, :].T * error\n",
    "        grads.append(gradient)\n",
    "        old_theta = theta\n",
    "        theta = theta - step * gradient  # update\n",
    "        history.append(theta)\n",
    "        i += 1\n",
    "        if i == m:  # reached one past the end.\n",
    "            # break\n",
    "            epoch += 1\n",
    "            neworder = np.random.permutation(m)\n",
    "            x = x[neworder]\n",
    "            y = y[neworder]\n",
    "            i = 0\n",
    "        pred = np.dot(x[i, :], theta)\n",
    "        error = pred - y[i]\n",
    "        current_cost = np.sum(error ** 2) / 2\n",
    "\n",
    "        if counter % 25 == 0:\n",
    "            preds.append(pred)\n",
    "        counter += 1\n",
    "        cost_sum += current_cost\n",
    "        old_cost = costs[counter - 2]\n",
    "        costs.append(cost_sum / counter)\n",
    "        if maxsteps:\n",
    "            if counter == maxsteps:\n",
    "                break\n",
    "\n",
    "    return history, costs, preds, grads, counter, epoch\n",
    "\n",
    "\n",
    "def error(X, Y, THETA):\n",
    "    return np.sum((X.dot(THETA) - Y)**2) / (2 * Y.size)\n",
    "\n",
    "\n",
    "ms = np.linspace(theta[0] - 20, theta[0] + 20, 20)\n",
    "bs = np.linspace(theta[1] - 40, theta[1] + 40, 40)\n",
    "\n",
    "M, B = np.meshgrid(ms, bs)\n",
    "\n",
    "zs = np.array([error(xaug, y, theta)\n",
    "               for theta in zip(np.ravel(M), np.ravel(B))])\n",
    "Z = zs.reshape(M.shape)\n",
    "\n",
    "fig = plt.figure(figsize=(10, 6))\n",
    "ax = fig.add_subplot(111, projection='3d')\n",
    "\n",
    "ax.plot_surface(M, B, Z, rstride=1, cstride=1, color='b', alpha=0.1)\n",
    "ax.contour(M, B, Z, 20, color='b', alpha=0.5, offset=0, stride=30)\n",
    "\n",
    "history2, cost2, preds2, grads2, iters2, epoch2 = sgd(\n",
    "    xaug, y, theta_i, maxsteps=5000, step=0.01)\n",
    "\n",
    "ax.set_xlabel('w0')\n",
    "ax.set_ylabel('w1')\n",
    "ax.set_zlabel('Cost')\n",
    "ax.view_init(elev=30., azim=20)\n",
    "ax.plot([history[0][0]],\n",
    "        [history[0][1]],\n",
    "        [cost[0]],\n",
    "        markerfacecolor='r',\n",
    "        markeredgecolor='r',\n",
    "        marker='o',\n",
    "        markersize=7)\n",
    "\n",
    "ax.view_init(elev=30., azim=10)\n",
    "ax.plot([t[0] for t in history2], [t[1] for t in history2], cost2,\n",
    "        markerfacecolor='r', markeredgecolor='r', marker='.', markersize=2)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}