Cnn From scratch .ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import cv2 \n",
    "import keras\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist=keras.datasets.mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test)=keras.datasets.mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 380,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CONV_LAYER_2D():  #2d convolution layer \n",
    "    \n",
    "    \n",
    "    def __init__(self,num_filters,dim_of_filter=3):\n",
    "        self.n_filters=num_filters #no of filters declared\n",
    "        self.dim_of_filter=dim_of_filter\n",
    "        #filters are of shape (n*m*m) where \n",
    "        #m=no of dimensions of kernel, and n=num of filters \n",
    "        self.filters=np.random.randn(num_filters,dim_of_filter,dim_of_filter)/dim_of_filter*dim_of_filter #normalized filters\n",
    "        #random weights for filters\n",
    "        \n",
    "    def iterate_regions(self,image):  #generator for roi for input images\n",
    "        \n",
    "        h,w=image.shape #2d image shape\n",
    "        \n",
    "        for i in range(h-(self.dim_of_filter)+1):\n",
    "            for j in range(w-(self.dim_of_filter)+1):\n",
    "                roi=image[i:(i+self.dim_of_filter),j:(j+self.dim_of_filter)]\n",
    "\n",
    "                yield roi,i,j #roi is the region of interest,i is the row,j is the col\n",
    "                \n",
    "    def convolve(self,image): #convolving layer\n",
    "        self.last_input=image\n",
    "        h,w=image.shape\n",
    "        output_h=h-self.dim_of_filter+1\n",
    "        output_w=w-self.dim_of_filter+1\n",
    "        \n",
    "        \n",
    "        output_image=np.zeros((output_h,output_w,self.n_filters))\n",
    "        \n",
    "        for roi,i,j in self.iterate_regions(image):\n",
    "            output_image[i][j]=np.sum(roi*self.filters,axis=(1,2))\n",
    "            \n",
    "        return output_image\n",
    "    \n",
    "    \n",
    "    def backward(self,input,lr=0.01):\n",
    "        dl_dfilters=np.zeros(self.filters.shape)\n",
    "        \n",
    "        for roi,i,j in self.iterate_regions(self.last_input):\n",
    "            for f in range(self.n_filters):\n",
    "                dl_dfilters+=input[i,j,f]*roi \n",
    "                \n",
    "        self.filters-=lr*dl_dfilters\n",
    "        \n",
    "        \n",
    "        return None\n",
    "                              \n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Max_Pooling():\n",
    "    \n",
    "    def iterate_regions(self,image):\n",
    "        h,w,_=image.shape\n",
    "        \n",
    "        for i in range(h//2):\n",
    "            for j in range(w//2):\n",
    "                roi=image[(i * 2):(i * 2 + 2), (j * 2):(j * 2 + 2)]\n",
    "                yield roi,i,j\n",
    "                \n",
    "    def Pool(self,image):\n",
    "        self.last_shape=image.shape\n",
    "        self.last_input=image\n",
    "        output=np.zeros((image.shape[0]//2,image.shape[1]//2,image.shape[2]))\n",
    "        \n",
    "        for roi,i,j in self.iterate_regions(image):\n",
    "            output[i,j]=np.max(roi,axis=(0,1))\n",
    "        \n",
    "        return output\n",
    "    \n",
    "    def backward(self,input):\n",
    "        self.output=np.zeros(self.last_shape)\n",
    "        \n",
    "        for roi,i,j in self.iterate_regions(self.last_input):\n",
    "            h,w,f=roi.shape\n",
    "            maximum=np.max(roi,axis=(0,1))\n",
    "            \n",
    "            for i2 in range(h):\n",
    "                for j2 in range(w):\n",
    "                    for f2 in range(f):\n",
    "                        if roi[i2,j2,f2]==maximum[f2]:\n",
    "                            self.output[i*2+i2,j*2+j2,f2]=input[i,j,f2]\n",
    "                            \n",
    "        return self.output\n",
    "                            \n",
    "                \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(26, 26, 10)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(13, 13, 10)"
      ]
     },
     "execution_count": 382,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conv=CONV_LAYER_2D(10,3)\n",
    "output=conv.convolve(x_train[0])\n",
    "print(output.shape)\n",
    "pool=Max_Pooling()\n",
    "pool.Pool(output).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 383,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Ann():\n",
    "    \n",
    "    def __init__(self,n_nodes,n_inputs):\n",
    "        self.n_nodes=n_nodes\n",
    "        self.n_ninputs=n_inputs\n",
    "        self.weights=np.random.randn(n_nodes,n_inputs)\n",
    "        self.bias=np.zeros(n_nodes)\n",
    "        \n",
    "    def forward(self,X):\n",
    "        \n",
    "        self.last_shape=X.shape\n",
    "        \n",
    "        #excepts the input to be a multidimensional array \n",
    "        X=X.flatten()\n",
    "        \n",
    "        self.last_inputs=X\n",
    "        output=np.dot(self.weights,X)+self.bias\n",
    "        \n",
    "        self.last_output=output\n",
    "        \n",
    "        softmax_output=np.exp(output)/(np.sum(np.exp(output),axis=0))\n",
    "        \n",
    "        return softmax_output\n",
    "    \n",
    "    \n",
    "    def backward(self,loss_wrt_out,learning_rate=0.01):\n",
    "        for i,gradient in enumerate(loss_wrt_out):\n",
    "            if gradient==0:\n",
    "                continue\n",
    "                \n",
    "            t_exp=np.exp(self.last_output)\n",
    "            \n",
    "            S=np.sum(t_exp)\n",
    "            \n",
    "            #gradients\n",
    "            d_out_dt=-t_exp[i]*t_exp/(S**2)\n",
    "            d_out_dt[i]=t_exp[i]*(S-t_exp[i])/(S**2)\n",
    "            \n",
    "            \n",
    "            #gradients of params\n",
    "            dt_dw=self.last_inputs\n",
    "            dt_db=1\n",
    "            dt_dinput=self.weights\n",
    "            \n",
    "            #gradient of loss against total\n",
    "            dl_dt=gradient*d_out_dt\n",
    "            \n",
    "            #final gradients\n",
    "            dl_dw=np.dot(dt_dw[np.newaxis].T,dl_dt[np.newaxis])\n",
    "            dl_db=1*dl_dt\n",
    "            dl_dinputs=np.dot(dt_dinput.T,dl_dt)\n",
    "            \n",
    "            #update_rule\n",
    "            self.weights-=learning_rate*(dl_dw.T)\n",
    "            self.bias-=learning_rate*dl_db\n",
    "            \n",
    "            return dl_dinputs.reshape(self.last_shape)\n",
    "        \n",
    "nn=Ann(10,13*13*10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward(image,label): #forward method for a single image\n",
    "    image=image/255.0*0.5\n",
    "    out=conv.convolve(image)\n",
    "    out=pool.Pool(out)\n",
    "    \n",
    "    output=nn.forward(out)\n",
    "    \n",
    "    loss=-np.log(output[label])\n",
    "    \n",
    "    prediction=np.argmax(output)\n",
    "    \n",
    "    \n",
    "    return output,loss,prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 418,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_network(X,y,epochs=3):\n",
    "    for epoch in range(epochs):\n",
    "        print(f\"////// {epoch} //////\")\n",
    "        permutations=np.random.permutation(len(X))\n",
    "        \n",
    "        X=X[permutations]\n",
    "        y=y[permutations]\n",
    "        \n",
    "        loss_nn=0\n",
    "        predictions=[]\n",
    "        j=0\n",
    "        for image,label in zip(X,y):\n",
    "            out,loss,prediction=forward(image,label)\n",
    "\n",
    "            gradient=np.zeros(10)\n",
    "            gradient[label]=-1/out[label]\n",
    "\n",
    "            gradient=nn.backward(gradient,0.005)\n",
    "\n",
    "            gradient_2=pool.backward(gradient)\n",
    "\n",
    "            conv.backward(gradient_2,lr=0.005)\n",
    "\n",
    "            loss_nn+=loss\n",
    "            predictions.append(prediction)\n",
    "            j+=1\n",
    "\n",
    "            if j%100==0:\n",
    "                print(f'Samples Done {j}')\n",
    "\n",
    "\n",
    "        \n",
    "        \n",
    "    return loss_nn/(len(X)),np.mean(y==predictions),predictions\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 419,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "////// 0 //////\n",
      "Samples Done 100\n",
      "Samples Done 200\n",
      "Samples Done 300\n",
      "Samples Done 400\n",
      "Samples Done 500\n",
      "Samples Done 600\n",
      "Samples Done 700\n",
      "Samples Done 800\n",
      "Samples Done 900\n",
      "Samples Done 1000\n",
      "////// 1 //////\n",
      "Samples Done 100\n",
      "Samples Done 200\n",
      "Samples Done 300\n",
      "Samples Done 400\n",
      "Samples Done 500\n",
      "Samples Done 600\n",
      "Samples Done 700\n",
      "Samples Done 800\n",
      "Samples Done 900\n",
      "Samples Done 1000\n",
      "////// 2 //////\n",
      "Samples Done 100\n",
      "Samples Done 200\n",
      "Samples Done 300\n",
      "Samples Done 400\n",
      "Samples Done 500\n",
      "Samples Done 600\n",
      "Samples Done 700\n",
      "Samples Done 800\n",
      "Samples Done 900\n",
      "Samples Done 1000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1.6435449526384507,\n",
       " 0.816,\n",
       " [9,\n",
       "  2,\n",
       "  3,\n",
       "  1,\n",
       "  2,\n",
       "  0,\n",
       "  1,\n",
       "  8,\n",
       "  5,\n",
       "  8,\n",
       "  8,\n",
       "  6,\n",
       "  7,\n",
       "  9,\n",
       "  2,\n",
       "  6,\n",
       "  7,\n",
       "  7,\n",
       "  4,\n",
       "  2,\n",
       "  9,\n",
       "  0,\n",
       "  1,\n",
       "  1,\n",
       "  5,\n",
       "  1,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  5,\n",
       "  1,\n",
       "  6,\n",
       "  9,\n",
       "  1,\n",
       "  3,\n",
       "  8,\n",
       "  3,\n",
       "  4,\n",
       "  0,\n",
       "  2,\n",
       "  3,\n",
       "  1,\n",
       "  0,\n",
       "  0,\n",
       "  2,\n",
       "  2,\n",
       "  9,\n",
       "  5,\n",
       "  1,\n",
       "  3,\n",
       "  9,\n",
       "  1,\n",
       "  5,\n",
       "  0,\n",
       "  5,\n",
       "  4,\n",
       "  9,\n",
       "  1,\n",
       "  7,\n",
       "  4,\n",
       "  2,\n",
       "  5,\n",
       "  9,\n",
       "  6,\n",
       "  7,\n",
       "  2,\n",
       "  5,\n",
       "  7,\n",
       "  0,\n",
       "  1,\n",
       "  3,\n",
       "  1,\n",
       "  9,\n",
       "  1,\n",
       "  1,\n",
       "  7,\n",
       "  3,\n",
       "  2,\n",
       "  2,\n",
       "  5,\n",
       "  4,\n",
       "  4,\n",
       "  1,\n",
       "  1,\n",
       "  8,\n",
       "  5,\n",
       "  1,\n",
       "  2,\n",
       "  7,\n",
       "  6,\n",
       "  0,\n",
       "  6,\n",
       "  4,\n",
       "  1,\n",
       "  9,\n",
       "  4,\n",
       "  0,\n",
       "  4,\n",
       "  2,\n",
       "  1,\n",
       "  8,\n",
       "  4,\n",
       "  5,\n",
       "  0,\n",
       "  7,\n",
       "  7,\n",
       "  0,\n",
       "  4,\n",
       "  4,\n",
       "  1,\n",
       "  9,\n",
       "  5,\n",
       "  1,\n",
       "  4,\n",
       "  3,\n",
       "  4,\n",
       "  5,\n",
       "  6,\n",
       "  7,\n",
       "  8,\n",
       "  0,\n",
       "  3,\n",
       "  7,\n",
       "  4,\n",
       "  2,\n",
       "  4,\n",
       "  2,\n",
       "  8,\n",
       "  7,\n",
       "  7,\n",
       "  0,\n",
       "  6,\n",
       "  1,\n",
       "  3,\n",
       "  7,\n",
       "  1,\n",
       "  3,\n",
       "  4,\n",
       "  8,\n",
       "  0,\n",
       "  5,\n",
       "  0,\n",
       "  9,\n",
       "  2,\n",
       "  7,\n",
       "  3,\n",
       "  1,\n",
       "  9,\n",
       "  1,\n",
       "  9,\n",
       "  6,\n",
       "  8,\n",
       "  6,\n",
       "  0,\n",
       "  2,\n",
       "  1,\n",
       "  7,\n",
       "  7,\n",
       "  8,\n",
       "  0,\n",
       "  0,\n",
       "  3,\n",
       "  0,\n",
       "  0,\n",
       "  7,\n",
       "  5,\n",
       "  9,\n",
       "  2,\n",
       "  2,\n",
       "  7,\n",
       "  6,\n",
       "  7,\n",
       "  8,\n",
       "  6,\n",
       "  7,\n",
       "  9,\n",
       "  1,\n",
       "  1,\n",
       "  7,\n",
       "  1,\n",
       "  2,\n",
       "  6,\n",
       "  1,\n",
       "  3,\n",
       "  7,\n",
       "  6,\n",
       "  3,\n",
       "  1,\n",
       "  1,\n",
       "  6,\n",
       "  8,\n",
       "  1,\n",
       "  8,\n",
       "  2,\n",
       "  4,\n",
       "  0,\n",
       "  5,\n",
       "  4,\n",
       "  3,\n",
       "  1,\n",
       "  4,\n",
       "  4,\n",
       "  5,\n",
       "  4,\n",
       "  7,\n",
       "  8,\n",
       "  6,\n",
       "  9,\n",
       "  3,\n",
       "  8,\n",
       "  1,\n",
       "  3,\n",
       "  4,\n",
       "  0,\n",
       "  5,\n",
       "  4,\n",
       "  2,\n",
       "  6,\n",
       "  8,\n",
       "  7,\n",
       "  4,\n",
       "  0,\n",
       "  6,\n",
       "  2,\n",
       "  2,\n",
       "  1,\n",
       "  3,\n",
       "  4,\n",
       "  2,\n",
       "  3,\n",
       "  2,\n",
       "  6,\n",
       "  8,\n",
       "  4,\n",
       "  3,\n",
       "  2,\n",
       "  0,\n",
       "  6,\n",
       "  7,\n",
       "  0,\n",
       "  5,\n",
       "  9,\n",
       "  5,\n",
       "  5,\n",
       "  5,\n",
       "  4,\n",
       "  7,\n",
       "  9,\n",
       "  7,\n",
       "  7,\n",
       "  4,\n",
       "  5,\n",
       "  5,\n",
       "  6,\n",
       "  7,\n",
       "  2,\n",
       "  8,\n",
       "  0,\n",
       "  3,\n",
       "  6,\n",
       "  3,\n",
       "  3,\n",
       "  1,\n",
       "  8,\n",
       "  9,\n",
       "  6,\n",
       "  9,\n",
       "  5,\n",
       "  5,\n",
       "  2,\n",
       "  2,\n",
       "  4,\n",
       "  8,\n",
       "  6,\n",
       "  0,\n",
       "  9,\n",
       "  4,\n",
       "  3,\n",
       "  3,\n",
       "  8,\n",
       "  4,\n",
       "  4,\n",
       "  7,\n",
       "  7,\n",
       "  3,\n",
       "  7,\n",
       "  9,\n",
       "  7,\n",
       "  5,\n",
       "  8,\n",
       "  8,\n",
       "  7,\n",
       "  7,\n",
       "  2,\n",
       "  8,\n",
       "  7,\n",
       "  5,\n",
       "  6,\n",
       "  4,\n",
       "  8,\n",
       "  8,\n",
       "  6,\n",
       "  5,\n",
       "  0,\n",
       "  7,\n",
       "  9,\n",
       "  3,\n",
       "  9,\n",
       "  2,\n",
       "  5,\n",
       "  1,\n",
       "  1,\n",
       "  7,\n",
       "  0,\n",
       "  3,\n",
       "  4,\n",
       "  6,\n",
       "  2,\n",
       "  2,\n",
       "  4,\n",
       "  8,\n",
       "  3,\n",
       "  7,\n",
       "  4,\n",
       "  6,\n",
       "  1,\n",
       "  6,\n",
       "  4,\n",
       "  6,\n",
       "  1,\n",
       "  7,\n",
       "  8,\n",
       "  7,\n",
       "  5,\n",
       "  1,\n",
       "  1,\n",
       "  3,\n",
       "  3,\n",
       "  1,\n",
       "  3,\n",
       "  3,\n",
       "  4,\n",
       "  4,\n",
       "  3,\n",
       "  2,\n",
       "  0,\n",
       "  5,\n",
       "  0,\n",
       "  3,\n",
       "  7,\n",
       "  7,\n",
       "  6,\n",
       "  5,\n",
       "  8,\n",
       "  1,\n",
       "  8,\n",
       "  5,\n",
       "  0,\n",
       "  8,\n",
       "  3,\n",
       "  5,\n",
       "  6,\n",
       "  1,\n",
       "  8,\n",
       "  6,\n",
       "  7,\n",
       "  2,\n",
       "  8,\n",
       "  4,\n",
       "  6,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  6,\n",
       "  4,\n",
       "  9,\n",
       "  7,\n",
       "  2,\n",
       "  6,\n",
       "  9,\n",
       "  5,\n",
       "  5,\n",
       "  0,\n",
       "  3,\n",
       "  7,\n",
       "  8,\n",
       "  2,\n",
       "  5,\n",
       "  1,\n",
       "  8,\n",
       "  7,\n",
       "  9,\n",
       "  3,\n",
       "  1,\n",
       "  3,\n",
       "  1,\n",
       "  7,\n",
       "  0,\n",
       "  1,\n",
       "  4,\n",
       "  4,\n",
       "  1,\n",
       "  0,\n",
       "  7,\n",
       "  1,\n",
       "  1,\n",
       "  3,\n",
       "  9,\n",
       "  8,\n",
       "  2,\n",
       "  2,\n",
       "  6,\n",
       "  3,\n",
       "  2,\n",
       "  0,\n",
       "  4,\n",
       "  5,\n",
       "  2,\n",
       "  7,\n",
       "  4,\n",
       "  0,\n",
       "  2,\n",
       "  4,\n",
       "  6,\n",
       "  1,\n",
       "  3,\n",
       "  6,\n",
       "  5,\n",
       "  9,\n",
       "  0,\n",
       "  4,\n",
       "  5,\n",
       "  9,\n",
       "  9,\n",
       "  8,\n",
       "  8,\n",
       "  5,\n",
       "  0,\n",
       "  0,\n",
       "  9,\n",
       "  7,\n",
       "  0,\n",
       "  2,\n",
       "  9,\n",
       "  5,\n",
       "  7,\n",
       "  4,\n",
       "  8,\n",
       "  6,\n",
       "  9,\n",
       "  4,\n",
       "  1,\n",
       "  5,\n",
       "  2,\n",
       "  0,\n",
       "  1,\n",
       "  5,\n",
       "  4,\n",
       "  2,\n",
       "  4,\n",
       "  8,\n",
       "  4,\n",
       "  6,\n",
       "  9,\n",
       "  2,\n",
       "  7,\n",
       "  0,\n",
       "  0,\n",
       "  4,\n",
       "  3,\n",
       "  6,\n",
       "  5,\n",
       "  1,\n",
       "  7,\n",
       "  0,\n",
       "  9,\n",
       "  1,\n",
       "  9,\n",
       "  6,\n",
       "  2,\n",
       "  8,\n",
       "  2,\n",
       "  8,\n",
       "  9,\n",
       "  0,\n",
       "  3,\n",
       "  6,\n",
       "  5,\n",
       "  6,\n",
       "  4,\n",
       "  0,\n",
       "  8,\n",
       "  8,\n",
       "  0,\n",
       "  5,\n",
       "  0,\n",
       "  5,\n",
       "  3,\n",
       "  2,\n",
       "  0,\n",
       "  6,\n",
       "  8,\n",
       "  2,\n",
       "  1,\n",
       "  8,\n",
       "  4,\n",
       "  3,\n",
       "  4,\n",
       "  5,\n",
       "  8,\n",
       "  5,\n",
       "  9,\n",
       "  9,\n",
       "  5,\n",
       "  4,\n",
       "  4,\n",
       "  3,\n",
       "  1,\n",
       "  7,\n",
       "  7,\n",
       "  1,\n",
       "  0,\n",
       "  9,\n",
       "  0,\n",
       "  9,\n",
       "  4,\n",
       "  4,\n",
       "  2,\n",
       "  6,\n",
       "  1,\n",
       "  2,\n",
       "  2,\n",
       "  0,\n",
       "  9,\n",
       "  9,\n",
       "  3,\n",
       "  0,\n",
       "  5,\n",
       "  9,\n",
       "  4,\n",
       "  2,\n",
       "  3,\n",
       "  0,\n",
       "  5,\n",
       "  3,\n",
       "  4,\n",
       "  4,\n",
       "  6,\n",
       "  7,\n",
       "  8,\n",
       "  8,\n",
       "  2,\n",
       "  4,\n",
       "  5,\n",
       "  2,\n",
       "  8,\n",
       "  0,\n",
       "  8,\n",
       "  8,\n",
       "  0,\n",
       "  7,\n",
       "  2,\n",
       "  8,\n",
       "  2,\n",
       "  2,\n",
       "  4,\n",
       "  3,\n",
       "  0,\n",
       "  3,\n",
       "  3,\n",
       "  7,\n",
       "  1,\n",
       "  5,\n",
       "  2,\n",
       "  5,\n",
       "  2,\n",
       "  2,\n",
       "  7,\n",
       "  3,\n",
       "  8,\n",
       "  4,\n",
       "  1,\n",
       "  9,\n",
       "  7,\n",
       "  0,\n",
       "  1,\n",
       "  2,\n",
       "  9,\n",
       "  1,\n",
       "  6,\n",
       "  8,\n",
       "  1,\n",
       "  7,\n",
       "  4,\n",
       "  3,\n",
       "  7,\n",
       "  9,\n",
       "  2,\n",
       "  9,\n",
       "  0,\n",
       "  0,\n",
       "  0,\n",
       "  3,\n",
       "  9,\n",
       "  7,\n",
       "  1,\n",
       "  8,\n",
       "  7,\n",
       "  7,\n",
       "  5,\n",
       "  5,\n",
       "  1,\n",
       "  9,\n",
       "  2,\n",
       "  3,\n",
       "  9,\n",
       "  4,\n",
       "  7,\n",
       "  2,\n",
       "  9,\n",
       "  9,\n",
       "  6,\n",
       "  1,\n",
       "  4,\n",
       "  2,\n",
       "  0,\n",
       "  2,\n",
       "  0,\n",
       "  7,\n",
       "  2,\n",
       "  4,\n",
       "  1,\n",
       "  9,\n",
       "  0,\n",
       "  5,\n",
       "  1,\n",
       "  7,\n",
       "  9,\n",
       "  7,\n",
       "  8,\n",
       "  6,\n",
       "  4,\n",
       "  1,\n",
       "  3,\n",
       "  5,\n",
       "  6,\n",
       "  1,\n",
       "  1,\n",
       "  6,\n",
       "  7,\n",
       "  0,\n",
       "  5,\n",
       "  8,\n",
       "  2,\n",
       "  6,\n",
       "  2,\n",
       "  1,\n",
       "  1,\n",
       "  0,\n",
       "  3,\n",
       "  9,\n",
       "  8,\n",
       "  7,\n",
       "  6,\n",
       "  0,\n",
       "  6,\n",
       "  0,\n",
       "  9,\n",
       "  1,\n",
       "  9,\n",
       "  9,\n",
       "  4,\n",
       "  2,\n",
       "  2,\n",
       "  1,\n",
       "  0,\n",
       "  1,\n",
       "  2,\n",
       "  2,\n",
       "  1,\n",
       "  9,\n",
       "  1,\n",
       "  7,\n",
       "  5,\n",
       "  1,\n",
       "  1,\n",
       "  7,\n",
       "  9,\n",
       "  1,\n",
       "  6,\n",
       "  7,\n",
       "  6,\n",
       "  9,\n",
       "  7,\n",
       "  7,\n",
       "  3,\n",
       "  9,\n",
       "  0,\n",
       "  9,\n",
       "  7,\n",
       "  3,\n",
       "  6,\n",
       "  9,\n",
       "  3,\n",
       "  0,\n",
       "  0,\n",
       "  9,\n",
       "  3,\n",
       "  6,\n",
       "  9,\n",
       "  2,\n",
       "  6,\n",
       "  2,\n",
       "  2,\n",
       "  6,\n",
       "  9,\n",
       "  7,\n",
       "  6,\n",
       "  8,\n",
       "  5,\n",
       "  4,\n",
       "  4,\n",
       "  6,\n",
       "  4,\n",
       "  1,\n",
       "  7,\n",
       "  3,\n",
       "  3,\n",
       "  3,\n",
       "  5,\n",
       "  8,\n",
       "  0,\n",
       "  5,\n",
       "  1,\n",
       "  6,\n",
       "  5,\n",
       "  6,\n",
       "  8,\n",
       "  4,\n",
       "  4,\n",
       "  3,\n",
       "  1,\n",
       "  2,\n",
       "  9,\n",
       "  7,\n",
       "  2,\n",
       "  0,\n",
       "  6,\n",
       "  5,\n",
       "  8,\n",
       "  8,\n",
       "  2,\n",
       "  2,\n",
       "  8,\n",
       "  1,\n",
       "  0,\n",
       "  5,\n",
       "  2,\n",
       "  1,\n",
       "  2,\n",
       "  6,\n",
       "  8,\n",
       "  6,\n",
       "  7,\n",
       "  8,\n",
       "  2,\n",
       "  3,\n",
       "  5,\n",
       "  0,\n",
       "  0,\n",
       "  1,\n",
       "  8,\n",
       "  4,\n",
       "  2,\n",
       "  8,\n",
       "  7,\n",
       "  1,\n",
       "  3,\n",
       "  5,\n",
       "  2,\n",
       "  5,\n",
       "  4,\n",
       "  7,\n",
       "  9,\n",
       "  2,\n",
       "  3,\n",
       "  3,\n",
       "  7,\n",
       "  3,\n",
       "  9,\n",
       "  9,\n",
       "  5,\n",
       "  3,\n",
       "  9,\n",
       "  8,\n",
       "  5,\n",
       "  0,\n",
       "  4,\n",
       "  7,\n",
       "  5,\n",
       "  3,\n",
       "  2,\n",
       "  2,\n",
       "  3,\n",
       "  2,\n",
       "  3,\n",
       "  9,\n",
       "  1,\n",
       "  5,\n",
       "  6,\n",
       "  8,\n",
       "  6,\n",
       "  0,\n",
       "  5,\n",
       "  1,\n",
       "  3,\n",
       "  6,\n",
       "  5,\n",
       "  9,\n",
       "  8,\n",
       "  4,\n",
       "  6,\n",
       "  8,\n",
       "  9,\n",
       "  4,\n",
       "  7,\n",
       "  8,\n",
       "  0,\n",
       "  0,\n",
       "  7,\n",
       "  6,\n",
       "  3,\n",
       "  9,\n",
       "  6,\n",
       "  9,\n",
       "  9,\n",
       "  5,\n",
       "  0,\n",
       "  3,\n",
       "  7,\n",
       "  7,\n",
       "  8,\n",
       "  0,\n",
       "  6,\n",
       "  3,\n",
       "  5,\n",
       "  9,\n",
       "  4,\n",
       "  2,\n",
       "  8,\n",
       "  8,\n",
       "  0,\n",
       "  9,\n",
       "  6,\n",
       "  6,\n",
       "  9,\n",
       "  8,\n",
       "  9,\n",
       "  0,\n",
       "  7,\n",
       "  4,\n",
       "  9,\n",
       "  3,\n",
       "  8,\n",
       "  1,\n",
       "  0,\n",
       "  1,\n",
       "  7,\n",
       "  0,\n",
       "  1,\n",
       "  6,\n",
       "  3,\n",
       "  7,\n",
       "  3,\n",
       "  0,\n",
       "  9,\n",
       "  3,\n",
       "  0,\n",
       "  3,\n",
       "  3,\n",
       "  1,\n",
       "  5,\n",
       "  9,\n",
       "  7,\n",
       "  7,\n",
       "  3,\n",
       "  6,\n",
       "  4,\n",
       "  4,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  1,\n",
       "  9,\n",
       "  1,\n",
       "  7,\n",
       "  4,\n",
       "  7,\n",
       "  7,\n",
       "  9,\n",
       "  7,\n",
       "  6,\n",
       "  6,\n",
       "  4,\n",
       "  3,\n",
       "  3,\n",
       "  8,\n",
       "  9,\n",
       "  5,\n",
       "  4,\n",
       "  5,\n",
       "  9,\n",
       "  7,\n",
       "  1,\n",
       "  0,\n",
       "  6,\n",
       "  5,\n",
       "  2,\n",
       "  1,\n",
       "  9,\n",
       "  1,\n",
       "  1,\n",
       "  7,\n",
       "  2,\n",
       "  0,\n",
       "  4,\n",
       "  0,\n",
       "  1,\n",
       "  8,\n",
       "  4,\n",
       "  4,\n",
       "  8,\n",
       "  8,\n",
       "  6,\n",
       "  4,\n",
       "  3,\n",
       "  8,\n",
       "  8,\n",
       "  6,\n",
       "  6,\n",
       "  9,\n",
       "  6,\n",
       "  0,\n",
       "  9,\n",
       "  7,\n",
       "  5,\n",
       "  1,\n",
       "  7,\n",
       "  9,\n",
       "  6,\n",
       "  0,\n",
       "  2,\n",
       "  0,\n",
       "  0,\n",
       "  6,\n",
       "  5,\n",
       "  8,\n",
       "  5,\n",
       "  5,\n",
       "  4,\n",
       "  8,\n",
       "  9,\n",
       "  9,\n",
       "  7,\n",
       "  1,\n",
       "  7,\n",
       "  4,\n",
       "  4,\n",
       "  7,\n",
       "  5,\n",
       "  4,\n",
       "  2,\n",
       "  8,\n",
       "  7,\n",
       "  2,\n",
       "  4,\n",
       "  3,\n",
       "  9,\n",
       "  2,\n",
       "  8,\n",
       "  1,\n",
       "  4,\n",
       "  9,\n",
       "  7,\n",
       "  4,\n",
       "  9,\n",
       "  9,\n",
       "  4,\n",
       "  0,\n",
       "  6,\n",
       "  6,\n",
       "  3,\n",
       "  2,\n",
       "  9,\n",
       "  6,\n",
       "  6,\n",
       "  5,\n",
       "  4,\n",
       "  8])"
      ]
     },
     "execution_count": 419,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_network(x_train[:1000],y_train[:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 426,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x20b4911a808>"
      ]
     },
     "execution_count": 426,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP0AAAD4CAYAAAAn+OBPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAO+0lEQVR4nO3dW4xd5XnG8efBjG3GNvb4NIwGQ9JwtBA19QhVokJUoZGDkEwuUoWLyLVQnYsgJVIuiuhFuERVkygXVSSnQJwqJYqUIHyB2lgoElRCwYNlfKhb7IJLxjPM+ICN7fF53l7McjoynvUtz177MPP9f9Jo71nvmrXe2Z7Ha++9vv0tR4QA5OOmdjcAoLUIPZAZQg9khtADmSH0QGZubuXOuru7Y+nSpa3cJZCVU6dOaXx83GXrNBR62xsk/VjSPEn/HBEvlq2/dOlSbd68uZFdAijxyiuvJNeZ8dN72/Mk/ZOkr0paK+lp22tnuj0ArdHIa/qHJR2KiA8j4qKkX0raWE9bAJqlkdD3S/rDlO+HimUAOlgjob/emwWfG9Nre4vtQduD4+PjDewOQB0aCf2QpDVTvr9d0vC1K0XE1ogYiIiB7u7uBnYHoA6NhH6npLttf9H2fEnfkLS9nrYANMuMT9lFxGXbz0r6d02esns5IvbX1hmApmjoPH1EvCHpjZp6AdACDMMFMkPogcwQeiAzhB7IDKEHMkPogcwQeiAzhB7IDKEHMkPogcwQeiAzhB7IDKEHMkPogcwQeiAzLb3YBdAst912W2n98ccfT27jgw8+KK0PDQ2V1sfGxpL7uHz5cnKdZuNID2SG0AOZIfRAZgg9kBlCD2SG0AOZIfRAZgg9kJlZNzhn9erVpfVly5Ylt3Hs2LHSeupCm+fPn0/uA62Vuk7iu+++m9xGavDN2rVrS+snT55M7oPBOQBajtADmSH0QGYIPZAZQg9khtADmSH0QGZm3Xn6pUuXNlSXpHPnzpXWU+d8U+dz0Xqpf7Ourq7kNg4ePFhaX7JkSWl9YmIiuY9O0FDobR+WdFrSFUmXI2KgjqYANE8dR/q/jIjyIW4AOgav6YHMNBr6kPRb2+/Z3nK9FWxvsT1oezA1ph1A8zX69P6RiBi2vVrSDtv/FRFvTV0hIrZK2ipJfX190eD+ADSooSN9RAwXt2OSXpP0cB1NAWieGYfe9iLbS67el/QVSfvqagxAczTy9L5X0mu2r27nXyPi32rpqsSaNWtK6x999FFyG6Ojo6X1O++884Z6QvOlzsPfc889pfW33347uY+I8lefV65caajeKWYc+oj4UNKf1tgLgBbglB2QGUIPZIbQA5kh9EBmCD2QGUIPZIbQA5mZdZNopC408fHHHye3ceHChdL6pUuXbqgnNN/69etL6wsWLCitV7kQRWoCltTfRervSpJuuqn9x9n2dwCgpQg9kBlCD2SG0AOZIfRAZgg9kBlCD2Smo87T9/T0JNfp7e0trR8/fjy5jYULF1buCZ3h9OnTpfXUBUiGh4eT+0hN1JE6x94J5+CrmB1dAqgNoQcyQ+iBzBB6IDOEHsgMoQcyQ+iBzHTUefqVK1cm10mdC120aFHD20Br3XLLLcl1+vr6Suu7du0qrV+8eDG5jzvuuKO0fubMmeQ2ZgP++oHMEHogM4QeyAyhBzJD6IHMEHogM4QeyAyhBzLTUYNzUhcskKSjR4+W1s+ePdvwfq5cuZLcBurz4IMPJtdZtWpVaT11IYoqg3NWrFhRWv/kk0+S25gNkkd62y/bHrO9b8qy5bZ32D5Y3KanvAHQEao8vf+ZpA3XLHtO0psRcbekN4vvAcwCydBHxFuSTlyzeKOkbcX9bZKeqrkvAE0y0zfyeiNiRJKK29XTrWh7i+1B24Pj4+Mz3B2AujT93fuI2BoRAxExkJptFEDzzTT0o7b7JKm4HauvJQDNNNPQb5e0qbi/SdLr9bQDoNmS5+ltvyrpMUkrbQ9J+r6kFyX9yvYzkj6W9PU6mrGdXOfIkSOl9SoTHaQudjFXJkuoQ1dXV2n99ttvT27jgQceKK2nJsiQpM8++6y0njqHfurUqeQ+zp07V1q/fPlychuzQTL0EfH0NKUv19wLgBZgGC6QGUIPZIbQA5kh9EBmCD2QGUIPZIbQA5npqEk05s2bl1wnNUlGlYk4UoNBUhMyVNHTUz7FQGqAUH9/f3IfqSsCLV68OLmNu+66q7S+ZMmS0vqFCxeS+9i7d29pvcpgqNSgrNS/WZVBRCdPnkyuMxdwpAcyQ+iBzBB6IDOEHsgMoQcyQ+iBzBB6IDMddZ6+ykUmIqK0XuXCCZs3by6tp85vVzmvnJoPMPW7VplQ5NixY6X1Kn2+8847pfX333+/tP7pp58m9zE8PFxaf+ihh5LbSE1wMTo6Wlpfv359ch9VJtqYCzjSA5kh9EBmCD2QGUIPZIbQA5kh9EBmCD2QmY46T79z587kOocOHSqtV7kgwc03l//a9913X3IbKRMTE6X1Xbt2ldarnP/es2dPab3K58NTn5dPXWQi9VhK0v33399QD5J0+PDh0nrq8V6xYkVyH8ePHy+t33TT3DhGzo3fAkBlhB7IDKEHMkPogcwQeiAzhB7IDKEHMkPogcx01OCc1MUbJGloaKi0vmPHjuQ2UoOAli9fXlqvMkgjNWglNYioymORujhI6oIaUvpiFatWrSqtV7mgxsDAQGk9NXhHkvbv319aT13sIjX5ijR3Bt+kJH9L2y/bHrO9b8qyF2wfsb27+HqiuW0CqEuV/9p+JmnDdZb/KCLWFV9v1NsWgGZJhj4i3pJ0ogW9AGiBRl7EPGt7T/H0f9qrNdreYnvQ9uD4+HgDuwNQh5mG/ieSviRpnaQRST+YbsWI2BoRAxExkJohFkDzzSj0ETEaEVciYkLSTyU9XG9bAJplRqG3PfUC71+TtG+6dQF0luR5etuvSnpM0krbQ5K+L+kx2+skhaTDkr5VRzNVLs6QuphF6hy7JI2NjZXWq1x0o1Gp8+NVfo/UOfKenmnfavmj1Ln+OqT6PHjwYHIbJ06Uv5ec2kdqgoycJEMfEU9fZ/FLTegFQAvkMQQJwB8ReiAzhB7IDKEHMkPogcwQeiAzhB7ITEdNolFF6kM7VQa1VFkH9enq6iqtj4yMNLyPe++9t7R+7ty5hvcxV3CkBzJD6IHMEHogM4QeyAyhBzJD6IHMEHogM7PuPD3mHtvJdRYsWFBa7+/vL62nJk6R0hObzBUc6YHMEHogM4QeyAyhBzJD6IHMEHogM4QeyAzn6dF2EZFc59KlS6X11Dn2+fPn31BPcxlHeiAzhB7IDKEHMkPogcwQeiAzhB7IDKEHMkPogcwwOAezQm9vb2l9YmKitD5v3rw625nVkkd622ts/872Adv7bX+nWL7c9g7bB4vbnua3C6BRVZ7eX5b0vYi4X9KfS/q27bWSnpP0ZkTcLenN4nsAHS4Z+ogYiYhdxf3Tkg5I6pe0UdK2YrVtkp5qVpMA6nNDb+TZ/oKkhyT9XlJvRIxIk/8xSFo9zc9ssT1oezB18UkAzVc59LYXS/q1pO9GxGdVfy4itkbEQEQMdHd3z6RHADWqFHrbXZoM/C8i4jfF4lHbfUW9T1J6jmEAbVfl3XtLeknSgYj44ZTSdkmbivubJL1ef3sA6lblPP0jkr4paa/t3cWy5yW9KOlXtp+R9LGkrzenRcx1VS52kZok48yZM6X1nh7OKF+VDH1E/Iek6f5VvlxvOwCajWG4QGYIPZAZQg9khtADmSH0QGYIPZAZPk+PphseHi6tp86xS+nPy588ebK0fuuttyb3kQuO9EBmCD2QGUIPZIbQA5kh9EBmCD2QGUIPZIbQA5lhcA6abmRkpLS+ffv25Db2799fWn/yySdL60yi8f840gOZIfRAZgg9kBlCD2SG0AOZIfRAZgg9kBnO06PpFi5cWFrfsGFDchvLli0rrZ89e7a0nhorIKX7nCs40gOZIfRAZgg9kBlCD2SG0AOZIfRAZgg9kBlCD2QmOTjH9hpJP5d0m6QJSVsj4se2X5D0t5KOFqs+HxFvNKtRzF7nz58vrc+fPz+5jUcffbSudrJXZUTeZUnfi4hdtpdIes/2jqL2o4j4x+a1B6BuydBHxIikkeL+adsHJPU3uzEAzXFDr+ltf0HSQ5J+Xyx61vYe2y/bZhIyYBaoHHrbiyX9WtJ3I+IzST+R9CVJ6zT5TOAH0/zcFtuDtgfHx8draBlAIyqF3naXJgP/i4j4jSRFxGhEXImICUk/lfTw9X42IrZGxEBEDHR3d9fVN4AZSobetiW9JOlARPxwyvK+Kat9TdK++tsDULcq794/Iumbkvba3l0se17S07bXSQpJhyV9qykdAqiVI6J1O7OPSvrfKYtWSjrWsgZmjj7rNRv6nA09Sp/v886IWFX2Ay0N/ed2bg9GxEDbGqiIPus1G/qcDT1KM+uTYbhAZgg9kJl2h35rm/dfFX3Wazb0ORt6lGbQZ1tf0wNovXYf6QG0GKEHMtO20NveYPu/bR+y/Vy7+kixfdj2Xtu7bQ+2u5+rig85jdneN2XZcts7bB8sbtv6IahpenzB9pHi8dxt+4l29lj0tMb272wfsL3f9neK5Z32eE7X5w09pm15TW97nqQPJP2VpCFJOyU9HRH/2fJmEmwfljQQER01UMP2o5LOSPp5RDxQLPsHSSci4sXiP9KeiPi7DuvxBUlnOmkehmJIed/UOSMkPSXpb9RZj+d0ff61buAxbdeR/mFJhyLiw4i4KOmXkja2qZdZKSLeknTimsUbJW0r7m/T5B9E20zTY8eJiJGI2FXcPy3p6pwRnfZ4TtfnDWlX6Psl/WHK90Pq3Ik5QtJvbb9ne0u7m0noLSY9uTr5yeo29zOdjp2H4Zo5Izr28Wxkbot2hd7XWdap5w4fiYg/k/RVSd8unrJi5irNw9AO15kzoiPNdG6Lq9oV+iFJa6Z8f7uk4Tb1UioihovbMUmvaZp5AzrE6NWPPBe3Y23u53OqzsPQatebM0Id+Hg2MrfFVe0K/U5Jd9v+ou35kr4haXubepmW7UXFGyayvUjSV9TZ8wZsl7SpuL9J0utt7OW6OnEehunmjFCHPZ61zW0REW35kvSEJt/B/x9Jf9+uPhI9/omk94uv/Z3Up6RXNflU7pImnzk9I2mFpDclHSxul3dgj/8iaa+kPZoMVV8HPJZ/ocmXl3sk7S6+nujAx3O6Pm/oMWUYLpAZRuQBmSH0QGYIPZAZQg9khtADmSH0QGYIPZCZ/wPUoaynLhDM/QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(conv.convolve(x_train[2])[:,:,3],cmap='gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}