Digit_recognizer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Importing All the necessary libraries**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Loading the MNIST Data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets(\"MNIST_data\", one_hot = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(55000, 784)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.train.images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(55000, 10)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.train.labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 784)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.test.images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 10)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.test.labels.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Initializing some variables**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_input = 784\n",
    "n_classes = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_nodes_hidden1 = 256\n",
    "n_nodes_hidden2 = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_samples = mnist.train.num_examples\n",
    "n_group = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.001\n",
    "training_epoches = 50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Initializing the weights and biases with values from normal distribution**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = {\n",
    "    'h1' : tf.Variable(tf.random_normal([n_input, n_nodes_hidden1]), name='h1'),\n",
    "    'h2' : tf.Variable(tf.random_normal([n_nodes_hidden1, n_nodes_hidden2]), name='h2'),\n",
    "    'out_w' : tf.Variable(tf.random_normal([n_nodes_hidden2, n_classes]), name='out_w')\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "biases = {\n",
    "    'bias1' : tf.Variable(tf.random_normal([n_nodes_hidden1]), name='bias1'),\n",
    "    'bias2' : tf.Variable(tf.random_normal([n_nodes_hidden2]), name='bias2'),\n",
    "    'out_bias' : tf.Variable(tf.random_normal([n_classes]), name='out_bias')\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Function for the Neural Network**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multilayer_perceptron(x, weights, biases):\n",
    "    # First Layer\n",
    "    net_i = tf.add(tf.matmul(x, weights['h1']), biases['bias1'])\n",
    "    layer1_out = tf.nn.relu(net_i)\n",
    "    \n",
    "    # Second Layer\n",
    "    net_i = tf.add(tf.matmul(layer1_out, weights['h2']), biases['bias2'])\n",
    "    layer2_out = tf.nn.relu(net_i)\n",
    "    \n",
    "    # Output Layer\n",
    "    output_out = tf.add(tf.matmul(layer1_out, weights['out_w']), biases['out_bias'])\n",
    "    \n",
    "    return output_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = tf.placeholder(dtype= 'float', shape= [None, n_input])\n",
    "y = tf.placeholder(dtype= 'float', shape= [None, n_classes])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction = multilayer_perceptron(x, weights, biases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = prediction, labels = y))\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "initialize = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.run(initialize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_groups = n_samples/n_group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1100.0"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch no. 1, Cost = 0.0\n",
      "Epoch no. 2, Cost = 0.0\n",
      "Epoch no. 3, Cost = 0.0\n",
      "Epoch no. 4, Cost = 0.0\n",
      "Epoch no. 5, Cost = 0.0\n",
      "Epoch no. 6, Cost = 0.0\n",
      "Epoch no. 7, Cost = 0.0\n",
      "Epoch no. 8, Cost = 0.0\n",
      "Epoch no. 9, Cost = 0.0\n",
      "Epoch no. 10, Cost = 0.0\n",
      "Epoch no. 11, Cost = 0.0\n",
      "Epoch no. 12, Cost = 0.0\n",
      "Epoch no. 13, Cost = 0.0\n",
      "Epoch no. 14, Cost = 0.0\n",
      "Epoch no. 15, Cost = 0.0\n",
      "Epoch no. 16, Cost = 0.0\n",
      "Epoch no. 17, Cost = 0.0\n",
      "Epoch no. 18, Cost = 0.0\n",
      "Epoch no. 19, Cost = 0.0\n",
      "Epoch no. 20, Cost = 0.0\n",
      "Epoch no. 21, Cost = 0.0\n",
      "Epoch no. 22, Cost = 0.0\n",
      "Epoch no. 23, Cost = 0.0\n",
      "Epoch no. 24, Cost = 0.0\n",
      "Epoch no. 25, Cost = 0.0\n",
      "Epoch no. 26, Cost = 0.0\n",
      "Epoch no. 27, Cost = 0.0\n",
      "Epoch no. 28, Cost = 0.0\n",
      "Epoch no. 29, Cost = 0.0\n",
      "Epoch no. 30, Cost = 0.0\n",
      "Epoch no. 31, Cost = 0.0\n",
      "Epoch no. 32, Cost = 0.0\n",
      "Epoch no. 33, Cost = 0.0\n",
      "Epoch no. 34, Cost = 0.0\n",
      "Epoch no. 35, Cost = 0.0\n",
      "Epoch no. 36, Cost = 0.0\n",
      "Epoch no. 37, Cost = 0.0\n",
      "Epoch no. 38, Cost = 0.0\n",
      "Epoch no. 39, Cost = 0.0\n",
      "Epoch no. 40, Cost = 0.0\n",
      "Epoch no. 41, Cost = 0.0\n",
      "Epoch no. 42, Cost = 0.0\n",
      "Epoch no. 43, Cost = 0.0\n",
      "Epoch no. 44, Cost = 0.0\n",
      "Epoch no. 45, Cost = 0.0\n",
      "Epoch no. 46, Cost = 0.0\n",
      "Epoch no. 47, Cost = 0.0\n",
      "Epoch no. 48, Cost = 0.0\n",
      "Epoch no. 49, Cost = 0.0\n",
      "Epoch no. 50, Cost = 0.0\n",
      "Model has been completed with 50 epochs of training.\n"
     ]
    }
   ],
   "source": [
    "for epochs in range(training_epoches):\n",
    "    avg_cost = 0.0\n",
    "    total_groups = int(n_samples/n_group)\n",
    "    for group in range(total_groups):\n",
    "        group_x, group_y = mnist.train.next_batch(n_group)\n",
    "        _,cos = sess.run([optimizer,cost], feed_dict={x : group_x, y : group_y})\n",
    "        \n",
    "        avg_cost += cos/total_groups\n",
    "    \n",
    "    print(f\"Epoch no. {epochs+1}, Cost = {avg_cost}\")\n",
    "print(f\"Model has been completed with {epochs+1} epochs of training.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Evaluation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "correct_prediction = tf.equal(tf.argmax(prediction,1), tf.argmax(y, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "correct_prediction = tf.cast(correct_prediction, 'float')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy = tf.reduce_mean(correct_prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9749"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy.eval(feed_dict = {x : mnist.test.images, y : mnist.test.labels})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Saving and Restoring the model in Tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./Digit Model/Digit Model'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver.save(sess, \"./Digit Model/Digit Model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Restoring**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./Digit Model/Digit Model\n"
     ]
    }
   ],
   "source": [
    "saver.restore(sess, \"./Digit Model/Digit Model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9749"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy.eval(feed_dict = {x : mnist.test.images, y : mnist.test.labels})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Saving and restoring the weights and biases as a list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = [{},{}]\n",
    "temp[0]['h1'] = weights['h1'].eval()\n",
    "temp[0]['h2'] = weights['h2'].eval()\n",
    "temp[0]['out_w'] = weights['out_w'].eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp[1]['bias1'] = biases['bias1'].eval()\n",
    "temp[1]['bias2'] = biases['bias2'].eval()\n",
    "temp[1]['out_bias'] = biases['out_bias'].eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['weight_as_list/weights_biases.pkl']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.externals import joblib\n",
    "joblib.dump(temp, 'weight_as_list/weights_biases.pkl') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Restoring**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.externals import joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = joblib.load('weight_as_list/weights_biases.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Saving and Restoring the weights and biases as numpy array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt('weights_as_np_array/h1.txt',weights['h1'].eval())\n",
    "np.savetxt('weights_as_np_array/h2.txt',weights['h2'].eval())\n",
    "np.savetxt('weights_as_np_array/out_w.txt',weights['out_w'].eval())\n",
    "\n",
    "np.savetxt('weights_as_np_array/bias1.txt',biases['bias1'].eval())\n",
    "np.savetxt('weights_as_np_array/bias2.txt',biases['bias2'].eval())\n",
    "np.savetxt('weights_as_np_array/out_bias.txt',biases['out_bias'].eval())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Restoring**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_weights = {}\n",
    "num_weights['h1'] = np.loadtxt('weights_as_np_array/h1.txt')\n",
    "num_weights['h2'] = np.loadtxt('weights_as_np_array/h2.txt')\n",
    "num_weights['out_w'] = np.loadtxt('weights_as_np_array/out_w.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_bias = {}\n",
    "num_bias['bias1'] = np.loadtxt('weights_as_np_array/bias1.txt')\n",
    "num_bias['bias2'] = np.loadtxt('weights_as_np_array/bias2.txt')\n",
    "num_bias['out_bias'] = np.loadtxt('weights_as_np_array/out_bias.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 97.49% Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}