CNNs with Noisy Labels.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This code is to test out whether a CNN that is trained on noisy labels is still able to acheive good accuracy. The experiment was talked about in this [paper](https://arxiv.org/pdf/1703.08774.pdf) and summarized in the below screenshot. I thought this was a really interesting paper, and wanted to try it out for myself since I was pretty surprised about the results. \n",
    "\n",
    "![](Data/Noisy.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import random\n",
    "import numpy as np\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n",
    "tf.reset_default_graph() \n",
    "sess = tf.InteractiveSession()\n",
    "x = tf.placeholder(\"float\", shape = [None, 784]) \n",
    "y_ = tf.placeholder(\"float\", shape = [None, 10]) \n",
    "x_image = tf.reshape(x, [-1,28,28,1])\n",
    "W_conv1 = tf.Variable(tf.truncated_normal([5, 5, 1, 32], stddev=0.1))\n",
    "b_conv1 = tf.Variable(tf.constant(.1, shape = [32]))\n",
    "h_conv1 = tf.nn.conv2d(input=x_image, filter=W_conv1, strides=[1, 1, 1, 1], padding='SAME') + b_conv1\n",
    "h_conv1 = tf.nn.relu(h_conv1)\n",
    "h_pool1 = tf.nn.max_pool(h_conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def conv2d(x, W):\n",
    "  return tf.nn.conv2d(input=x, filter=W, strides=[1, 1, 1, 1], padding='SAME')\n",
    "\n",
    "def max_pool_2x2(x):\n",
    "  return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Second Conv and Pool Layers\n",
    "W_conv2 = tf.Variable(tf.truncated_normal([5, 5, 32, 64], stddev=0.1))\n",
    "b_conv2 = tf.Variable(tf.constant(.1, shape = [64]))\n",
    "h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)\n",
    "h_pool2 = max_pool_2x2(h_conv2)\n",
    "\n",
    "#First Fully Connected Layer\n",
    "W_fc1 = tf.Variable(tf.truncated_normal([7 * 7 * 64, 1024], stddev=0.1))\n",
    "b_fc1 = tf.Variable(tf.constant(.1, shape = [1024]))\n",
    "h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])\n",
    "h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)\n",
    "\n",
    "#Dropout Layer\n",
    "keep_prob = tf.placeholder(\"float\")\n",
    "h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "#Second Fully Connected Layer\n",
    "W_fc2 = tf.Variable(tf.truncated_normal([1024, 10], stddev=0.1))\n",
    "b_fc2 = tf.Variable(tf.constant(.1, shape = [10]))\n",
    "\n",
    "#Softmax Layer\n",
    "y_conv=tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)\n",
    "\n",
    "#Cross Entropy Loss Function\n",
    "cross_entropy = -tf.reduce_sum(y_*tf.log(y_conv))\n",
    "#Mean Squared Error Loss Function\n",
    "MSE = tf.reduce_sum(tf.square(y_ - y_conv))\n",
    "train_step = tf.train.AdamOptimizer().minimize(MSE)\n",
    "correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Noisy Labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here, we want to see whether changing labels with a probability of .5 during training affects test accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "changeLabelProbability = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.summary.scalar('MSE_Loss', MSE)\n",
    "tf.summary.scalar('Accuracy', accuracy)\n",
    "merged = tf.summary.merge_all()\n",
    "logdir = \"tensorboard/\" + \"ChangeLabelProb:\" + str(changeLabelProbability) + \"/\"\n",
    "writer = tf.summary.FileWriter(logdir, sess.graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 0, training accuracy 0.04\n",
      "step 100, training accuracy 0.1\n",
      "step 200, training accuracy 0.18\n",
      "step 300, training accuracy 0.12\n",
      "step 400, training accuracy 0.7\n",
      "step 500, training accuracy 0.38\n",
      "step 600, training accuracy 0.74\n",
      "step 700, training accuracy 0.9\n",
      "step 800, training accuracy 0.72\n",
      "step 900, training accuracy 0.92\n",
      "step 1000, training accuracy 0.88\n",
      "step 1100, training accuracy 0.52\n",
      "step 1200, training accuracy 0.54\n",
      "step 1300, training accuracy 0.56\n",
      "step 1400, training accuracy 0.46\n",
      "step 1500, training accuracy 0.46\n",
      "step 1600, training accuracy 0.4\n",
      "step 1700, training accuracy 0.46\n",
      "step 1800, training accuracy 0.5\n",
      "step 1900, training accuracy 0.48\n"
     ]
    }
   ],
   "source": [
    "batchSize = 50\n",
    "\n",
    "for i in range(2000):\n",
    "    batch = mnist.train.next_batch(batchSize)\n",
    "    if i%10 == 0:\n",
    "        summary = sess.run(merged, {x: batch[0], y_: batch[1], keep_prob: 1.0})\n",
    "        writer.add_summary(summary, i)\n",
    "    if i%100 == 0:\n",
    "        train_accuracy = accuracy.eval(session=sess, feed_dict={x:batch[0], y_: batch[1], keep_prob: 1.0})\n",
    "        print (\"step %d, training accuracy %g\"%(i, train_accuracy))\n",
    "        \n",
    "    # NOISY LABELS\n",
    "    if (random.random() < changeLabelProbability):\n",
    "        for i in range(batchSize):\n",
    "            originalTrainingLabel = np.argmax(batch[1][i])\n",
    "            validOtherChoices = list(range(0,originalTrainingLabel)) + list(range(originalTrainingLabel+1,10))\n",
    "            newTrainingLabel = random.choice(validOtherChoices)\n",
    "            batch[1][i] = np.zeros(10)\n",
    "            batch[1][i][newTrainingLabel] = 1\n",
    "    train_step.run(session=sess, feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.2})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.94\n",
      "0.88\n",
      "0.92\n",
      "0.98\n",
      "0.92\n",
      "0.9\n",
      "0.9\n",
      "0.96\n",
      "0.9\n",
      "0.88\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    batch = mnist.test.next_batch(50)\n",
    "    print (sess.run(accuracy, feed_dict={x: batch[0], y_: batch[1], keep_prob: 1}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:py3]",
   "language": "python",
   "name": "conda-env-py3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}