multi-layer-neural-network-with-tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "class NeuralNetworkLayer(object):\n",
    "    '''\n",
    "    A Neural Network Layer\n",
    "    '''\n",
    "    def __init__(self, input_size, output_size, name=None, tf_session=None):\n",
    "        self.number_of_neurons = output_size\n",
    "        self.inputs_per_neuron = input_size\n",
    "        self.name = name\n",
    "        prefix = self.name+'.' if self.name != None else 'layer_'\n",
    "        self.synaptic_weights = tf.Variable(tf.random_normal([input_size, output_size]), name=prefix+'synaptic_weights')\n",
    "        self.biases = tf.Variable(tf.random_normal([output_size]), name=prefix+'biases')\n",
    "        self.tf_session = tf_session\n",
    "    \n",
    "    def set_tf_session(self, tf_session):\n",
    "        self.tf_session = tf_session\n",
    "    \n",
    "    def __repr__(self):\n",
    "        if(self.tf_session == None):\n",
    "            return str({\n",
    "                'number_of_neurons': self.number_of_neurons, 'inputs_per_neuron': self.inputs_per_neuron, 'name': self.name\n",
    "            })\n",
    "        else:\n",
    "            return str({\n",
    "                'number_of_neurons': self.number_of_neurons, 'inputs_per_neuron': self.inputs_per_neuron, 'name': self.name,\n",
    "                'synaptic_weights': self.synaptic_weights.eval(session=self.tf_session),\n",
    "                'biases': self.biases.eval(session=self.tf_session)\n",
    "            })\n",
    "\n",
    "class MultiLayerNeuralNetwork(object):\n",
    "    '''\n",
    "    A MultiLayer Neural Network\n",
    "    '''\n",
    "    def __init__(self, layer_sizes, input_type, output_type):\n",
    "        # Placeholders\n",
    "        self.x_ = tf.placeholder(input_type, [None, layer_sizes[0]], name='x_')\n",
    "        self.y_ = tf.placeholder(output_type, [None, layer_sizes[-1]], name='y_')\n",
    "        # Neural Network Layers\n",
    "        self.layers = [ NeuralNetworkLayer(input_size, output_size, name='layer_{}'.format(i)) \n",
    "                       for (i, (input_size, output_size)) in enumerate(zip(layer_sizes[:-1], layer_sizes[1:])) ]\n",
    "        # Prediction, loss and optimizer\n",
    "        self.prediction = self.predict(self.x_)[-1]\n",
    "        #self.loss = tf.reduce_mean(tf.squared_difference(self.y_, self.prediction))\n",
    "        #self.optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01).minimize(self.loss)\n",
    "    \n",
    "    def set_training_params(self, loss_function, optimizer):\n",
    "        self.loss = loss_function(self.y_, self.prediction)\n",
    "        self.optimizer = optimizer.minimize(self.loss)\n",
    "    \n",
    "    def predict(self, inputs):\n",
    "        '''\n",
    "        Returns the list of the outputs from each layer (from first to last)\n",
    "        '''\n",
    "        layer_outputs_stack = []\n",
    "        layer_inputs = inputs # layer_inputs for next iteration\n",
    "        for layer in self.layers:\n",
    "            layer_outputs = tf.add(tf.matmul(layer_inputs, layer.synaptic_weights), layer.biases)\n",
    "            layer_outputs = tf.nn.sigmoid(layer_outputs)\n",
    "            layer_outputs_stack.append(layer_outputs)\n",
    "            layer_inputs = layer_outputs\n",
    "        return layer_outputs_stack\n",
    "    \n",
    "    def set_tf_session(self, tf_session):\n",
    "        for layer in self.layers:\n",
    "            layer.set_tf_session(tf_session)\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return str(self.layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model before initializing weights\n",
      "[{'number_of_neurons': 4, 'inputs_per_neuron': 3, 'name': 'layer_0'}, {'number_of_neurons': 3, 'inputs_per_neuron': 4, 'name': 'layer_1'}, {'number_of_neurons': 1, 'inputs_per_neuron': 3, 'name': 'layer_2'}]\n",
      "[{'number_of_neurons': 4, 'inputs_per_neuron': 3, 'biases': array([ 0.86882907,  1.82817304, -0.79447335,  0.62770885], dtype=float32), 'name': 'layer_0', 'synaptic_weights': array([[-0.75024509, -0.30697507, -0.37605351, -1.88932753],\n",
      "       [ 0.34027651,  1.36733377,  0.36677638,  0.28121865],\n",
      "       [-0.7164005 , -0.05920342, -0.29389295,  0.40518087]], dtype=float32)}, {'number_of_neurons': 3, 'inputs_per_neuron': 4, 'biases': array([-1.16655111,  0.31444797, -1.38888073], dtype=float32), 'name': 'layer_1', 'synaptic_weights': array([[ 0.04697009, -1.58462417, -2.65292788],\n",
      "       [ 1.14246023,  1.06249595,  1.42925096],\n",
      "       [-0.24149585, -0.75541091, -1.01969469],\n",
      "       [-0.55983084, -0.29572338,  0.07365815]], dtype=float32)}, {'number_of_neurons': 1, 'inputs_per_neuron': 3, 'biases': array([ 1.59209383], dtype=float32), 'name': 'layer_2', 'synaptic_weights': array([[-1.01731014],\n",
      "       [-0.43706268],\n",
      "       [-0.05710965]], dtype=float32)}]\n",
      "Training...\n",
      "Epoch: 0, loss=0.27012452483177185\n",
      "Epoch: 1000, loss=0.0005827572313137352\n",
      "Epoch: 2000, loss=0.0001536105846753344\n",
      "Epoch: 3000, loss=6.548607780132443e-05\n",
      "Epoch: 4000, loss=3.305829159216955e-05\n",
      "Epoch: 5000, loss=1.8076138076139614e-05\n",
      "Epoch: 6000, loss=1.031437477649888e-05\n",
      "Epoch: 7000, loss=6.029473752278136e-06\n",
      "Epoch: 8000, loss=3.575253003873513e-06\n",
      "Epoch: 9000, loss=2.1381165424827486e-06\n",
      "Epoch: 10000, loss=1.2856168041253113e-06\n",
      "Epoch: 11000, loss=7.7540317988678e-07\n",
      "Epoch: 12000, loss=4.687048544838035e-07\n",
      "Epoch: 13000, loss=2.8368995685923437e-07\n",
      "Epoch: 14000, loss=1.7188826006986346e-07\n",
      "Epoch: 15000, loss=1.0425636531863347e-07\n",
      "Epoch: 16000, loss=6.325303303356122e-08\n",
      "Epoch: 17000, loss=3.842339779680515e-08\n",
      "Epoch: 18000, loss=2.3370025914459802e-08\n",
      "Epoch: 19000, loss=1.4246607449308613e-08\n",
      "Epoch: 20000, loss=8.715771215861423e-09\n",
      "Epoch: 21000, loss=5.356610710549603e-09\n",
      "Epoch: 22000, loss=3.3154845535676714e-09\n",
      "Epoch: 23000, loss=2.0744121975724283e-09\n",
      "Epoch: 24000, loss=1.3183321101450929e-09\n",
      "Epoch: 25000, loss=8.570671949925668e-10\n",
      "Epoch: 26000, loss=5.715188322596987e-10\n",
      "Epoch: 27000, loss=3.966149353384907e-10\n",
      "Epoch: 28000, loss=2.853267344615773e-10\n",
      "Epoch: 29000, loss=2.145644495410437e-10\n",
      "Epoch: 30000, loss=1.670199256675886e-10\n",
      "Epoch: 31000, loss=1.3476200710904607e-10\n",
      "Epoch: 32000, loss=1.1207323957762583e-10\n",
      "Epoch: 33000, loss=9.493712066488413e-11\n",
      "Epoch: 34000, loss=8.229234654821838e-11\n",
      "Epoch: 35000, loss=7.29841187485647e-11\n",
      "Epoch: 36000, loss=6.485371267794804e-11\n",
      "Epoch: 37000, loss=5.864641411390537e-11\n",
      "Epoch: 38000, loss=5.3219369883228396e-11\n",
      "Epoch: 39000, loss=4.925693186663693e-11\n",
      "Epoch: 40000, loss=4.5090715883322474e-11\n",
      "Epoch: 41000, loss=4.2188735144277345e-11\n",
      "Epoch: 42000, loss=3.946648910457817e-11\n",
      "Epoch: 43000, loss=3.7111352185759117e-11\n",
      "Epoch: 44000, loss=3.468404383144552e-11\n",
      "Epoch: 45000, loss=3.302603329702336e-11\n",
      "Epoch: 46000, loss=3.125732658815217e-11\n",
      "Epoch: 47000, loss=2.976697013878926e-11\n",
      "Epoch: 48000, loss=2.811886834486188e-11\n",
      "Epoch: 49000, loss=2.695919529283053e-11\n",
      "Epoch: 50000, loss=2.584618803702643e-11\n",
      "Epoch: 51000, loss=2.4776486418076615e-11\n",
      "Epoch: 52000, loss=2.3745628727200874e-11\n",
      "Epoch: 53000, loss=2.2919844841484682e-11\n",
      "Epoch: 54000, loss=2.195929549309028e-11\n",
      "Epoch: 55000, loss=2.1385426415831965e-11\n",
      "Epoch: 56000, loss=2.0483951340688478e-11\n",
      "Epoch: 57000, loss=1.994419386586177e-11\n",
      "Epoch: 58000, loss=1.9091379518942908e-11\n",
      "Epoch: 59000, loss=1.874451462102744e-11\n",
      "Epoch: 60000, loss=1.8104842278154898e-11\n",
      "Epoch: 61000, loss=1.7620904735338172e-11\n",
      "Epoch: 62000, loss=1.7006760583138103e-11\n",
      "Epoch: 63000, loss=1.6708153957600835e-11\n",
      "Epoch: 64000, loss=1.6120785262252468e-11\n",
      "Epoch: 65000, loss=1.5846763137816744e-11\n",
      "Epoch: 66000, loss=1.5283867921267458e-11\n",
      "Epoch: 67000, loss=1.5019114424363877e-11\n",
      "Epoch: 68000, loss=1.4481841073554769e-11\n",
      "Epoch: 69000, loss=1.4239930415382851e-11\n",
      "Epoch: 70000, loss=1.3857410011142157e-11\n",
      "Epoch: 71000, loss=1.363698043377326e-11\n",
      "Epoch: 72000, loss=1.3254395844763955e-11\n",
      "Epoch: 73000, loss=1.3032622724062914e-11\n",
      "Epoch: 74000, loss=1.2688633997670618e-11\n",
      "Epoch: 75000, loss=1.2472902051474666e-11\n",
      "Epoch: 76000, loss=1.2272108676492088e-11\n",
      "Epoch: 77000, loss=1.1948158608332538e-11\n",
      "Epoch: 78000, loss=1.1737862418148559e-11\n",
      "Epoch: 79000, loss=1.1538910452135731e-11\n",
      "Epoch: 80000, loss=1.1225373929002469e-11\n",
      "Epoch: 81000, loss=1.1030854181470762e-11\n",
      "Epoch: 82000, loss=1.085435994557793e-11\n",
      "Epoch: 83000, loss=1.0805967232130342e-11\n",
      "Epoch: 84000, loss=1.0382762358207565e-11\n",
      "Epoch: 85000, loss=1.0330581876050182e-11\n",
      "Epoch: 86000, loss=1.0159752379668152e-11\n",
      "Epoch: 87000, loss=1.0109719485174029e-11\n",
      "Epoch: 88000, loss=9.808658225918254e-12\n",
      "Epoch: 89000, loss=9.64866641445239e-12\n",
      "Epoch: 90000, loss=9.487711831457357e-12\n",
      "Epoch: 91000, loss=9.444862426877254e-12\n",
      "Epoch: 92000, loss=9.162332351153601e-12\n",
      "Epoch: 93000, loss=9.12679046921605e-12\n",
      "Epoch: 94000, loss=8.988243309360211e-12\n",
      "Epoch: 95000, loss=8.841939333481541e-12\n",
      "Epoch: 96000, loss=8.572112637772467e-12\n",
      "Epoch: 97000, loss=8.53330513889139e-12\n",
      "Epoch: 98000, loss=8.498076374541252e-12\n",
      "Epoch: 99000, loss=8.363604080130482e-12\n",
      "Finished training!\n",
      "[{'number_of_neurons': 4, 'inputs_per_neuron': 3, 'biases': array([-2.89507937,  2.19741654, -2.93689775,  3.3493855 ], dtype=float32), 'name': 'layer_0', 'synaptic_weights': array([[-4.75761127, -4.78987122, -4.30772686, -7.47814846],\n",
      "       [ 5.40316486,  5.8400631 ,  5.20959473,  6.0762887 ],\n",
      "       [-0.4987627 , -0.04027947, -0.44667691,  1.02943444]], dtype=float32)}, {'number_of_neurons': 3, 'inputs_per_neuron': 4, 'biases': array([-4.70457458, -4.91372681, -5.6134057 ], dtype=float32), 'name': 'layer_1', 'synaptic_weights': array([[-4.78274202, -5.93470764, -7.85266399],\n",
      "       [ 4.92414904,  4.88311195,  5.50805473],\n",
      "       [-5.56545734, -5.5594449 , -5.43240452],\n",
      "       [ 4.48321772,  5.19387674,  6.11334372]], dtype=float32)}, {'number_of_neurons': 1, 'inputs_per_neuron': 3, 'biases': array([ 12.89077568], dtype=float32), 'name': 'layer_2', 'synaptic_weights': array([[-9.62860489],\n",
      "       [-8.26689434],\n",
      "       [-8.43047333]], dtype=float32)}]\n",
      "Predicting on the training set:\n",
      "[[  2.16388344e-06]\n",
      " [  9.99996901e-01]\n",
      " [  9.99996543e-01]\n",
      " [  9.99997139e-01]\n",
      " [  9.99996662e-01]\n",
      " [  2.42081433e-06]\n",
      " [  2.47121284e-06]]\n",
      "Predicting on the training set:\n",
      "[[  4.31284479e-06]]\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "# Creating the model\n",
    "model = MultiLayerNeuralNetwork([3, 4, 3, 1], tf.float32, tf.float32)\n",
    "print('Model before initializing weights')\n",
    "print(model)\n",
    "\n",
    "# The training set. We have 7 examples, each consisting of 3 input values and 1 output value\n",
    "training_set_inputs = np.array([[0, 0, 1], [0, 1, 1], [1, 0, 1], [0, 1, 0], [1, 0, 0], [1, 1, 1], [0, 0, 0]]).astype(np.float32)\n",
    "training_set_outputs = np.array([[0, 1, 1, 1, 1, 0, 0]]).T.astype(np.float32)\n",
    "\n",
    "# We will use MSE as loss function, and optimize with AdamOptimizer\n",
    "loss_function = lambda y, pred: tf.reduce_mean(tf.squared_difference(y, pred))\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=0.01)\n",
    "model.set_training_params(loss_function, optimizer)\n",
    "\n",
    "# Launch the graph\n",
    "with tf.Session() as sess:\n",
    "    init = tf.global_variables_initializer()\n",
    "    # Initializing the variables\n",
    "    sess.run(init)\n",
    "    model.set_tf_session(sess)\n",
    "    print(model)\n",
    "    # Now training\n",
    "    print(\"Training...\")\n",
    "    for epoch in range(100000):\n",
    "        # Run optimization op (backprop) and loss op (to get loss value)\n",
    "        (_, loss) = sess.run(\n",
    "            [model.optimizer, model.loss], \n",
    "            feed_dict={model.x_: training_set_inputs, model.y_: training_set_outputs})\n",
    "        # Display logs per epoch step\n",
    "        if epoch % 1000 == 0:\n",
    "            print('Epoch: {}, loss={}'.format(epoch, loss))\n",
    "    print(\"Finished training!\")\n",
    "    print(model)\n",
    "    \n",
    "    # Just checking how it adjusts to the training set\n",
    "    print(\"Predicting on the training set:\")\n",
    "    layer_outputs = model.predict(training_set_inputs)\n",
    "    print(layer_outputs[-1].eval(session=sess))\n",
    "    \n",
    "    # Just checking how it adjusts to the training set\n",
    "    print(\"Predicting on the training set:\")\n",
    "    test_set_inputs = np.array([[1, 1, 0]]).astype(np.float32)\n",
    "    layer_outputs = model.predict(test_set_inputs)\n",
    "    print(layer_outputs[-1].eval(session=sess))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "layer_0.synaptic_weights:0\n",
      "layer_0.biases:0\n",
      "layer_1.synaptic_weights:0\n",
      "layer_1.biases:0\n",
      "layer_2.synaptic_weights:0\n",
      "layer_2.biases:0\n",
      "beta1_power:0\n",
      "beta2_power:0\n",
      "layer_0.synaptic_weights/Adam:0\n",
      "layer_0.synaptic_weights/Adam_1:0\n",
      "layer_0.biases/Adam:0\n",
      "layer_0.biases/Adam_1:0\n",
      "layer_1.synaptic_weights/Adam:0\n",
      "layer_1.synaptic_weights/Adam_1:0\n",
      "layer_1.biases/Adam:0\n",
      "layer_1.biases/Adam_1:0\n",
      "layer_2.synaptic_weights/Adam:0\n",
      "layer_2.synaptic_weights/Adam_1:0\n",
      "layer_2.biases/Adam:0\n",
      "layer_2.biases/Adam_1:0\n"
     ]
    }
   ],
   "source": [
    "# Having a look at the variables generated by tensorflow\n",
    "for variable in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) + tf.get_collection(tf.GraphKeys.SAVEABLE_OBJECTS):\n",
    "    print(variable.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}