ConvBonsai.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ConvBonsai Tree\n",
    "\n",
    "### Importing Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "from IPython.display import display, clear_output\n",
    "from sklearn.model_selection import train_test_split as tts\n",
    "import sys\n",
    "import time\n",
    "import os\n",
    "import argparse\n",
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "import matplotlib.image as img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "_StoreAction(option_strings=['-regularization', '--reg'], dest='reg', nargs=None, const=None, default=1e-06, type=None, choices=None, help=None, metavar=None)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parser = argparse.ArgumentParser(description='Process some integers.')\n",
    "parser.add_argument('-lr', '--lr', default = 0.01)\n",
    "parser.add_argument('-tree_depth', '--tdepth', default = 2)\n",
    "parser.add_argument('-conv_depth', '--cdepth', default = 2)\n",
    "parser.add_argument('-sparsity', '--sty', default = 0.995)\n",
    "parser.add_argument('-regularization', '--reg', default = 0.000001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "args = parser.parse_args('')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-15T12:06:06.104645Z",
     "start_time": "2018-08-15T12:06:06.058368Z"
    }
   },
   "source": [
    "# Dataset\n",
    "\n",
    "We have used mnist dataset you can use your own"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-15T12:06:06.104645Z",
     "start_time": "2018-08-15T12:06:06.058368Z"
    }
   },
   "outputs": [],
   "source": [
    "#Loading Pre-processed dataset for Bonsai\n",
    "# dirc = './Datasets/mnist_small/'\n",
    "# Xtrain = np.load(dirc + 'Xtrain.npy')\n",
    "# Ytrain = np.load(dirc + 'Ytrain.npy')\n",
    "# Xtrain, Xtest, Ytrain, Ytest = tts(Xtrain, Ytrain, stratify = Ytrain, test_size = 0.25)\n",
    "\n",
    "(Xtrain, Ytrain), (Xtest, Ytest) = tf.keras.datasets.cifar10.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-15T12:06:06.104645Z",
     "start_time": "2018-08-15T12:06:06.058368Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/abhikcr/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:415: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.\n",
      "If you want the future behaviour and silence this warning, you can specify \"categories='auto'\".\n",
      "In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "one hot encoder for converting integer values to one hot vector\n",
    "\"\"\"\n",
    "from sklearn.preprocessing import LabelEncoder as LE \n",
    "from sklearn.preprocessing import OneHotEncoder as OHE\n",
    "mo1 = LE()\n",
    "mo2 = OHE()\n",
    "Ytrain = mo2.fit_transform(mo1.fit_transform((Ytrain.ravel())).reshape(-1,1)).todense()\n",
    "Ytest = mo2.transform(mo1.transform((Ytest.ravel())).reshape(-1,1)).todense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-15T12:06:06.104645Z",
     "start_time": "2018-08-15T12:06:06.058368Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Size: 50000 ,Data Dims: 3072 ,No. Classes: 10\n"
     ]
    }
   ],
   "source": [
    "# N, dDims = X_train.shape\n",
    "N,W,H,C = Xtrain.shape\n",
    "dDims = W*H*C\n",
    "# nClasses = len(np.unique(Y_train))\n",
    "nClasses = Ytrain.shape[1]\n",
    "print('Training Size:',N,',Data Dims:', dDims,',No. Classes:', nClasses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-15T12:06:06.104645Z",
     "start_time": "2018-08-15T12:06:06.058368Z"
    }
   },
   "outputs": [],
   "source": [
    "# Preprocessing files\n",
    "def preprocess(x):\n",
    "    z = (x - x.mean(axis=(0,1,2), keepdims=True)) / x.std(axis=(0,1,2), keepdims=True)\n",
    "    N, W, H, X = z.shape\n",
    "    return z.reshape(N, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-15T12:06:06.104645Z",
     "start_time": "2018-08-15T12:06:06.058368Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preprocessing the dataset...\n"
     ]
    }
   ],
   "source": [
    "print(\"Preprocessing the dataset...\")\n",
    "Xtrain = preprocess(Xtrain)\n",
    "Xtest = preprocess(Xtest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tensorboard Visualization writer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "writer = tf.summary.FileWriter('convbonsai')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Class Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvBonsai():\n",
    "    def __init__(self, nClasses, dDims, pDims, tDepth, sigma, kernelsshp, strides, cDepth = 2, W=None, T=None, V=None, Z=None, ch = None):\n",
    "        '''\n",
    "        dDims : data Dimensions\n",
    "        pDims : projected Dimesions\n",
    "        nClasses : num Classes\n",
    "        tDepth : tree Depth\n",
    "        \n",
    "        Expected Dimensions:\n",
    "        --------------------\n",
    "        Bonsai Params // Optional\n",
    "        \n",
    "        W [numClasses*totalNodes, projectionDimension]\n",
    "        V [numClasses*totalNodes, projectionDimension]\n",
    "        Z [projectionDimension, dataDimension + 1]\n",
    "        T [internalNodes, projectionDimension]\n",
    "\n",
    "        internalNodes = 2**treeDepth - 1\n",
    "        totalNodes = 2*internalNodes + 1\n",
    "\n",
    "        sigma - tanh non-linearity\n",
    "        sigmaI - Indicator function for node probabilities\n",
    "        sigmaI - has to be set to infinity(1e9 for practicality)\n",
    "        \n",
    "        while doing testing/inference\n",
    "        numClasses will be reset to 1 in binary case\n",
    "        '''\n",
    "        \n",
    "        # Initialization of parameter variables\n",
    "        \n",
    "        self.dDims = dDims\n",
    "        self.pDims = pDims\n",
    "        \n",
    "        # If number of classes is two we dont need to calculate other class probability\n",
    "        if nClasses == 2:\n",
    "            self.nClasses = 1\n",
    "        else:\n",
    "            self.nClasses = nClasses\n",
    "\n",
    "        self.tDepth = tDepth\n",
    "        self.sigma = sigma\n",
    "        self.iNodes = 2**self.tDepth - 1\n",
    "        self.tNodes = 2*self.iNodes + 1\n",
    "        \n",
    "        self.cDepth = cDepth\n",
    "        self.ciNodes = 2**self.cDepth - 1\n",
    "        self.ctNodes = 2*self.ciNodes + 1\n",
    "        \n",
    "        \n",
    "        self.kernelsT = []\n",
    "        \n",
    "        self.strides = []\n",
    "        \n",
    "        if(ch is None):\n",
    "            ch = 3\n",
    "        \n",
    "        self.channels = ch    \n",
    "        var = int(np.sqrt(self.dDims/self.channels))\n",
    "        self.d1 = var\n",
    "        self.d2 = var\n",
    "        d1 = self.d1\n",
    "        d2 = self.d2\n",
    "        \n",
    "        assert d1*d2*ch == self.dDims, \" Dimension mismatch, doesn't seem like it's a image or set channel(ch) = 1\"\n",
    "        \n",
    "        oD1 = d1\n",
    "        oD2 = d2\n",
    "        \n",
    "        self.wts = []\n",
    "        self.wts1 = []\n",
    "        self.wts2 = []\n",
    "        self.bs = []\n",
    "        \n",
    "        \n",
    "        h = 0\n",
    "        h_old = 0\n",
    "        Codims1 = self.d1\n",
    "        Codims2 = self.d2\n",
    "        \n",
    "        with tf.name_scope(\"Params\"):\n",
    "            for i in range(self.ctNodes):\n",
    "\n",
    "                h = int(np.floor(np.log(i+1)/np.log(2)))\n",
    "\n",
    "                self.kernelsT.append(\n",
    "                    tf.get_variable('kernelT'+str(i), kernelsshp[h], \n",
    "                                 initializer=tf.truncated_normal_initializer(stddev=5e-2, dtype=tf.float32),\n",
    "                                 dtype=tf.float32)\n",
    "                )\n",
    "\n",
    "                self.strides.append(strides[h])\n",
    "\n",
    "\n",
    "            for i in range(self.cDepth+1):\n",
    "                Codims1 = np.floor((Codims1 - kernelsshp[i][0])/(strides[i][1])) + 1\n",
    "                Codims2 = np.floor((Codims2 - kernelsshp[i][1])/(strides[i][2])) + 1\n",
    "\n",
    "\n",
    "            self.CoDims = int(Codims1*Codims2) + 1\n",
    "            self.pDims = self.CoDims\n",
    "            self.Z = tf.Variable(tf.random_normal([2,2]), name='Z', dtype=tf.float32) \n",
    "\n",
    "            self.W = tf.Variable(tf.random_normal([self.ctNodes - self.ciNodes, self.nClasses * self.tNodes, self.pDims]), name='W', dtype=tf.float32)\n",
    "            self.V = tf.Variable(tf.random_normal([self.ctNodes - self.ciNodes, self.nClasses * self.tNodes, self.pDims]), name='V', dtype=tf.float32)\n",
    "            self.T = tf.Variable(tf.random_normal([self.ctNodes - self.ciNodes, self.iNodes, self.pDims]), name='T', dtype=tf.float32)\n",
    "        \n",
    "        \n",
    "        self.score = None\n",
    "        self.X_ = None\n",
    "        self.prediction = None\n",
    "        self.convs = []\n",
    "        self.cnodeProb = []\n",
    "        self.nodeProb = []\n",
    "        self.scores = []\n",
    "    \n",
    "    def __call__(self, X, sigmaI):\n",
    "        '''\n",
    "        Function to build the Bonsai Tree graph\n",
    "        \n",
    "        Expected Dimensions\n",
    "        -------------------\n",
    "        X is [_, self.dDims]\n",
    "        X_ is [_, self.pDims]\n",
    "        '''\n",
    "        errmsg = \"Dimension Mismatch, X is [_, self.dataDimension]\"\n",
    "        assert (len(X.shape) == 2 and int(X.shape[1]) == self.dDims), errmsg\n",
    "        \n",
    "        sigmaI = tf.reshape(sigmaI, [1,1])\n",
    "        \n",
    "        # return score, X_ if exists where X_ is the projected X, i.e X_ = (Z.X)/(D^)\n",
    "        if self.score is not None:\n",
    "            return self.score, self.X_\n",
    "        \n",
    "        \n",
    "        Ximg = tf.reshape(X, [-1,self.d1,self.d2,self.channels])\n",
    "        \n",
    "        self.convs = []\n",
    "        \n",
    "        \n",
    "        # For Root Node score...\n",
    "        self.__cnodeProb = [] # node probability list\n",
    "        self.__cnodeProb.append(1) # probability of x passing through root is 1.\n",
    "        \n",
    "        with tf.name_scope('ConvNode'+str(0)):\n",
    "        # All score sums variable initialized to root score... for each tree (Note: can be negative)\n",
    "            convT = 0.1*tf.nn.leaky_relu(tf.nn.conv2d(Ximg,\n",
    "                self.kernelsT[0],\n",
    "                padding=\"VALID\",\n",
    "                strides = self.strides[0]), name = 'convT0')\n",
    "\n",
    "            self.convs.append(convT)\n",
    "\n",
    "            flatConv = tf.layers.Flatten()(convT)\n",
    "            b = tf.squeeze(flatConv.shape[1])\n",
    "            self.wts.append(tf.Variable(tf.random_normal([1, b]), name='wts' + str(0), dtype=tf.float32))\n",
    "            self.bs.append(tf.Variable(tf.random_normal([1, 1]), name='bs' + str(0), dtype=tf.float32))\n",
    "\n",
    "            finalImg =  None\n",
    "\n",
    "\n",
    "            fscore_ = None\n",
    "            fX_ = None\n",
    "            self.__nodeProbs = []\n",
    "        \n",
    "        for i in range(1,self.ctNodes):\n",
    "            with tf.name_scope('ConvNode'+str(i)):\n",
    "                \n",
    "                parent_id = int(np.ceil(i / 2.0) - 1.0)\n",
    "\n",
    "                convTprev = self.convs[parent_id]\n",
    "                flatConvP = tf.layers.Flatten()(convTprev)\n",
    "\n",
    "\n",
    "                cscore = tf.multiply(sigmaI, tf.matmul(self.wts[parent_id], flatConvP, transpose_b = True) + self.bs[parent_id])# 1 x _\n",
    "\n",
    "                # Calculating probability that x should come to this node next given it is in parent node...\n",
    "                cprob = tf.divide((1 + ((-1)**(i + 1))*tf.tanh(cscore)),2.0) # : scalar 1 x_\n",
    "                cprob = self.__cnodeProb[parent_id] * cprob # : scalar 1 x _\n",
    "\n",
    "\n",
    "                # adding prob to node prob list\n",
    "                self.__cnodeProb.append(cprob)\n",
    "\n",
    "                convT = 0.1*tf.nn.leaky_relu(tf.nn.conv2d(convTprev,\n",
    "                    self.kernelsT[i],\n",
    "                    padding=\"VALID\",\n",
    "                    strides = self.strides[i]), name = 'convT' + str(i))\n",
    "\n",
    "                self.convs.append(convT)\n",
    "\n",
    "                flatConv = tf.layers.Flatten()(convT)\n",
    "                b = tf.squeeze(flatConv.shape[1])\n",
    "\n",
    "                self.wts.append(tf.Variable(tf.random_normal([1, b]), name='wts' + str(i), dtype=tf.float32))\n",
    "                self.bs.append(tf.Variable(tf.random_normal([1, 1]), name='bs' + str(i), dtype=tf.float32))\n",
    "            \n",
    "\n",
    "            \n",
    "            if(i+1 > self.ciNodes):\n",
    "                # projected output of convolutional layers....\n",
    "                \n",
    "                iinum = i - self.ciNodes\n",
    "                \n",
    "                a,b = flatConv.shape\n",
    "                onesmat = flatConv[:,0:1]*0 + 1\n",
    "\n",
    "                flat_imgs = tf.concat([flatConv, onesmat], axis = 1)\n",
    "   \n",
    "                X_ = tf.transpose(flat_imgs)#tf.matmul(self.Z, flat_imgs, transpose_b = True)\n",
    "\n",
    "                # For Root Node score...\n",
    "                tnodeProb = [] # node probability list\n",
    "                tnodeProb.append(cprob) # probability of x passing through root is 1.\n",
    "                W_ = self.W[iinum, 0:(self.nClasses),:]# first K trees root W params : KxD^\n",
    "                V_ = self.V[iinum, 0:(self.nClasses),:]# first K trees root V params : KxD^\n",
    "\n",
    "                # All score sums variable initialized to root score... for each tree (Note: can be negative)\n",
    "                score_ = tnodeProb[0]*tf.multiply(tf.matmul(W_, X_), tf.tanh(self.sigma * tf.matmul(V_, X_))) # : Kx_\n",
    "                self.scores.append(flat_imgs)\n",
    "\n",
    "                for t in range(1, self.tNodes):\n",
    "                    with tf.name_scope('BonNode'+str(i)+str(t)):\n",
    "                    # current node is i\n",
    "                    # W, V of K different trees for current node\n",
    "                        W_ = self.W[iinum,t * self.nClasses:((t + 1) * self.nClasses),:]# : KxD^\n",
    "                        V_ = self.V[iinum,t * self.nClasses:((t + 1) * self.nClasses),:]# : KxD^\n",
    "\n",
    "\n",
    "                        # i's parent node shared theta param reshaping to 1xD^\n",
    "                        T_ = tf.reshape(self.T[iinum,int(np.ceil(t / 2.0) - 1.0),:],[-1, self.pDims])# : 1xD^\n",
    "\n",
    "                        # Calculating probability that x should come to this node next given it is in parent node...\n",
    "                        prob = tf.divide((1 + ((-1)**(t + 1))*tf.tanh(tf.multiply(sigmaI, tf.matmul(T_, X_)))),2.0) # : scalar 1x_\n",
    "\n",
    "                        # Actual probability that x will come to this node...p(parent)*p(this|parent)...\n",
    "                        prob = tnodeProb[int(np.ceil(t / 2.0) - 1.0)] * prob # : scalar 1x_\n",
    "\n",
    "                        # adding prob to node prob list\n",
    "                        tnodeProb.append(prob)\n",
    "                        # New score addes to sum of scores...\n",
    "                        score_ += tnodeProb[t]*tf.multiply(tf.matmul(W_, X_), tf.tanh(self.sigma * tf.matmul(V_, X_))) # Kx_\n",
    "\n",
    "                self.scores.append(score_)\n",
    "                self.__nodeProbs.append(tnodeProb[1:])\n",
    "\n",
    "                if(fscore_ is None):\n",
    "                    fscore_ = score_\n",
    "                    fX_ = tf.matmul(T_, X_)*cprob\n",
    "                else:\n",
    "                    fscore_ = fscore_ + score_\n",
    "                    fX_ = fX_ + tf.matmul(T_, X_)*cprob\n",
    "            else:\n",
    "                pass\n",
    "                \n",
    "        self.score = fscore_ \n",
    "        self.X_ = fX_\n",
    "        self.nodeProb = tf.convert_to_tensor(self.__nodeProbs[:])\n",
    "        self.cnodeProb = tf.convert_to_tensor(self.__cnodeProb[1:])\n",
    "        self.layers = self.convs\n",
    "        return self.score, self.X_\n",
    "                \n",
    "   \n",
    "        \n",
    "    \n",
    "    def predict(self):\n",
    "        '''\n",
    "        Takes in a score tensor and outputs a integer class for each data point\n",
    "        '''\n",
    "        if self.prediction is not None:\n",
    "            return self.prediction\n",
    "        if self.nClasses > 2:\n",
    "            self.prediction = tf.argmax(tf.transpose(self.score), 1) # score is 1xk\n",
    "        else:\n",
    "            self.prediction = tf.argmax(tf.concat([tf.transpose(self.score),0*tf.transpose(self.score)], 1), 1)\n",
    "        return self.prediction\n",
    "\n",
    "    def assert_params(self):\n",
    "        \n",
    "        # Asserting Initializaiton\n",
    "        \n",
    "        errRank = \"All Parameters must has only two dimensions shape = [a, b]\"\n",
    "        assert len(self.W.shape) == len(self.Z.shape), errRank\n",
    "        assert len(self.W.shape) == len(self.T.shape), errRank\n",
    "        assert len(self.W.shape) == 2, errRank\n",
    "        msg = \"W and V should be of same Dimensions\"\n",
    "        assert self.W.shape == self.V.shape, msg\n",
    "        errW = \"W and V are [numClasses*totalNodes, projectionDimension]\"\n",
    "        assert self.W.shape[0] == self.nClasses * self.tNodes, errW\n",
    "        assert self.W.shape[1] == self.pDims, errW\n",
    "        errZ = \"Z is [projectionDimension, dataDimension]\"\n",
    "        assert self.Z.shape[0] == self.pDims, errZ\n",
    "        assert self.Z.shape[1] == self.dDims, errZ\n",
    "        errT = \"T is [internalNodes, projectionDimension]\"\n",
    "        assert self.T.shape[0] == self.iNodes, errT\n",
    "        assert self.T.shape[1] == self.pDims, errT\n",
    "        assert int(self.nClasses) > 0, \"numClasses should be > 1\"\n",
    "        msg = \"# of features in data should be > 0\"\n",
    "        assert int(self.dDims) > 0, msg\n",
    "        msg = \"Projection should be  > 0 dims\"\n",
    "        assert int(self.pDims) > 0, msg\n",
    "        msg = \"treeDepth should be >= 0\"\n",
    "        assert int(self.tDepth) >= 0, msg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvBonsaiTrainer():\n",
    "    \n",
    "    def __init__(self, tree, lW, lT, lV, lZ, lr, X, Y, sW, sV, sZ, sT):\n",
    "        \n",
    "        '''\n",
    "        bonsaiObj - Initialised Bonsai Object and Graph...\n",
    "        lW, lT, lV and lZ are regularisers to Bonsai Params...\n",
    "        sW, sT, sV and sZ are sparsity factors to Bonsai Params...\n",
    "        lr - learningRate fro optimizer...\n",
    "        X is the Data Placeholder - Dims [_, dataDimension]\n",
    "        Y - Label placeholder for loss computation\n",
    "        useMCHLoss - For choice between HingeLoss vs CrossEntropy\n",
    "        useMCHLoss - True - MultiClass - multiClassHingeLoss\n",
    "        useMCHLoss - False - MultiClass - crossEntropyLoss\n",
    "        '''\n",
    "        #  Intializations of training parameters\n",
    "        self.tree = tree\n",
    "        \n",
    "        # regularization params lambdas(l) (all are scalars)\n",
    "        self.lW = lW\n",
    "        self.lV = lV\n",
    "        self.lT = lT\n",
    "        self.lZ = lZ\n",
    "\n",
    "        # sparsity parameters (scalars all...) will be used to calculate percentiles to make other cells zero\n",
    "        self.sW = sW \n",
    "        self.sV = sV\n",
    "        self.sT = sT\n",
    "        self.sZ = sZ\n",
    "\n",
    "        # placeholders for inputs and labels\n",
    "        self.Y = Y # _ x nClasses\n",
    "        self.X = X # _ x D\n",
    "        \n",
    "        # learning rate\n",
    "        self.lr = lr\n",
    "        \n",
    "        # Asserting initialization\n",
    "        self.assert_params()\n",
    "        \n",
    "        # place holder for path selection parameter sigmaI\n",
    "        self.sigmaI = tf.placeholder(tf.float32, name='sigmaI')\n",
    "        # invoking __call__ of tree getting initial values of score and projected X\n",
    "        self.score, self.X_ = self.tree(self.X, self.sigmaI)\n",
    "        # defining loss function tensorflow graph variables.....\n",
    "        self.loss, self.marginLoss, self.regLoss = self.lossGraph()\n",
    "        # defining single training step graph process ...\n",
    "        self.tree.TrainStep = tf.train.AdamOptimizer(self.lr).minimize(self.loss)\n",
    "        self.trainStep = self.tree.TrainStep\n",
    "        # defining accuracy and prediction graph objects\n",
    "        self.accuracy = self.accuracyGraph()\n",
    "        self.prediction = self.tree.predict()\n",
    "        \n",
    "        \n",
    "        # set all parameters above 0.99 if dont want to use IHT\n",
    "        if self.sW > 0.99 and self.sV > 0.99 and self.sZ > 0.99 and self.sT > 0.99:\n",
    "            self.isDenseTraining = True\n",
    "        else:\n",
    "            self.isDenseTraining = False\n",
    "            \n",
    "        # setting the hard thresholding graph obejcts\n",
    "        self.hardThrsd()\n",
    "        \n",
    "    def hardThrsd(self):\n",
    "        '''\n",
    "        Set up for hard Thresholding Functionality\n",
    "        '''\n",
    "        with tf.name_scope(\"IHT\"):\n",
    "            # place holders for sparse parameters....\n",
    "            self.__Wth = tf.placeholder(tf.float32, name='Wth')\n",
    "            self.__Vth = tf.placeholder(tf.float32, name='Vth')\n",
    "            self.__Zth = tf.placeholder(tf.float32, name='Zth')\n",
    "            self.__Tth = tf.placeholder(tf.float32, name='Tth')\n",
    "\n",
    "            # assigning the thresholded values to params as a graph object for tensorflow....\n",
    "            self.__Woph = self.tree.W.assign(self.__Wth)\n",
    "            self.__Voph = self.tree.V.assign(self.__Vth)\n",
    "            self.__Toph = self.tree.T.assign(self.__Tth)\n",
    "            self.__Zoph = self.tree.Z.assign(self.__Zth)\n",
    "\n",
    "            # grouping the graph objects as one object....\n",
    "            self.hardThresholdGroup = tf.group(\n",
    "                self.__Woph, self.__Voph, self.__Toph, self.__Zoph)\n",
    "        \n",
    "    def hardThreshold(self, A, s):\n",
    "        '''\n",
    "        Hard thresholding function on Tensor A with sparsity s\n",
    "        '''\n",
    "        # copying to avoid errors....\n",
    "        A_ = np.copy(A)\n",
    "        # flattening the tensor...\n",
    "        A_ = A_.ravel()\n",
    "        if len(A_) > 0:\n",
    "            # calculating the threshold value for sparse limit...\n",
    "            th = np.percentile(np.abs(A_), (1 - s) * 100.0, interpolation='higher')\n",
    "            # making sparse.......\n",
    "            A_[np.abs(A_) < th] = 0.0\n",
    "        # reconstructing in actual shape....\n",
    "        A_ = A_.reshape(A.shape)\n",
    "        return A_\n",
    "\n",
    "    def accuracyGraph(self):\n",
    "        '''\n",
    "        Accuracy Graph to evaluate accuracy when needed\n",
    "        '''\n",
    "        with tf.name_scope(\"ACC\"):\n",
    "            if (self.tree.nClasses > 2):\n",
    "                correctPrediction = tf.equal(tf.argmax(tf.transpose(self.score), 1), tf.argmax(self.Y, 1))\n",
    "                self.accuracy = tf.reduce_mean(tf.cast(correctPrediction, tf.float32))\n",
    "            else:\n",
    "                # some accuracy functional analysis for 2 classes could be different from this...\n",
    "                y_ = self.Y * 2 - 1\n",
    "                correctPrediction = tf.multiply(tf.transpose(self.score), y_)\n",
    "                correctPrediction = tf.nn.relu(correctPrediction)\n",
    "                correctPrediction = tf.ceil(tf.tanh(correctPrediction)) # final predictions.... round to(0 or 1)\n",
    "                self.accuracy = tf.reduce_mean(\n",
    "                    tf.cast(correctPrediction, tf.float32))\n",
    "\n",
    "        return self.accuracy\n",
    "        \n",
    "    \n",
    "    def lossGraph(self):\n",
    "        '''\n",
    "        Loss Graph for given tree\n",
    "        '''\n",
    "        with tf.name_scope(\"Loss\"):\n",
    "            # regularization losses.....\n",
    "            self.regLoss = 0.5 * (self.lZ * tf.square(tf.norm(self.tree.Z)) +\n",
    "                              self.lW * tf.square(tf.norm(self.tree.W)) +\n",
    "                              self.lV * tf.square(tf.norm(self.tree.V)) +\n",
    "                              self.lT * tf.square(tf.norm(self.tree.T)))\n",
    "\n",
    "            llen = self.tree.ciNodes\n",
    "            var = 0\n",
    "            for i in range(llen):\n",
    "                var = var +  self.lT * tf.square(tf.norm(self.tree.wts[i]))\n",
    "\n",
    "            self.regLoss = self.regLoss + var\n",
    "\n",
    "            # emperical actual loss.....\n",
    "            if (self.tree.nClasses > 2):\n",
    "                '''\n",
    "                Cross Entropy loss for MultiClass case in joint training for\n",
    "                faster convergence\n",
    "                '''\n",
    "                # cross entropy loss....\n",
    "                self.marginLoss = tf.reduce_mean(\n",
    "                    tf.nn.softmax_cross_entropy_with_logits_v2(logits=tf.transpose(self.score),\n",
    "                                                                   labels=tf.stop_gradient(self.Y)))\n",
    "            else:\n",
    "                # sigmoid loss....\n",
    "                self.marginLoss = tf.reduce_mean(tf.nn.relu(1.0 - (2 * self.Y - 1) * tf.transpose(self.score)))\n",
    "\n",
    "            # adding the losses...\n",
    "            self.loss = self.marginLoss + self.regLoss\n",
    "        return self.loss, self.marginLoss, self.regLoss\n",
    "        \n",
    "    def assert_params(self):\n",
    "        # asserting the initialization....\n",
    "        err = \"sparsity must be between 0 and 1\"\n",
    "        assert self.sW >= 0 and self.sW <= 1, \"W \" + err\n",
    "        assert self.sV >= 0 and self.sV <= 1, \"V \" + err\n",
    "        assert self.sZ >= 0 and self.sZ <= 1, \"Z \" + err\n",
    "        assert self.sT >= 0 and self.sT <= 1, \"T \" + err\n",
    "        errMsg = \"Dimension Mismatch, Y has to be [_, \" + str(self.tree.nClasses) + \"]\"\n",
    "        errCont = \" numClasses are 1 in case of Binary case by design\"\n",
    "        assert (len(self.Y.shape) == 2 and self.Y.shape[1] == self.tree.nClasses), errMsg + errCont\n",
    "        \n",
    "        \n",
    "\n",
    "        \n",
    "    def train(self, batchSize, totalEpochs, sess, Xtrain, Xval, Ytrain, Yval, saver, filename,valsig):\n",
    "        iht = 0 # to keep a note if thresholding has been started ...\n",
    "        numIters = Xtrain.shape[0] / batchSize # number of batches at a time...\n",
    "        totalBatches = numIters * totalEpochs # total number of batch operations...\n",
    "        treeSigmaI = valsig # controls the fidelity of the approximation too high can saturate tanh.\n",
    "            \n",
    "        maxTestAcc = -10000\n",
    "        itersInPhase = 0\n",
    "        \n",
    "        for i in range(totalEpochs):\n",
    "            print(\"\\nEpoch Number: \" + str(i))\n",
    "            # defining training acc and loss\n",
    "            trainAcc = 0.0\n",
    "            trainAccOld = 0.0\n",
    "            trainLoss = 0.0\n",
    "            trainBest = 0.0\n",
    "            \n",
    "            numIters = int(numIters)\n",
    "            \n",
    "            for j in range(numIters):\n",
    "                # creating batch.....sequentiall could be done randomly using choice function...\n",
    "                mini_batchX = Xtrain[j*batchSize:(j+1)*batchSize,:] # B x D\n",
    "                mini_batchY = Ytrain[j*batchSize:(j+1)*batchSize] # B x \n",
    "            \n",
    "                # feed for training using tensorflow graph based gradient descent approach......\n",
    "                _feed_dict = {self.X: mini_batchX, self.Y: mini_batchY,\n",
    "                                  self.sigmaI: treeSigmaI}\n",
    "                \n",
    "                # training the tensorflow graph\n",
    "                _, batchLoss, batchAcc = sess.run(\n",
    "                    [self.trainStep, self.loss, self.accuracy],\n",
    "                    feed_dict=_feed_dict)\n",
    "                \n",
    "                # calculating acc....\n",
    "                trainAcc += batchAcc\n",
    "                trainLoss += batchLoss\n",
    "                \n",
    "                \n",
    "                \n",
    "                # to update sigmaI.....\n",
    "                if ((itersInPhase) % 100 == 0):\n",
    "                    \n",
    "                    # Making a random batch....\n",
    "                    indices = np.random.choice(Xtrain.shape[0], 100)\n",
    "                    rand_batchX = Xtrain[indices, :]\n",
    "                    rand_batchY = Ytrain[indices, :]\n",
    "                    rand_batchY = np.reshape(rand_batchY, [-1, self.tree.nClasses])\n",
    "\n",
    "                    _feed_dict = {self.X: rand_batchX,\n",
    "                                  self.sigmaI: treeSigmaI}\n",
    "                    # Projected matrix...\n",
    "                    Xcapeval = self.X_.eval(feed_dict=_feed_dict) # D^ x 1\n",
    "                    sum_tr = 0.0 \n",
    "                    for k in range(0, self.tree.iNodes):\n",
    "                        sum_tr += (np.sum(np.abs(Xcapeval)))\n",
    "\n",
    "                    \n",
    "                    if(self.tree.iNodes > 0):\n",
    "                        sum_tr /= (self.tree.iNodes) # normalizing all sums\n",
    "                        sum_tr = 1 / sum_tr # inverse of average sum\n",
    "                    else:\n",
    "                        sum_tr = 0.1\n",
    "                    # thresholding inverse of sum as min(1000, sum_inv*2^(cuurent batch number / total bacthes / 30))\n",
    "                    sum_tr = min(\n",
    "                        1000, sum_tr * (2**(float(itersInPhase) /\n",
    "                                            (float(totalBatches) )))*valsig/30)\n",
    "                    # assiging higher values as convergence is reached...\n",
    "                    treeSigmaI = max(sum_tr, treeSigmaI)\n",
    "                    \n",
    "                itersInPhase+=1\n",
    "                \n",
    "                \n",
    "                # to start hard thresholding after half_time(could vary) ......\n",
    "                if((itersInPhase//numIters > (1/2)*totalEpochs) and (not self.isDenseTraining)):\n",
    "                    if(iht == 0):\n",
    "                        print('\\n\\nHard Thresolding Started\\n\\n')\n",
    "                        iht = 1\n",
    "                    \n",
    "                    # getting the current estimates of  W,V,Z,T...\n",
    "                    currW = self.tree.W.eval()\n",
    "                    currV = self.tree.V.eval()\n",
    "                    currZ = self.tree.Z.eval()\n",
    "                    currT = self.tree.T.eval()\n",
    "\n",
    "                    # Setting a method to make some values of matrix zero....\n",
    "                    self.__thrsdW = self.hardThreshold(currW, self.sW)\n",
    "                    self.__thrsdV = self.hardThreshold(currV, self.sV)\n",
    "                    self.__thrsdZ = self.hardThreshold(currZ, self.sZ)\n",
    "                    self.__thrsdT = self.hardThreshold(currT, self.sT)\n",
    "\n",
    "                    # runnign the hard thresholding graph....\n",
    "                    fd_thrsd = {self.__Wth: self.__thrsdW, self.__Vth: self.__thrsdV,\n",
    "                                self.__Zth: self.__thrsdZ, self.__Tth: self.__thrsdT}\n",
    "                    sess.run(self.hardThresholdGroup, feed_dict=fd_thrsd)\n",
    "                    \n",
    "            \n",
    "            \n",
    "            print(\"Train Loss: \" + str(trainLoss / numIters) +\n",
    "                  \" Train accuracy: \" + str(trainAcc / numIters))\n",
    "            print(\"SigmaI :\",treeSigmaI,\":LR:\",self.lr)\n",
    "            \n",
    "            # calculating the test accuracies with sigmaI as expected -> inf.. = 10^9\n",
    "            oldSigmaI = treeSigmaI\n",
    "            treeSigmaI = 1e9\n",
    "            \n",
    "            # test feed for tf...\n",
    "            _feed_dict = {self.X: Xval, self.Y: Yval,\n",
    "                                  self.sigmaI: treeSigmaI}\n",
    "            \n",
    "            # calculating losses....\n",
    "            testAcc, testLoss, regTestLoss = sess.run([self.accuracy, self.loss, self.regLoss], feed_dict=_feed_dict)\n",
    "            \n",
    "            \n",
    "            if maxTestAcc <= testAcc:\n",
    "                maxTestAccEpoch = i\n",
    "                maxTestAcc = testAcc\n",
    "                saver.save(sess, filename + \"/model_best\")\n",
    "                \n",
    "            \n",
    "            print(\"Test accuracy %g\" % testAcc)\n",
    "            print(\"MarginLoss + RegLoss: \" + str(testLoss - regTestLoss) +\n",
    "                  \" + \" + str(regTestLoss) + \" = \" + str(testLoss) + \"\\n\", end='\\r')\n",
    "            \n",
    "            \n",
    "            treeSigmaI = oldSigmaI\n",
    "            \n",
    "        # sigmaI has to be set to infinity to ensure\n",
    "        # only a single path is used in inference\n",
    "        treeSigmaI = 1e9\n",
    "        print(\"\\nMaximum Test accuracy at compressed\" +\n",
    "              \" model size(including early stopping): \" +\n",
    "              str(maxTestAcc) + \" at Epoch: \" +\n",
    "              str(maxTestAccEpoch + 1) + \"\\nFinal Test\" +\n",
    "              \" Accuracy: \" + str(testAcc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating the model graph and training graph..\n",
      "Done Creating the graphs.\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "kernelsshp : kernel sizes for convolutional filters at each level in tree\n",
    "strides : strides for convolutional layers at each level in tree\n",
    "tDepth : bonsai tree depth after conv tree ends\n",
    "cDepth : depth on convolutional tree\n",
    "ch : number of channels in image\n",
    "lW, lT, lV, lZ : regularization params\n",
    "lr : learning rate\n",
    "sZ,sW,sV,sT : sparsity constraints on params Z,W,V,T\n",
    "\"\"\"\n",
    "tf.reset_default_graph()\n",
    "print(\"Creating the model graph and training graph..\")\n",
    "kernelsshp = [[4,4,3,3],[4,4,3,2],[3,3,2,1],[2,2,5,1],[3,3,1,1],[3,3,1,1],[3,3,1,1]]\n",
    "strides = [[1,2,2,1],[1,2,2,1],[1,1,1,1],[1,1,1,1],[1,1,1,1],[1,1,1,1],[1,1,1,1]]\n",
    "\n",
    "cdepth = args.cdepth\n",
    "tdepth = args.tdepth\n",
    "tree = ConvBonsai(nClasses = nClasses, dDims = dDims, pDims = 28, tDepth = tdepth, sigma = 1,\n",
    "              kernelsshp = kernelsshp, strides = strides , cDepth = cdepth, ch = 3)\n",
    "\n",
    "X = tf.placeholder(\"float32\", [None, dDims])\n",
    "Y = tf.placeholder(\"float32\", [None, nClasses])\n",
    "reg = args.reg\n",
    "sty = args.sty\n",
    "lrm = args.lr\n",
    "bonsaiTrainer = ConvBonsaiTrainer(tree, lW = reg, lT = reg, lV = reg, lZ = reg, lr = lrm, X = X, Y = Y,\n",
    "                              sZ = sty, sW = sty, sV = sty, sT = sty)\n",
    "init_op = tf.global_variables_initializer()\n",
    "print(\"Done Creating the graphs.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Restoring/Initializing the model state...\n",
      "\n",
      "Epoch Number: 0\n",
      "Train Loss: 2.2191270446777343 Train accuracy: 0.164200000166893\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.1588\n",
      "MarginLoss + RegLoss: 2.3659592 + 0.00397164 = 2.3699307\n",
      "\n",
      "Epoch Number: 1\n",
      "Train Loss: 1.9791626977920531 Train accuracy: 0.26801999926567077\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.2406\n",
      "MarginLoss + RegLoss: 2.1238356 + 0.0038187152 = 2.1276543\n",
      "\n",
      "Epoch Number: 2\n",
      "Train Loss: 1.8547274899482726 Train accuracy: 0.3238200014829636\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.2657\n",
      "MarginLoss + RegLoss: 2.093099 + 0.003893574 = 2.0969927\n",
      "\n",
      "Epoch Number: 3\n",
      "Train Loss: 1.8013080430030823 Train accuracy: 0.34439999997615817\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.2907\n",
      "MarginLoss + RegLoss: 2.0603678 + 0.00400654 = 2.0643744\n",
      "\n",
      "Epoch Number: 4\n",
      "Train Loss: 1.7617854833602906 Train accuracy: 0.36010000050067903\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3078\n",
      "MarginLoss + RegLoss: 2.0451021 + 0.0041098446 = 2.049212\n",
      "\n",
      "Epoch Number: 5\n",
      "Train Loss: 1.7320349526405334 Train accuracy: 0.3714799988269806\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3168\n",
      "MarginLoss + RegLoss: 2.0586343 + 0.0042029186 = 2.0628371\n",
      "\n",
      "Epoch Number: 6\n",
      "Train Loss: 1.7058154892921449 Train accuracy: 0.38257999837398526\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3208\n",
      "MarginLoss + RegLoss: 2.0861695 + 0.004291445 = 2.090461\n",
      "\n",
      "Epoch Number: 7\n",
      "Train Loss: 1.6853181838989257 Train accuracy: 0.3898399990797043\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.324\n",
      "MarginLoss + RegLoss: 2.1252735 + 0.0043651937 = 2.1296387\n",
      "\n",
      "Epoch Number: 8\n",
      "Train Loss: 1.665972204208374 Train accuracy: 0.3971800011396408\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3276\n",
      "MarginLoss + RegLoss: 2.1636856 + 0.0044255354 = 2.168111\n",
      "\n",
      "Epoch Number: 9\n",
      "Train Loss: 1.6498264408111571 Train accuracy: 0.4036199998855591\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3246\n",
      "MarginLoss + RegLoss: 2.1917632 + 0.0044782283 = 2.1962414\n",
      "\n",
      "Maximum Test accuracy at compressed model size(including early stopping): 0.3276 at Epoch: 9\n",
      "Final Test Accuracy: 0.3246\n",
      "Done sequence  0\n",
      "\n",
      "Epoch Number: 0\n",
      "Train Loss: 1.6366750144958495 Train accuracy: 0.40899999856948854\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3221\n",
      "MarginLoss + RegLoss: 2.2355044 + 0.004527044 = 2.2400315\n",
      "\n",
      "Epoch Number: 1\n",
      "Train Loss: 1.624858434200287 Train accuracy: 0.412739999294281\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.322\n",
      "MarginLoss + RegLoss: 2.2647343 + 0.00457689 = 2.2693112\n",
      "\n",
      "Epoch Number: 2\n",
      "Train Loss: 1.6143588161468505 Train accuracy: 0.41673999786376953\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3239\n",
      "MarginLoss + RegLoss: 2.299505 + 0.004627322 = 2.3041322\n",
      "\n",
      "Epoch Number: 3\n",
      "Train Loss: 1.604576289653778 Train accuracy: 0.42070000171661376\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3268\n",
      "MarginLoss + RegLoss: 2.3224723 + 0.0046758875 = 2.3271482\n",
      "\n",
      "Epoch Number: 4\n",
      "Train Loss: 1.5958826017379761 Train accuracy: 0.42388000190258024\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3283\n",
      "MarginLoss + RegLoss: 2.349085 + 0.0047229747 = 2.3538082\n",
      "\n",
      "Epoch Number: 5\n",
      "Train Loss: 1.5879599308967591 Train accuracy: 0.42667999923229216\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3325\n",
      "MarginLoss + RegLoss: 2.3583686 + 0.0047678985 = 2.3631365\n",
      "\n",
      "Epoch Number: 6\n",
      "Train Loss: 1.580740671157837 Train accuracy: 0.4299200004339218\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3334\n",
      "MarginLoss + RegLoss: 2.3838875 + 0.004812995 = 2.3887005\n",
      "\n",
      "Epoch Number: 7\n",
      "Train Loss: 1.573757209777832 Train accuracy: 0.43210000097751616\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3355\n",
      "MarginLoss + RegLoss: 2.4046543 + 0.004856806 = 2.409511\n",
      "\n",
      "Epoch Number: 8\n",
      "Train Loss: 1.5675632929801941 Train accuracy: 0.4341399985551834\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3378\n",
      "MarginLoss + RegLoss: 2.4154012 + 0.0049007274 = 2.420302\n",
      "\n",
      "Epoch Number: 9\n",
      "Train Loss: 1.5617989706993103 Train accuracy: 0.43640000104904175\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3387\n",
      "MarginLoss + RegLoss: 2.4328241 + 0.004944475 = 2.4377687\n",
      "\n",
      "Maximum Test accuracy at compressed model size(including early stopping): 0.3387 at Epoch: 10\n",
      "Final Test Accuracy: 0.3387\n",
      "Done sequence  1\n",
      "\n",
      "Epoch Number: 0\n",
      "Train Loss: 1.5565672087669373 Train accuracy: 0.43875999927520754\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3393\n",
      "MarginLoss + RegLoss: 2.4456458 + 0.0049884147 = 2.4506342\n",
      "\n",
      "Epoch Number: 1\n",
      "Train Loss: 1.5516237330436706 Train accuracy: 0.4405600011348724\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3417\n",
      "MarginLoss + RegLoss: 2.4652991 + 0.0050306027 = 2.4703298\n",
      "\n",
      "Epoch Number: 2\n",
      "Train Loss: 1.546924867630005 Train accuracy: 0.4427800005674362\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3424\n",
      "MarginLoss + RegLoss: 2.4911137 + 0.005073229 = 2.496187\n",
      "\n",
      "Epoch Number: 3\n",
      "Train Loss: 1.5423449492454528 Train accuracy: 0.44457999885082244\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.342\n",
      "MarginLoss + RegLoss: 2.509455 + 0.0051163565 = 2.5145714\n",
      "\n",
      "Epoch Number: 4\n",
      "Train Loss: 1.5380521273612977 Train accuracy: 0.4461199992895126\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3422\n",
      "MarginLoss + RegLoss: 2.528488 + 0.0051584938 = 2.5336463\n",
      "\n",
      "Epoch Number: 5\n",
      "Train Loss: 1.5336831283569337 Train accuracy: 0.4477399998903275\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3419\n",
      "MarginLoss + RegLoss: 2.5530777 + 0.005201386 = 2.558279\n",
      "\n",
      "Epoch Number: 6\n",
      "Train Loss: 1.5298376893997192 Train accuracy: 0.4497200018167496\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.341\n",
      "MarginLoss + RegLoss: 2.5773096 + 0.005245168 = 2.5825548\n",
      "\n",
      "Epoch Number: 7\n",
      "Train Loss: 1.5264435601234436 Train accuracy: 0.4510400003194809\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3379\n",
      "MarginLoss + RegLoss: 2.606741 + 0.005287953 = 2.6120288\n",
      "\n",
      "Epoch Number: 8\n",
      "Train Loss: 1.5232838797569275 Train accuracy: 0.45202000081539156\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3347\n",
      "MarginLoss + RegLoss: 2.6316164 + 0.005331125 = 2.6369474\n",
      "\n",
      "Epoch Number: 9\n",
      "Train Loss: 1.520127112865448 Train accuracy: 0.45370000183582304\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3335\n",
      "MarginLoss + RegLoss: 2.645924 + 0.005373162 = 2.6512973\n",
      "\n",
      "Maximum Test accuracy at compressed model size(including early stopping): 0.3424 at Epoch: 3\n",
      "Final Test Accuracy: 0.3335\n",
      "Done sequence  2\n",
      "\n",
      "Epoch Number: 0\n",
      "Train Loss: 1.5154305911064148 Train accuracy: 0.45614000082015993\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3356\n",
      "MarginLoss + RegLoss: 2.6627433 + 0.005416923 = 2.6681602\n",
      "\n",
      "Epoch Number: 1\n",
      "Train Loss: 1.5109895539283753 Train accuracy: 0.45876000106334686\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.336\n",
      "MarginLoss + RegLoss: 2.6885793 + 0.005461072 = 2.6940403\n",
      "\n",
      "Epoch Number: 2\n",
      "Train Loss: 1.507265532016754 Train accuracy: 0.46047999858856203\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3338\n",
      "MarginLoss + RegLoss: 2.7196636 + 0.0055052517 = 2.725169\n",
      "\n",
      "Epoch Number: 3\n",
      "Train Loss: 1.504065101146698 Train accuracy: 0.46185999989509585\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3337\n",
      "MarginLoss + RegLoss: 2.7361073 + 0.005549074 = 2.7416565\n",
      "\n",
      "Epoch Number: 4\n",
      "Train Loss: 1.5008386182785034 Train accuracy: 0.46289999842643736\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3322\n",
      "MarginLoss + RegLoss: 2.7607703 + 0.005592715 = 2.7663631\n",
      "\n",
      "Epoch Number: 5\n",
      "Train Loss: 1.497693314552307 Train accuracy: 0.4641399985551834\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3313\n",
      "MarginLoss + RegLoss: 2.775086 + 0.005636255 = 2.7807221\n",
      "\n",
      "Epoch Number: 6\n",
      "Train Loss: 1.4943758177757263 Train accuracy: 0.4650599992275238\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3313\n",
      "MarginLoss + RegLoss: 2.7911942 + 0.0056800167 = 2.7968743\n",
      "\n",
      "Epoch Number: 7\n",
      "Train Loss: 1.4915647459030152 Train accuracy: 0.46555999875068665\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3289\n",
      "MarginLoss + RegLoss: 2.8155804 + 0.0057235057 = 2.8213038\n",
      "\n",
      "Epoch Number: 8\n",
      "Train Loss: 1.4886921381950378 Train accuracy: 0.4667200005054474\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3284\n",
      "MarginLoss + RegLoss: 2.833225 + 0.0057669054 = 2.8389919\n",
      "\n",
      "Epoch Number: 9\n",
      "Train Loss: 1.4859913468360901 Train accuracy: 0.4678400003910065\n",
      "SigmaI : 1 :LR: 0.01\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0820 20:16:05.446081 139896027801408 deprecation.py:323] From /home/abhikcr/anaconda3/lib/python3.6/site-packages/tensorflow/python/training/saver.py:960: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to delete files with this prefix.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy 0.3269\n",
      "MarginLoss + RegLoss: 2.858096 + 0.0058098254 = 2.8639057\n",
      "\n",
      "Maximum Test accuracy at compressed model size(including early stopping): 0.336 at Epoch: 2\n",
      "Final Test Accuracy: 0.3269\n",
      "Done sequence  3\n",
      "\n",
      "Epoch Number: 0\n",
      "Train Loss: 1.4832683396339417 Train accuracy: 0.46901999950408935\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3265\n",
      "MarginLoss + RegLoss: 2.8746254 + 0.0058524865 = 2.880478\n",
      "\n",
      "Epoch Number: 1\n",
      "Train Loss: 1.4802012300491334 Train accuracy: 0.4698999977111816\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3261\n",
      "MarginLoss + RegLoss: 2.8892198 + 0.005896507 = 2.8951163\n",
      "\n",
      "Epoch Number: 2\n",
      "Train Loss: 1.47764310836792 Train accuracy: 0.4710599988698959\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3257\n",
      "MarginLoss + RegLoss: 2.9106605 + 0.0059384 = 2.9165988\n",
      "\n",
      "Epoch Number: 3\n",
      "Train Loss: 1.4750825381278991 Train accuracy: 0.4724400007724762\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3259\n",
      "MarginLoss + RegLoss: 2.9208748 + 0.00598032 = 2.926855\n",
      "\n",
      "Epoch Number: 4\n",
      "Train Loss: 1.4727976417541504 Train accuracy: 0.4738400018215179\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3226\n",
      "MarginLoss + RegLoss: 2.9448755 + 0.0060241856 = 2.9508996\n",
      "\n",
      "Epoch Number: 5\n",
      "Train Loss: 1.4702343654632568 Train accuracy: 0.47451999902725217\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3207\n",
      "MarginLoss + RegLoss: 2.962416 + 0.006066667 = 2.9684825\n",
      "\n",
      "Epoch Number: 6\n",
      "Train Loss: 1.4677894830703735 Train accuracy: 0.4753199988603592\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3188\n",
      "MarginLoss + RegLoss: 2.9761558 + 0.006110537 = 2.9822662\n",
      "\n",
      "Epoch Number: 7\n",
      "Train Loss: 1.4653960037231446 Train accuracy: 0.47625999987125395\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3184\n",
      "MarginLoss + RegLoss: 2.9941509 + 0.0061523844 = 3.0003033\n",
      "\n",
      "Epoch Number: 8\n",
      "Train Loss: 1.4630174350738525 Train accuracy: 0.4773800009489059\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3169\n",
      "MarginLoss + RegLoss: 3.0085702 + 0.0061949748 = 3.0147653\n",
      "\n",
      "Epoch Number: 9\n",
      "Train Loss: 1.4605724120140076 Train accuracy: 0.4791000032424927\n",
      "SigmaI : 1 :LR: 0.01\n",
      "Test accuracy 0.3151\n",
      "MarginLoss + RegLoss: 3.0360067 + 0.0062366584 = 3.0422432\n",
      "\n",
      "Maximum Test accuracy at compressed model size(including early stopping): 0.3265 at Epoch: 1\n",
      "Final Test Accuracy: 0.3151\n",
      "Done sequence  4\n"
     ]
    }
   ],
   "source": [
    "print(\"Restoring/Initializing the model state...\")\n",
    "directory = \"./bonsaiconv\"\n",
    "filename = directory + \"/model\"  #filename to save model\n",
    "try:\n",
    "    os.stat(directory)\n",
    "except:\n",
    "    os.mkdir(directory) \n",
    "with tf.name_scope('hidden') as scope:\n",
    "    with tf.Session() as sess:\n",
    "        saver = tf.train.Saver()\n",
    "        try:\n",
    "            saver.restore(sess, filename)\n",
    "        except:\n",
    "            sess.run(init_op)\n",
    "            \n",
    "###   __ uncomment if using tensorboard__\n",
    "#         writer = tf.summary.FileWriter('convbonsai')\n",
    "#         writer.add_graph(sess.graph)\n",
    "        \n",
    "        saver.save(sess, filename)\n",
    "        totalEpochs = 10\n",
    "        batchSize = np.maximum(1000, int(np.ceil(np.sqrt(Ytrain.shape[0]))))\n",
    "        for i in range(5):\n",
    "            bonsaiTrainer.train(batchSize, totalEpochs, sess, Xtrain, Xtest, Ytrain, Ytest, saver, filename,1)\n",
    "            saver.save(sess, filename + str(i))\n",
    "            print(\"Done sequence \", i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Checking Sparsity "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_zero_ratios(tree):\n",
    "    zs = np.sum(np.abs(tree.Z.eval())>0.000000000000001)\n",
    "    ws = np.sum(np.abs(tree.W.eval())>0.000000000000001)\n",
    "    vs = np.sum(np.abs(tree.V.eval())>0.000000000000001)\n",
    "    ts = np.sum(np.abs(tree.T.eval())>0.000000000000001)\n",
    "    print('Number of non zeros achieved...\\nW:',ws,'\\nV:',vs,'\\nT:',ts,'\\nZ:',zs)\n",
    "    var = (ws+vs+ts)\n",
    "    return var"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "image = Xtrain[:10000,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sparse ratios achieved...\n",
      "W: 4760 \n",
      "V: 4760 \n",
      "T: 204 \n",
      "Z: 4\n"
     ]
    },
    {
     "ename": "InvalidArgumentError",
     "evalue": "input and filter must have the same depth: 3 vs 1\n\t [[node ConvNode0/Conv2D (defined at <ipython-input-10-92afeeeed7e0>:149) ]]\n\nErrors may have originated from an input operation.\nInput Source operations connected to node ConvNode0/Conv2D:\n Reshape_1 (defined at <ipython-input-10-92afeeeed7e0>:135)\t\n kernelT0/read (defined at <ipython-input-10-92afeeeed7e0>:88)\n\nOriginal stack trace for 'ConvNode0/Conv2D':\n  File \"/home/abhikcr/anaconda3/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/ipykernel/kernelapp.py\", line 505, in start\n    self.io_loop.start()\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tornado/platform/asyncio.py\", line 148, in start\n    self.asyncio_loop.run_forever()\n  File \"/home/abhikcr/anaconda3/lib/python3.6/asyncio/base_events.py\", line 438, in run_forever\n    self._run_once()\n  File \"/home/abhikcr/anaconda3/lib/python3.6/asyncio/base_events.py\", line 1451, in _run_once\n    handle._run()\n  File \"/home/abhikcr/anaconda3/lib/python3.6/asyncio/events.py\", line 145, in _run\n    self._callback(*self._args)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py\", line 690, in <lambda>\n    lambda f: self._run_callback(functools.partial(callback, future))\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n    ret = callback()\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tornado/gen.py\", line 787, in inner\n    self.run()\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tornado/gen.py\", line 748, in run\n    yielded = self.gen.send(value)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 365, in process_one\n    yield gen.maybe_future(dispatch(*args))\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tornado/gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 272, in dispatch_shell\n    yield gen.maybe_future(handler(stream, idents, msg))\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tornado/gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 542, in execute_request\n    user_expressions, allow_stdin,\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tornado/gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/ipykernel/ipkernel.py\", line 294, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/ipykernel/zmqshell.py\", line 536, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2855, in run_cell\n    raw_cell, store_history, silent, shell_futures)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2881, in _run_cell\n    return runner(coro)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/IPython/core/async_helpers.py\", line 68, in _pseudo_sync_runner\n    coro.send(None)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3058, in run_cell_async\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3249, in run_ast_nodes\n    if (await self.run_code(code, result,  async_=asy)):\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3326, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-12-8d9fd02fc719>\", line 21, in <module>\n    sZ = 0.995, sW = 0.995, sV = 0.995, sT = 0.995)\n  File \"<ipython-input-11-28d7b6900192>\", line 44, in __init__\n    self.score, self.X_ = self.tree(self.X, self.sigmaI)\n  File \"<ipython-input-10-92afeeeed7e0>\", line 149, in __call__\n    strides = self.strides[0]), name = 'convT0')\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py\", line 1953, in conv2d\n    name=name)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py\", line 1071, in conv2d\n    data_format=data_format, dilations=dilations, name=name)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py\", line 788, in _apply_op_helper\n    op_def=op_def)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py\", line 507, in new_func\n    return func(*args, **kwargs)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 3616, in create_op\n    op_def=op_def)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 2005, in __init__\n    self._traceback = tf_stack.extract_stack()\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1355\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1356\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1357\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1340\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1341\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1428\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1429\u001b[0;31m         run_metadata)\n\u001b[0m\u001b[1;32m   1430\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m: input and filter must have the same depth: 3 vs 1\n\t [[{{node ConvNode0/Conv2D}}]]",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-18-96153637cf5b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0mval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtree\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprediction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtree\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtree\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtree\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconvs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtree\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkernelsT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtree\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcnodeProb\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0m_feed_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m     \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    948\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 950\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    951\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    952\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1171\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1172\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1173\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1174\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1175\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1348\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1349\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1350\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1351\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1368\u001b[0m           \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1369\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0merror_interpolation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minterpolate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1370\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1371\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1372\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m: input and filter must have the same depth: 3 vs 1\n\t [[node ConvNode0/Conv2D (defined at <ipython-input-10-92afeeeed7e0>:149) ]]\n\nErrors may have originated from an input operation.\nInput Source operations connected to node ConvNode0/Conv2D:\n Reshape_1 (defined at <ipython-input-10-92afeeeed7e0>:135)\t\n kernelT0/read (defined at <ipython-input-10-92afeeeed7e0>:88)\n\nOriginal stack trace for 'ConvNode0/Conv2D':\n  File \"/home/abhikcr/anaconda3/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/ipykernel/kernelapp.py\", line 505, in start\n    self.io_loop.start()\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tornado/platform/asyncio.py\", line 148, in start\n    self.asyncio_loop.run_forever()\n  File \"/home/abhikcr/anaconda3/lib/python3.6/asyncio/base_events.py\", line 438, in run_forever\n    self._run_once()\n  File \"/home/abhikcr/anaconda3/lib/python3.6/asyncio/base_events.py\", line 1451, in _run_once\n    handle._run()\n  File \"/home/abhikcr/anaconda3/lib/python3.6/asyncio/events.py\", line 145, in _run\n    self._callback(*self._args)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py\", line 690, in <lambda>\n    lambda f: self._run_callback(functools.partial(callback, future))\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n    ret = callback()\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tornado/gen.py\", line 787, in inner\n    self.run()\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tornado/gen.py\", line 748, in run\n    yielded = self.gen.send(value)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 365, in process_one\n    yield gen.maybe_future(dispatch(*args))\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tornado/gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 272, in dispatch_shell\n    yield gen.maybe_future(handler(stream, idents, msg))\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tornado/gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 542, in execute_request\n    user_expressions, allow_stdin,\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tornado/gen.py\", line 209, in wrapper\n    yielded = next(result)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/ipykernel/ipkernel.py\", line 294, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/ipykernel/zmqshell.py\", line 536, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2855, in run_cell\n    raw_cell, store_history, silent, shell_futures)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2881, in _run_cell\n    return runner(coro)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/IPython/core/async_helpers.py\", line 68, in _pseudo_sync_runner\n    coro.send(None)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3058, in run_cell_async\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3249, in run_ast_nodes\n    if (await self.run_code(code, result,  async_=asy)):\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 3326, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-12-8d9fd02fc719>\", line 21, in <module>\n    sZ = 0.995, sW = 0.995, sV = 0.995, sT = 0.995)\n  File \"<ipython-input-11-28d7b6900192>\", line 44, in __init__\n    self.score, self.X_ = self.tree(self.X, self.sigmaI)\n  File \"<ipython-input-10-92afeeeed7e0>\", line 149, in __call__\n    strides = self.strides[0]), name = 'convT0')\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py\", line 1953, in conv2d\n    name=name)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py\", line 1071, in conv2d\n    data_format=data_format, dilations=dilations, name=name)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py\", line 788, in _apply_op_helper\n    op_def=op_def)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py\", line 507, in new_func\n    return func(*args, **kwargs)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 3616, in create_op\n    op_def=op_def)\n  File \"/home/abhikcr/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 2005, in __init__\n    self._traceback = tf_stack.extract_stack()\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver = tf.train.Saver()\n",
    "    saver.restore(sess, filename)\n",
    "    var = calc_zero_ratios(tree)\n",
    "    _feed_dict = {bonsaiTrainer.X:(image*255).astype(int).reshape(-1,dDims),bonsaiTrainer.sigmaI:float(1)}\n",
    "    \n",
    "    start = time.time()\n",
    "    val = sess.run([tree.prediction, tree.wts, tree.bs, tree.convs, tree.kernelsT, tree.cnodeProb], feed_dict=_feed_dict)\n",
    "    end = time.time()\n",
    "    \n",
    "    size = 0\n",
    "    for i in range(len(val[1])):\n",
    "        size += np.sum(val[1][i]>0.0000000001)\n",
    "        size += np.sum(val[2][i]>0.0000000001)\n",
    "\n",
    "    print('Number of non_zero paramters : ',var + size,' Time taken : ', end-start)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Checking Categorization effect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "LL = []\n",
    "LR = []\n",
    "RR = []\n",
    "RL = []\n",
    "for i in range(len(val[0])):\n",
    "\n",
    "    if(np.round(val[-1][:,:,i])[0] == 1 and np.round(val[-1][:,:,i])[2] == 1):\n",
    "        LL.append(val[0][i])\n",
    "    elif(np.round(val[-1][:,:,i])[0] == 1 and np.round(val[-1][:,:,i])[3] == 1):\n",
    "        LR.append(val[0][i])\n",
    "    elif(np.round(val[-1][:,:,i])[1] == 1 and np.round(val[-1][:,:,i])[4] == 1):\n",
    "        RL.append(val[0][i])\n",
    "    elif(np.round(val[-1][:,:,i])[1] == 1 and np.round(val[-1][:,:,i])[5] == 1):\n",
    "        RR.append(val[0][i])\n",
    "    else:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'RL' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-1c02e33878d4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mRL\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_counts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mLL\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_counts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mRR\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_counts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mLR\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_counts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'RL' is not defined"
     ]
    }
   ],
   "source": [
    "np.unique(RL, return_counts = True)\n",
    "np.unique(LL, return_counts = True)\n",
    "np.unique(RR, return_counts = True)\n",
    "np.unique(LR, return_counts = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}