data_cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/tron/Desktop/recom/ml-100k\n"
     ]
    }
   ],
   "source": [
    "cd ml-100k/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0m\u001b[01;32mallbut.pl\u001b[0m*  u1.base  u2.test  u4.base  u5.test  ub.base  u.data~  u.item\r\n",
      "\u001b[01;32mmku.sh\u001b[0m*     u1.test  u3.base  u4.test  ua.base  ub.test  u.genre  u.occupation\r\n",
      "README      u2.base  u3.test  u5.base  ua.test  u.data   u.info   u.user\r\n"
     ]
    }
   ],
   "source": [
    "ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy import array\n",
    "import matplotlib as plt\n",
    "from random import sample\n",
    "from numpy import zeros\n",
    "from math import sqrt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "org_dataset = np.zeros((944,1683))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(944, 1683)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "org_dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(\"u.data\") as infile:\n",
    "    for line in infile:\n",
    "        temp = line.split()\n",
    "        org_dataset[int(temp[0]),int(temp[1])] = int(temp[2])\n",
    "\n",
    "org_dataset[0,:] = range(org_dataset.shape[1])\n",
    "org_dataset[:,0] = range(org_dataset.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def matr(shp):\n",
    "    temp = zeros(shp)\n",
    "    temp[0,:] = range(shp[1])\n",
    "    temp[:,0] = range(shp[0])\n",
    "    return temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# a = array([1,2,3,7,4,55,6])\n",
    "# a[a[:]<6]\n",
    "# a[array([True,True,False,False,False,True,False])]\n",
    "# a[:]<6\n",
    "# a = array([[0,0,0],[0,1,1]])\n",
    "# a.nonzero()[0].shape[0]\n",
    "# a[0,:] = range(a.shape[1])\n",
    "# a[:,0] = range(a.shape[0])\n",
    "# print a.shape\n",
    "# a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "498\n"
     ]
    }
   ],
   "source": [
    "effect_list = [0]\n",
    "idx = 0\n",
    "for row in org_dataset[1:]:\n",
    "    if row.nonzero()[0].shape[0] > 60:\n",
    "        effect_list.append(idx)\n",
    "    idx+=1;\n",
    "print len(effect_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "eff_list = array(effect_list)\n",
    "dataset = org_dataset[eff_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "shp = dataset.shape\n",
    "# dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(498, 1683)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "498"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# org_dataset\n",
    "print shp\n",
    "len(eff_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Effective matrix extracted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# eff_list\n",
    "# eff_dic = {}\n",
    "# idx = 0\n",
    "# for ele in eff_list:\n",
    "#     eff_dic[ele] = idx\n",
    "#     idx+=1\n",
    "\n",
    "# rand_samp = sample(eff_list, 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# eff_dic[185]\n",
    "# eff_list[90]\n",
    "# a = array([[1,2],[3,4],[5,6],[7,8],[9,0]])\n",
    "# a[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print len(rand_samp)\n",
    "# dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "active = 51\n",
    "# active_user = zeros((i,shp[1]))\n",
    "# passive_user = zeros((shp[0]-i,shp[1]))\n",
    "# active_user = dataset[array([eff_dic[x] for x in rand_samp])]\n",
    "# passive_user = dataset[array([eff_dic[x] for x in eff_list if eff_dic[x] not in rand_samp])]\n",
    "np.random.shuffle(dataset[1:])\n",
    "active_user = zeros((active,shp[1]))\n",
    "passive_user = zeros((shp[0]-active+1,shp[1]))\n",
    "active_user = dataset[:active]\n",
    "passive_user[0] = dataset[0]\n",
    "passive_user[1:] = dataset[active:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  0.00000000e+00,   1.00000000e+00,   2.00000000e+00, ...,\n",
       "         1.68000000e+03,   1.68100000e+03,   1.68200000e+03])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dataset[0]\n",
    "active_user[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(51, 1683)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "active_user.shape\n",
    "# active_user[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(448, 1683)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "passive_user.shape\n",
    "# passive_user[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training and Testing Active User"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ratio_train = (70 * active_user.shape[1])/100\n",
    "# ratio_test = (30 * active_user.shape[1])/100\n",
    "ratio_test = active_user.shape[1]-ratio_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# itm_list = range(shp[1])\n",
    "# train_list = sample(itm_list,ratio_train)\n",
    "# test_list = sample(itm_list,ratio_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# np.random.rand(1,1682)\n",
    "# a = a.transpose()\n",
    "# print a\n",
    "# np.random.shuffle(a[1:])\n",
    "# print a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# idx_dataset = np.c_[eff_list,dataset]\n",
    "# idx_dataset = np.r_[array([range(0,1683)]),idx_dataset]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# print idx_dataset[0,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(range(0,1683))\n",
    "# a = array(range(0,1683)).transpose()\n",
    "# a.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# def sync(data):\n",
    "#     mat_idx,idx_mat,i  = {},{},0\n",
    "#     for idx in data[:,0]:\n",
    "#         mat_idx[idx],idx_mat[i] = i,idx\n",
    "#         i+=1\n",
    "#     return mat_idx,idx_mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# np.random.shuffle(np.transpose(a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# math.sqrt(2)\n",
    "# b = [1,2,3,4,5,6]\n",
    "# np.random.shuffle(b)\n",
    "# b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# a = array([[1,2],[3,4],[6,7],[8,9]])\n",
    "# np.random.shuffle(a)\n",
    "# np.random.rand(200,1,64,64).shape\n",
    "# type(shp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "arr_1683 = range(1683)\n",
    "\n",
    "# np.random.shuffle(np.transpose(active_user))\n",
    "# np.random.shuffle()\n",
    "ratio_train = 60*shp[1]/100\n",
    "train = matr((active,shp[1]))\n",
    "test = matr((active,shp[1]))\n",
    "train[:,0] = active_user[:,0]\n",
    "test[:,0] = active_user[:,0]\n",
    "rand_list = np.random.permutation(arr_1683)\n",
    "ltrain,ltest = rand_list[:ratio_train],rand_list[ratio_train:]\n",
    "for i in ltrain:\n",
    "    train[:,i] = active_user[:,i]\n",
    "for j in ltest:\n",
    "    test[:,j] = active_user[:,j]\n",
    "\n",
    "# test_list = []\n",
    "# for idx in train_list:\n",
    "\n",
    "# mat_idx,idx_mat = sync(idx_dataset)\n",
    "# train, test = idx_dataset[:,:ratio_train], idx_dataset[:,ratio_train:]\n",
    "# test = np.c_[[idx_mat[i] for i in range(test.shape[0])],test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 480,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# a = array([[1,2],[3,4]])\n",
    "# np.c_[[7,7],a]\n",
    "# test[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(51, 1683)\n",
      "(51, 1683)\n"
     ]
    }
   ],
   "source": [
    "# print train.shape\n",
    "# print test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# print train[:,0]\n",
    "# print test[:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 482,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# print ratio_train\n",
    "# print  ratio_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# def simlarity(row1,row2):\n",
    "\n",
    "#     temp1,temp2,temp3 = 0,0,0\n",
    "#     print np.sum([row1], axis=1)\n",
    "#     mu1 = np.sum([row1], axis=1)[0]/row1.nonzero()[0].shape[0]\n",
    "#     mu2 = np.sum([row2], axis=1)[0]/row2.nonzero()[0].shape[0]\n",
    "#     for i in range(len(row1)):\n",
    "#         if row1[i] is not 0 and row2[i] is not 0:\n",
    "#             temp1 += ((row1[i] - mu1) * (row2[i] - mu2))\n",
    "#             temp2 += (row1[i] - mu1) * (row1[i] - mu1)\n",
    "#             temp3 += (row2[i] - mu2) * (row2[i] - mu2)\n",
    "    \n",
    "#     return temp1/(sqrt(temp2) * sqrt(temp3))\n",
    "    \n",
    "# np.sum([[0, 1], [0, 5]], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 504,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# a[1].nonzero()[0].shape[0]\n",
    "# np.sum([a[1]], axis=1)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(range(3,7))\n",
    "# range(len(a[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# np.sum([[0, 1], [0, 5]], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# simlarity(train[1,1:],passive_user[1,1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.savetxt(\"train.csv\",train,delimiter=',')\n",
    "np.savetxt(\"test.csv\",test,delimiter=',')\n",
    "np.savetxt(\"passive.csv\",passive_user,delimiter=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}