EDA- Do visitors go to different levels in a visit.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Author: Eswar Lakshminarayanan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pickle\n",
    "import datetime\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.27 s, sys: 93.5 ms, total: 2.36 s\n",
      "Wall time: 1.17 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df = pd.read_csv(\"kiana_data.csv\",header=0,nrows=1000000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Are all timestamps are UTC ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.46 s, sys: 157 ms, total: 2.62 s\n",
      "Wall time: 2.62 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df[['just_date','just_time',\"ignore\"]] = df['localtime'].str.split(expand=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['UTC'], dtype=object)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.ignore.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Do visitors go to different levels in a visit?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.81 ms, sys: 9 µs, total: 2.81 ms\n",
      "Wall time: 1.82 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import pandas as pd\n",
    "chunk_df = pd.read_csv(\"kiana_data.csv\",header=0, chunksize=10000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### The two functions below reshapes the data into a long trajectory format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "counter = 0\n",
    "list_ = []\n",
    "for d in chunk_df:\n",
    "    if counter<10:\n",
    "        counter+=1\n",
    "        temp_d = d.groupby([\"ClientMacAddr\",\"just_date\"]).apply(lambda grp: grp[[\"Level\",\"lat\",\"lng\",\"just_time\"]].values.tolist()).to_frame()\n",
    "        temp_d = temp_d.reset_index()\n",
    "        list_.append(temp_d)\n",
    "    else:\n",
    "        break\n",
    "        \n",
    "frame = pd.concat(list_, axis = 0, ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mac_id</th>\n",
       "      <th>date</th>\n",
       "      <th>loc_info</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2018-07-11</td>\n",
       "      <td>[[L5, 1.29086614, 103.85172392, 18:42:35.942],...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2018-07-12</td>\n",
       "      <td>[[L5, 1.29086614, 103.85172392, 19:12:05.708],...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mac_id        date                                           loc_info\n",
       "0       0  2018-07-11  [[L5, 1.29086614, 103.85172392, 18:42:35.942],...\n",
       "1       0  2018-07-12  [[L5, 1.29086614, 103.85172392, 19:12:05.708],..."
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frame.columns = [\"mac_id\",\"date\",\"loc_info\"]\n",
    "frame.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 9.07 s, sys: 228 ms, total: 9.3 s\n",
      "Wall time: 9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "t2 = frame.groupby([\"mac_id\",\"date\"]).agg({'loc_info': 'sum'}).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mac_id</th>\n",
       "      <th>date</th>\n",
       "      <th>loc_info</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8</td>\n",
       "      <td>2018-04-04</td>\n",
       "      <td>[[L1, 1.2901341, 103.851295, 16:52:15]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11</td>\n",
       "      <td>2018-04-04</td>\n",
       "      <td>[[L1, 1.2900982, 103.85135, 15:38:57]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>36</td>\n",
       "      <td>2018-04-06</td>\n",
       "      <td>[[L1, 1.2903618, 103.851746, 14:03:53]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>36</td>\n",
       "      <td>2018-04-15</td>\n",
       "      <td>[[L1, 1.2901032, 103.85134, 13:51:42], [L1, 1....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>36</td>\n",
       "      <td>2018-05-13</td>\n",
       "      <td>[[L1, 1.2901355, 103.85129, 16:56:10]]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mac_id        date                                           loc_info\n",
       "0       8  2018-04-04            [[L1, 1.2901341, 103.851295, 16:52:15]]\n",
       "1      11  2018-04-04             [[L1, 1.2900982, 103.85135, 15:38:57]]\n",
       "2      36  2018-04-06            [[L1, 1.2903618, 103.851746, 14:03:53]]\n",
       "3      36  2018-04-15  [[L1, 1.2901032, 103.85134, 13:51:42], [L1, 1....\n",
       "4      36  2018-05-13             [[L1, 1.2901355, 103.85129, 16:56:10]]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t2.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The function below checks if the trajectories have more than one levels (L1,LB1,L5) in them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check(z_list):\n",
    "    first_val = None\n",
    "    if len(z_list)>1:\n",
    "        for sub_l in z_list:\n",
    "            if not first_val:\n",
    "                first_val = sub_l[0]\n",
    "            else:\n",
    "                if first_val== sub_l[0]:\n",
    "                    return False\n",
    "                else:\n",
    "                    return True\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = t2[\"loc_info\"].apply(lambda z: check(z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['L5', 1.2904733, 103.85147434, '02:48:18.336'],\n",
       " ['L1', 1.2896977862160839, 103.8511046543257, '23:54:48.226']]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t2.loc[tmp].iloc[0,2]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}