_02_nb.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img tyle=\"float: right;\"  src=\"http://minneanalytics.org/wp/wp-content/uploads/2018/04/BDT18_LP-02-02.jpg\" \\>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## import numpy and pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.simplefilter(\"ignore\")\n",
    "warnings.filterwarnings(\"ignore\", \"DeprecationWarning\") \n",
    "warnings.filterwarnings(\"ignore\", \"SAWarning\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:07.136941Z",
     "start_time": "2017-09-02T13:03:06.601445Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:07.497129Z",
     "start_time": "2017-09-02T13:03:07.143030Z"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext cython"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set some pandas global options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:07.508333Z",
     "start_time": "2017-09-02T13:03:07.500269Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', 20)\n",
    "pd.set_option('display.max_rows', 20)\n",
    "pd.set_option('max_info_rows', 20)\n",
    "pd.set_option('expand_frame_repr', True)\n",
    "pd.set_option('mode.chained_assignment','warn')\n",
    "pd.set_option('precision', 2)\n",
    "pd.set_option('float_format', '{:6.2f}'.format)\n",
    "pd.set_option('display.notebook_repr_html', True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get public datasets from the web"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### IMDB movie rating dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:07.847513Z",
     "start_time": "2017-09-02T13:03:07.510936Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "imdbratings = pd.read_csv('http://bit.ly/imdbratings')\n",
    "imdbratings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### UFO sightings dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:08.317861Z",
     "start_time": "2017-09-02T13:03:07.849875Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "uforeports = pd.read_csv('http://bit.ly/uforeports')\n",
    "uforeports"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Food orders dataset (Chipotle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:08.702808Z",
     "start_time": "2017-09-02T13:03:08.320879Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "chiporders = pd.read_table('http://bit.ly/chiporders')\n",
    "chiporders"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Movie users dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:09.015952Z",
     "start_time": "2017-09-02T13:03:08.705734Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "user_cols = ['user_id','age','gender','occupation','zip_code']\n",
    "movieusers = pd.read_table('http://bit.ly/movieusers',sep='|',header=None,names=user_cols)\n",
    "movieusers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Titanic dataset (Machine learning training set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:09.347517Z",
     "start_time": "2017-09-02T13:03:09.020011Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('http://bit.ly/kaggletrain')\n",
    "train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-11-02T14:52:22.624000",
     "start_time": "2016-11-02T14:52:22.623000"
    }
   },
   "source": [
    "### Drinks per Country dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:09.621807Z",
     "start_time": "2017-09-02T13:03:09.350294Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks = pd.read_csv('http://bit.ly/drinksbycountry')\n",
    "drinks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Series and Dataframe methods"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## map series function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:09.633100Z",
     "start_time": "2017-09-02T13:03:09.624963Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train['Sex_num'] = train.Sex.map({'male':0,'female':1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:09.646854Z",
     "start_time": "2017-09-02T13:03:09.635998Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:09.665571Z",
     "start_time": "2017-09-02T13:03:09.650209Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train.loc[0:4, ['Sex','Sex_num']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## apply series function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:09.682010Z",
     "start_time": "2017-09-02T13:03:09.669102Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train['Name_length'] = train.Name.apply(len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:09.702151Z",
     "start_time": "2017-09-02T13:03:09.684758Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train.loc[0:4, ['Name','Name_length']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:09.713339Z",
     "start_time": "2017-09-02T13:03:09.705307Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train['Fare_ceil'] = train.Fare.apply(np.ceil)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:09.734372Z",
     "start_time": "2017-09-02T13:03:09.717222Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train.loc[0:4, ['Fare','Fare_ceil']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## string method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:09.765980Z",
     "start_time": "2017-09-02T13:03:09.736849Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train.filter(like=\"Name\").head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:09.788537Z",
     "start_time": "2017-09-02T13:03:09.771993Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train.Name.str.split(',').head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### define python function called with pandas apply method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.064447Z",
     "start_time": "2017-09-02T13:03:09.795823Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%cython\n",
    "def get_element(my_list,position=None):\n",
    "    return my_list[position]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### apply a defined function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.085240Z",
     "start_time": "2017-09-02T13:03:10.069557Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train.Name.str.split(',').apply(get_element,position=0).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### apply anonymous function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.106390Z",
     "start_time": "2017-09-02T13:03:10.090288Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train.Name.str.split(',').apply(lambda x: x[0]).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataframe apply"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.128409Z",
     "start_time": "2017-09-02T13:03:10.110899Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.146534Z",
     "start_time": "2017-09-02T13:03:10.130642Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks.loc[:, 'beer_servings':'wine_servings'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.157031Z",
     "start_time": "2017-09-02T13:03:10.149856Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks.loc[:, 'beer_servings':'wine_servings'].apply(max, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.170941Z",
     "start_time": "2017-09-02T13:03:10.160113Z"
    },
    "run_control": {
     "marked": false
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks['max'] = drinks.loc[:, 'beer_servings':'wine_servings'].apply(\n",
    "    max, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.190535Z",
     "start_time": "2017-09-02T13:03:10.173512Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks['max_col'] = drinks.loc[:, 'beer_servings':'wine_servings'].apply(\n",
    "    np.argmax, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.208633Z",
     "start_time": "2017-09-02T13:03:10.193360Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks.loc[:, ['max', 'max_col']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.222397Z",
     "start_time": "2017-09-02T13:03:10.210768Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks.loc[:, 'beer_servings':'wine_servings'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.232750Z",
     "start_time": "2017-09-02T13:03:10.224520Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.251862Z",
     "start_time": "2017-09-02T13:03:10.235260Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks.loc[:, 'beer_servings':'wine_servings'].applymap(float).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.268267Z",
     "start_time": "2017-09-02T13:03:10.254087Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.288510Z",
     "start_time": "2017-09-02T13:03:10.271431Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks.loc[: , 'continent':'max']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.305748Z",
     "start_time": "2017-09-02T13:03:10.290845Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks.iloc[: , 5:7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "drinks.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.335173Z",
     "start_time": "2017-09-02T13:03:10.309173Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks.loc[:, 'beer_servings':'wine_servings'] = drinks.loc[:, 'beer_servings':'wine_servings'].applymap(float)\n",
    "drinks.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "drinks.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.346473Z",
     "start_time": "2017-09-02T13:03:10.337761Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks['beer_servings'] = drinks['beer_servings'].astype(np.object)\n",
    "drinks.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bonus: how to update the content of a  column on a specific row."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.353832Z",
     "start_time": "2017-09-02T13:03:10.350048Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# row at index position 0!\n",
    "drinks.at[0,'beer_servings'] = '*** updated *** number 1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.362437Z",
     "start_time": "2017-09-02T13:03:10.357244Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks.loc[1,'beer_servings'] = '*** updated *** number 2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.369110Z",
     "start_time": "2017-09-02T13:03:10.365657Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks.iloc[2,1] = '*** updated *** number 3'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.395893Z",
     "start_time": "2017-09-02T13:03:10.371958Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.416916Z",
     "start_time": "2017-09-02T13:03:10.398831Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Removing rows by the row index\n",
    "drinks.drop(drinks.index[[1,3]]).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.439608Z",
     "start_time": "2017-09-02T13:03:10.418779Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Selecting specific rows that meet the desired criteria\n",
    "drinks.ix[drinks['spirit_servings'] == 5].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.468226Z",
     "start_time": "2017-09-02T13:03:10.442311Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Removing rows that do not meet the desired criteria\n",
    "drinks.ix[~(drinks['spirit_servings'] < 130)].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.501736Z",
     "start_time": "2017-09-02T13:03:10.471815Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks.where(drinks.wine_servings > 100).head(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-02T13:03:10.526881Z",
     "start_time": "2017-09-02T13:03:10.504873Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "drinks.filter(like='serv')"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python [conda env:snake27]",
   "language": "python",
   "name": "conda-env-snake27-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 0,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "nav_menu": {},
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "67px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_position": {
    "height": "663px",
    "left": "0px",
    "right": "1351.2px",
    "top": "57px",
    "width": "201px"
   },
   "toc_section_display": "block",
   "toc_window_display": true,
   "widenNotebook": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}