exploratory_analysis_worksheet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.style.use('ggplot')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 🐼 Exploratory Analysis - Coding Challenge\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## By Ruben Kindl & Valentin Lorenzen"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Read in the ratings and movies tables\n",
    "\n",
    "- Download the data from https://grouplens.org/datasets/movielens/\n",
    "- recommended for education and development\n",
    "- Take the **small** dataset: `ml-latest-small.zip`\n",
    "- Extract the tables from the `.zip` archive\n",
    "\n",
    "- ⭐ Advanced Challenge: Try out to process the larger datasets\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ratings = pd.read_csv(\"ml-latest-small/ratings.csv\")\n",
    "movies = pd.read_csv(\"ml-latest-small/movies.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>191</th>\n",
       "      <td>224</td>\n",
       "      <td>Don Juan DeMarco (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     movieId                    title                genres\n",
       "191      224  Don Juan DeMarco (1995)  Comedy|Drama|Romance"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.loc[movies['movieId'] == 224]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [movieId, title, genres]\n",
       "Index: []"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.loc[movies['title'].isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3556</th>\n",
       "      <td>4867</td>\n",
       "      <td>Riding in Cars with Boys</td>\n",
       "      <td>Comedy|Drama</td>\n",
       "      <td>2001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6212</th>\n",
       "      <td>45517</td>\n",
       "      <td>Cars</td>\n",
       "      <td>Animation|Children|Comedy</td>\n",
       "      <td>2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7639</th>\n",
       "      <td>87876</td>\n",
       "      <td>Cars 2</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|IMAX</td>\n",
       "      <td>2011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9509</th>\n",
       "      <td>170957</td>\n",
       "      <td>Cars 3</td>\n",
       "      <td>Adventure|Animation|Comedy</td>\n",
       "      <td>2017</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      movieId                     title  \\\n",
       "3556     4867  Riding in Cars with Boys   \n",
       "6212    45517                      Cars   \n",
       "7639    87876                    Cars 2   \n",
       "9509   170957                    Cars 3   \n",
       "\n",
       "                                        genres  year  \n",
       "3556                              Comedy|Drama  2001  \n",
       "6212                 Animation|Children|Comedy  2006  \n",
       "7639  Adventure|Animation|Children|Comedy|IMAX  2011  \n",
       "9509                Adventure|Animation|Comedy  2017  "
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.loc[movies['title'].str.contains('Cars')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "rating = ratings.groupby('movieId')['rating'].mean()[548]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "round(rating)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Ratings table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### How many unique users and movies are in the dataset?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "610"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings['userId'].nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### What is the average rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.501556983616962"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings['rating'].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### which rating appears the most?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    4.0\n",
       "Name: rating, dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings['rating'].mode()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### What is the percentage of 5 star ratings?\n",
    "\n",
    "- ⭐**Bonus**: How are the ratings distributed? Derive a relative frequency table for the `rating` column!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "relative_frequencies = ratings['rating'].value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "relative_frequencies = pd.DataFrame(relative_frequencies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.265957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.198808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5.0</th>\n",
       "      <td>0.131015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.5</th>\n",
       "      <td>0.130271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.5</th>\n",
       "      <td>0.084801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.074884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.5</th>\n",
       "      <td>0.055040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.027877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.5</th>\n",
       "      <td>0.017762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.5</th>\n",
       "      <td>0.013586</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       rating\n",
       "4.0  0.265957\n",
       "3.0  0.198808\n",
       "5.0  0.131015\n",
       "3.5  0.130271\n",
       "4.5  0.084801\n",
       "2.0  0.074884\n",
       "2.5  0.055040\n",
       "1.0  0.027877\n",
       "1.5  0.017762\n",
       "0.5  0.013586"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "relative_frequencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of 5 star ratings:  13.101 %\n"
     ]
    }
   ],
   "source": [
    "print(\"Percentage of 5 star ratings: \", round(len(ratings.loc[ratings['rating'] == 5]) / len(ratings) * 100, ndigits=3), \"%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "#### Count the number of ratings *per* user\n",
    "\n",
    "- Hint: Use a `groupby` statement and the `count` method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "userId\n",
       "1       232\n",
       "2        29\n",
       "3        39\n",
       "4       216\n",
       "5        44\n",
       "       ... \n",
       "606    1115\n",
       "607     187\n",
       "608     831\n",
       "609      37\n",
       "610    1302\n",
       "Name: rating, Length: 610, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.groupby('userId')['rating'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- How many users have rated more than 20 movies?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "593 users have rated more than 20 movies\n"
     ]
    }
   ],
   "source": [
    "print(len(ratings.groupby('userId').count().loc[ratings['movieId'] > 20]), \"users have rated more than 20 movies\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- How many movies rated the user with the most/ least ratings?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The user who has rated the most movies rated 2698 movies\n"
     ]
    }
   ],
   "source": [
    "print(\"The user who has rated the most movies rated\" , ratings.groupby('userId')['rating'].count().sort_values().max(), \"movies\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The user who has rated the least movies rated 20 movies\n"
     ]
    }
   ],
   "source": [
    "print(\"The user who has rated the least movies rated\" , ratings.groupby('userId')['rating'].count().sort_values().min(), \"movies\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-  What is the *median* number of ratings?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The median number of ratings is: 70.5\n"
     ]
    }
   ],
   "source": [
    "print(\"The median number of ratings is:\", ratings.groupby('userId')['rating'].count().median())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Draw a histogram or density plot of the distribution of the number of ratings per user! \n",
    "\n",
    "    - How would you characterize the distribution?\n",
    "    - Hint: `sns.histplot()`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='rating', ylabel='Count'>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEJCAYAAACOr7BbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZaElEQVR4nO3dfWxT973H8Y8dh6eFJHYcYOFhGg1sRbClLKEjpaQFU3UFTSiltEgMQVERAlaVqGhMm6iqFClTGwKrwigT2taiddABqbR/qhsCQWsGzYZa7oXRNjxMUBiJY+fpkkAcn/tHh29IfyFOSXwc5/36Kz7n5Pj7zYn88Tk/n58dlmVZAgCgB6fdBQAA4hMBAQAwIiAAAEYEBADAiIAAABgREAAAI5fdBQyka9eu9Wt7r9crv98/SNXEB3pMDPQ49MVrf1lZWb2u4wwCAGBEQAAAjAgIAIARAQEAMCIgAABGBAQAwIiAAAAYERAAAKOEulHufjgcjsjPfEUGABAQkr4Mhz/9T6Ma2m4rM2WEnpuZQUgAGPYIiP9oaLut66237S4DAOIGYxAAACMCAgBgREAAAIwICACAEQEBADAiIAAARgQEAMCIgAAAGBEQAAAjAgIAYERAAACMCAgAgBEBAQAwIiAAAEYEBADAiIAAABgREAAAIwICAGBEQAAAjAgIAIARAQEAMCIgAABGBAQAwIiAAAAYERAAACMCAgBgREAAAIxcsXgSv9+v8vJyNTU1yeFwyOfz6amnnlJbW5vKysrU0NCgzMxMbd68WSkpKZKkI0eOqKqqSk6nU2vWrFFOTk4sSgUA/EdMAiIpKUk/+clPNHXqVLW3t2vr1q363ve+p+PHj2vWrFlaunSpKioqVFFRoZUrV+rq1auqqanRjh07FAwGVVxcrF27dsnp5IQHAGIlJq+4brdbU6dOlSSNHj1aEydOVCAQUG1trQoKCiRJBQUFqq2tlSTV1tYqPz9fycnJGjdunCZMmKC6urpYlAoA+I+YnEF0V19fr0uXLik7O1vNzc1yu92SvgyRlpYWSVIgENC0adMiv+PxeBQIBL6yr8rKSlVWVkqSSkpK5PV6+1WLy+WS1+tVOByWy9Wg5GRLLpdLbrc7Yc5W7vSYyOgxMSR6j0Oxv5gGREdHh0pLS7V69WqNGTOm1+0sy4pqfz6fTz6fL/LY7/f3qx6v1yu/3y+Hw6FQKKTOzk6FQg4Fg8Goa4h3d3pMZPSYGBK9x3jtLysrq9d1MXubHAqFVFpaqkcffVQPP/ywJCktLU3BYFCSFAwGlZqaKknKyMhQY2Nj5HcDgYA8Hk+sSgUAKEYBYVmW9uzZo4kTJ2rJkiWR5bm5uaqurpYkVVdXKy8vL7K8pqZGnZ2dqq+v1/Xr15WdnR2LUgEA/xGTS0yffvqpTpw4oSlTpmjLli2SpBUrVmjp0qUqKytTVVWVvF6vioqKJEmTJ0/W3LlzVVRUJKfTqbVr1ybMmAAADBUxCYjvfve7OnjwoHHdtm3bjMsLCwtVWFg4mGUBAO6Bt+UAACMCAgBgREAAAIwICACAEQEBADAiIAAARgQEAMCIgAAAGBEQAAAjAgIAYERAAACMCAgAgBEBAQAwIiAAAEYEBADAiIAAABgREAAAIwICAGBEQAAAjAgIAIARAQEAMCIgAABGBAQAwIiAAAAYERAAACMCAgBgREAAAIwICACAEQEBADAiIAAARgQEAMCIgAAAGBEQAAAjAgIAYERAAACMXLF4kt27d+v06dNKS0tTaWmpJOngwYM6evSoUlNTJUkrVqzQ7NmzJUlHjhxRVVWVnE6n1qxZo5ycnFiUCQDoJiYB8dhjj+nJJ59UeXn5XcsXL16sH//4x3ctu3r1qmpqarRjxw4Fg0EVFxdr165dcjo52QGAWIrJq+6MGTOUkpIS1ba1tbXKz89XcnKyxo0bpwkTJqiurm6QKwQA9BSTM4jefPDBBzpx4oSmTp2qVatWKSUlRYFAQNOmTYts4/F4FAgEjL9fWVmpyspKSVJJSYm8Xm+/nt/lcsnr9SocDsvlalBysiWXyyW3250wZyx3ekxk9JgYEr3HodifbQHxxBNPaNmyZZKkAwcO6O2339aGDRtkWVbU+/D5fPL5fJHHfr+/XzV4vV75/X45HA6FQiF1dnYqFHIoGAz2q454dqfHREaPiSHRe4zX/rKysnpdZ9vb5PT0dDmdTjmdTi1cuFAXLlyQJGVkZKixsTGyXSAQkMfjsatMABi2bAuIYDAY+fmjjz7S5MmTJUm5ubmqqalRZ2en6uvrdf36dWVnZ9tVJgAMWzG5xLRz506dO3dOra2tWr9+vZYvX66zZ8/q8uXLcjgcyszM1Lp16yRJkydP1ty5c1VUVCSn06m1a9cmzHgAAAwlMQmIl1566SvLFixY0Ov2hYWFKiwsHMSKAAB94a05AMAo6oD429/+Zlx+8uTJASsGABA/og6IPXv2GJe/9dZbA1YMACB+9DkGcePGDUlSOBxWfX39XfcH3LhxQyNGjBi86gAAtukzIF588cXIzz/96U/vWpeenq5nnnlm4KsCANiuz4A4cOCAJOmVV17Rq6++OugFAQDiQ9RjEIQDAAwvUd8HUV9fr3fffVeXL19WR0fHXet+85vfDHhhAAB7RR0Qu3bt0vjx47Vq1SqNHDlyMGsCAMSBqAPi6tWrKi4uZtoLABgmon61f/DBB3X58uVBLAUAEE+iPoPIzMzU9u3bNWfOHKWnp9+17tlnnx3ougAANos6IG7duqUf/OAH6urquuv7GgAAiSnqgNiwYcNg1gEAiDNRB8SdKTdMxo8fPyDFAADiR9QB0X3KjZ7u3G0NAEgcUQdEzxBoamrSe++9pwcffHDAiwIA2O9r39SQnp6u1atX649//ONA1gMAiBP3ddfbtWvXdOvWrYGqBQAQR6K+xLRt2zY5HI7I41u3bunKlStatmzZoBQGALBX1AGxYMGCux6PGjVK3/rWt/TNb35zwIsCANgv6oB47LHHBrEMAEC8iTogQqGQDh8+rBMnTigYDMrtdmv+/PkqLCyUyxX1bgAAQ0TUr+z79+/XhQsX9MILLygzM1MNDQ06dOiQbt68qdWrVw9iiQAAO0QdECdPntTrr7+usWPHSpKysrL07W9/W1u2bCEgACABRf0xV8uyBrMOAECcifoMYu7cufrVr36lZcuWyev1yu/369ChQ/rhD384mPUBAGwSdUCsXLlShw4d0r59+xQMBuXxePTII4/o6aefHsz6AAA26TMgzp8/r7///e9auXKlnn322bu+HGj//v26ePGipk+fPqhFAgBir88xiCNHjmjGjBnGdTNnztThw4cHvCgAgP36DIjLly8rJyfHuG7WrFm6dOnSQNcEAIgDfQZEe3u7QqGQcV1XV5fa29sHvCgAgP36DIiJEyfqk08+Ma775JNPNHHixAEvCgBgvz4DYvHixdq7d69OnTqlcDgsSQqHwzp16pR++9vfavHixYNeJAAg9vr8FNO8efPU1NSk8vJydXZ2KjU1VS0tLRoxYoSeeeYZzZs3r88n2b17t06fPq20tDSVlpZKktra2lRWVqaGhgZlZmZq8+bNSklJkfTlwHhVVZWcTqfWrFnT6xgIAGDwRHUfxJIlS7RgwQJ99tlnamtrU0pKiqZPn64xY8ZE9SSPPfaYnnzySZWXl0eWVVRUaNasWVq6dKkqKipUUVGhlStX6urVq6qpqdGOHTsUDAZVXFysXbt2yem8r+82AgD0U9SvumPGjFFOTo7mzZunnJycqMNBkmbMmBE5O7ijtrZWBQUFkqSCggLV1tZGlufn5ys5OVnjxo3ThAkTVFdXF/VzAQAGhm3zdDc3N8vtdkuS3G63WlpaJEmBQEDTpk2LbOfxeBQIBIz7qKysVGVlpSSppKREXq+3XzW4XC55vV6Fw2G5XA1KTrbkcrnkdrsT5ozlTo+JjB4TQ6L3OBT7i7svcujPpIA+n08+ny/y2O/39+u57swp5XA4FAqF1NnZqVDIoWAwmDCTE97pMZHRY2JI9B7jtb+srKxe19n2NjktLU3BYFCSFAwGlZqaKknKyMhQY2NjZLtAICCPx2NLjQAwnNkWELm5uaqurpYkVVdXKy8vL7K8pqZGnZ2dqq+v1/Xr15WdnW1XmQAwbMXkEtPOnTt17tw5tba2av369Vq+fLmWLl2qsrIyVVVVyev1qqioSJI0efJkzZ07V0VFRXI6nVq7dm3CjAcAwFASk4B46aWXjMu3bdtmXF5YWKjCwsJBrAgA0Je4G6S2m8MhORyOu5YlyoA1APQHAdFDxphkvfvffjW03ZYkZaaM0HMzMwgJAMMOAWHQ0HZb11tv210GANiK0V8AgBEBAQAwIiAAAEYEBADAiIAAABgREAAAIwICAGBEQAAAjAgIAIARAQEAMCIgAABGBAQAwIiAAAAYERAAACMCAgBgREAAAIwICACAEQEBADAiIAAARgQEAMCIgAAAGBEQAAAjAgIAYERAAACMCAgAgJHL7gLincMhORyOyGPLsmysBgBih4DoQ8aYZL373341tN1WZsoIPTczg5AAMCwQEFFoaLut66237S4DAGKKMQgAgBEBAQAwIiAAAEYEBADAyPZB6o0bN2rUqFFyOp1KSkpSSUmJ2traVFZWpoaGBmVmZmrz5s1KSUmxu1QAGFZsDwhJeuWVV5Samhp5XFFRoVmzZmnp0qWqqKhQRUWFVq5caWOFADD8xOUlptraWhUUFEiSCgoKVFtba3NFADD8xMUZxPbt2yVJixYtks/nU3Nzs9xutyTJ7XarpaXF+HuVlZWqrKyUJJWUlMjr9fbreV0ul7xer8LhsFyuBiUnW0pKcsmVZCk5+cub4bo/drlccrvdcjrjMleN7vSYyOgxMSR6j0OxP9sDori4WB6PR83NzXrttdeUlZUV9e/6fD75fL7IY7/f36/n9nq98vv9cjgcCoVC6uzsVFdXSKGuL3+WdNfjUMihYDA4pO6kvtNjIqPHxJDoPcZrf/d6zbX9rbDH45EkpaWlKS8vT3V1dUpLS1MwGJQkBYPBu8YnAACxYWtAdHR0qL29PfLzmTNnNGXKFOXm5qq6ulqSVF1drby8PDvLBIBhydZLTM3NzXrjjTckSV1dXZo3b55ycnL0wAMPqKysTFVVVfJ6vSoqKrKzzIieM7tKzO4KIHHZGhDjx4/X66+//pXlY8eO1bZt22yo6N66z+wqidldASQ02wephxpmdgUwXNg+SA0AiE8EBADAiIAAABgREAAAIwICAGBEQAAAjAgIAIARAQEAMCIgAABGBAQAwIiAAAAYMRfTfeg5uyuT9gFIJATEfeg+uyszuwJINATEfWJ2VwCJijEIAIARAQEAMCIgAABGjEEMIj7hBGAoIyAGSM+PvDocDj7hBGBIIyAGSPePvErStMwxfMIJwJBGQAyg7oHg/UayzdUAwP1hkBoAYERAAACMCAgAgBFjEDbhI7AA4h0BYQOHw6E//U8jH4EFENcIiBgw3SPBR2ABxDsCIgZM90gAQLwjIGKEeyQADDUExBDHYDeAwUJAxLnuAWBax3xPAAYLAWGzngPYd69zfGXsoqk9ZJzvqbf9hMNhORwOggNAvxEQNjMNYN8JgZ4T/nm/kSz//3YaxzJ620+wo0HuUU7OLgD0GwERB3oLgf4OZpv24++wFAp99cxiIMYuep6xEEBAYonrgPj444/1u9/9TuFwWAsXLtTSpUvtLmlIutd3VYxLGaHnZnn7/eLe8/JXf8dA7jW20n0f0W7Xc1vCCrh/cRsQ4XBY+/bt0y9/+UtlZGTo5z//uXJzczVp0iS7Sxty7vVdFd5v3PsSV88xj94uf/UMoXu9QHe/k7znc3QPrHuNwfQMpO77NIXevUKnr1p7c6+A6mvbRGPH2WQ8vyEY7Npi9feO24Coq6vThAkTNH78eElSfn6+amtrBy0gMlNGSJLcY5Lv+uN3fxxP66Ld1uUKyz0mWU3toV777bkuWnf2IUnZ3jGqvNiiYHun0kcna9HU1F7/ae/1Qpo22hXZz+T0Ufd8/jv7uTMQb9qHpLvqcTgc+q+LLWoyrDPtv/u2k9NHqe1Wl7HH7tt2366v54hWzx7jSX/+pvfSnx67P+dA/H0HUm+1DdQxNP29fd8eOzhBZMXLX7WHkydP6uOPP9b69eslSSdOnNDnn3+utWvXRraprKxUZWWlJKmkpMSWOgEgUcXtdN+9vaPrzufzqaSk5GuHw9atW7/W7w0l9JgY6HHoG4r9xW1AZGRkqLGxMfK4sbFRbrfbxooAYHiJ24B44IEHdP36ddXX1ysUCqmmpka5ubl2lwUAw0bcDlInJSXp+eef1/bt2xUOh/X4449r8uTJA/ocPp9vQPcXj+gxMdDj0DcU+4vbQWoAgL3i9hITAMBeBAQAwChuxyAGW6JM47Fx40aNGjVKTqdTSUlJKikpUVtbm8rKytTQ0KDMzExt3rxZKSkpkqQjR46oqqpKTqdTa9asUU5Ojr0NGOzevVunT59WWlqaSktLJelr9XTx4kWVl5fr9u3beuihh7RmzZq4udnM1OPBgwd19OhRpaamSpJWrFih2bNnSxqaPfr9fpWXl6upqUkOh0M+n09PPfVUwhzL3vpLqONoDUNdXV3Wpk2brH//+99WZ2en9fLLL1tXrlyxu6yvZcOGDVZzc/Ndy9555x3ryJEjlmVZ1pEjR6x33nnHsizLunLlivXyyy9bt2/ftm7cuGFt2rTJ6urqinXJfTp79qx14cIFq6ioKLLs6/S0detW69NPP7XC4bC1fft26/Tp0zHvpTemHg8cOGC9//77X9l2qPYYCASsCxcuWJZlWTdv3rRefPFF68qVKwlzLHvrL5GO47C8xNR9Gg+XyxWZxiNR1NbWqqCgQJJUUFAQ6a22tlb5+flKTk7WuHHjNGHCBNXV1dlZqtGMGTMi7yjv6G9PwWBQ7e3tmj59uhwOh+bPnx9Xx9jUY2+Gao9ut1tTp06VJI0ePVoTJ05UIBBImGPZW3+9GWr9ScP0ElMgEFBGRkbkcUZGhj7//HMbK7o/27dvlyQtWrRIPp9Pzc3NkZsK3W63WlpaJH3Z97Rp0yK/5/F47vkPHU/621NSUtJXjvFQ6PWDDz7QiRMnNHXqVK1atUopKSkJ0WN9fb0uXbqk7OzshDyW3fs7f/58whzHYRkQVhTTeAwVxcXF8ng8am5u1muvvaasrKxetzX1PdT11tNQ7PWJJ57QsmXLJEkHDhzQ22+/rQ0bNgz5Hjs6OlRaWqrVq1drzJgxvW43VPvs2V8iHcdheYkpkabx8Hg8kqS0tDTl5eWprq5OaWlpCgaDkqRgMBgZLOvZdyAQiPx+vOtvT6ZjHO+9pqeny+l0yul0auHChbpw4YKkod1jKBRSaWmpHn30UT388MOSEutYmvpLpOM4LAMiUabx6OjoUHt7e+TnM2fOaMqUKcrNzVV1dbUkqbq6Wnl5eZKk3Nxc1dTUqLOzU/X19bp+/bqys7Ntq78/+tuT2+3W6NGj9dlnn8myLJ04cSLuj/GdF01J+uijjyIzBwzVHi3L0p49ezRx4kQtWbIksjxRjmVv/SXScRy2d1KfPn1af/jDHyLTeBQWFtpdUr/duHFDb7zxhiSpq6tL8+bNU2FhoVpbW1VWVia/3y+v16uioqLIgOjhw4d17NgxOZ1OrV69Wg899JCdLRjt3LlT586dU2trq9LS0rR8+XLl5eX1u6cLFy5o9+7dun37tnJycvT888/HzaVEU49nz57V5cuX5XA4lJmZqXXr1kXObIdij+fPn9e2bds0ZcqUSE0rVqzQtGnTEuJY9tbfhx9+mDDHcdgGBADg3oblJSYAQN8ICACAEQEBADAiIAAARgQEAMCIgAAG2d69e/XnP//Z7jKAfuNjrsAAOn78uI4ePari4mK7SwHuG2cQQD90dXXZXQIQM5xBAH3YuHGjFi1apL/+9a+6du2ann76aR0/flzNzc3KyMjQihUrNGfOHF29elU/+9nPFAqFNGLECCUlJen3v/+9ysvLlZGRoeeee05nz57Vm2++qcWLF+v999+X0+nUihUr9Pjjj0uSWltbVV5ern/+85/KysrS97//fZ09e5YzEthiWM7mCvTXhx9+qK1btyo1NVX/+Mc/9Oqrryo9PV0nT57Um2++qV//+teaNGmSXnjhhT4vMTU1NenmzZvas2ePzpw5ox07digvL08pKSnat2+fRo0apb1796qhoUHbt2+X1+uNYafA/+MSExCFH/3oR/J6vRoxYoTmzp0rj8cjp9Op/Pz8fn/xUlJSkpYtWyaXy6XZs2dr1KhRunbtmsLhsE6dOqXly5dr5MiRmjRpUuSLdQA7cAYBRKH7u/jq6mr95S9/UUNDg6QvZ9JtbW2Nel9jx45VUlJS5PHIkSPV0dGhlpYWdXV1feXLYwC7EBBAPzQ0NOitt97Stm3bNH36dDmdTm3ZsmVAvvQlNTVVSUlJamxsjHzxU/fvCQBijUtMQD/cunVLDocj8iU3x44d05UrVyLr09PTFQgEFAqF+r1vp9OpOXPm6L333tOtW7f0xRdfRL43AbADZxBAP0yaNElLlizRL37xCzmdTs2fP1/f+c53IutnzpwZGax2Op3at29fv/a/du1alZeXa926dcrKytIjjzyiixcvDnQbQFT4mCsQx/bv36+mpiZt2rTJ7lIwDHGJCYgjX3zxhf71r3/JsizV1dXp2LFjmjNnjt1lYZjiEhMQR9rb27Vr1y4Fg0GlpaVpyZIlke9sBmKNS0wAACMuMQEAjAgIAIARAQEAMCIgAABGBAQAwOj/AHGqJ/tyl3kJAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.histplot(data=ratings.groupby('userId')['rating'].count())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "#### Calculate the average rating *per user*\n",
    "\n",
    "- Hint: use a `groupby` statement, select the `rating` column and apply the `mean` method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "userId\n",
       "1      4.366379\n",
       "2      3.948276\n",
       "3      2.435897\n",
       "4      3.555556\n",
       "5      3.636364\n",
       "         ...   \n",
       "606    3.657399\n",
       "607    3.786096\n",
       "608    3.134176\n",
       "609    3.270270\n",
       "610    3.688556\n",
       "Name: rating, Length: 610, dtype: float64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.groupby('userId')['rating'].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Which user gave the worst ratings on average, which user gave the best ratings?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "userId\n",
       "442    1.275000\n",
       "139    2.144330\n",
       "508    2.145833\n",
       "153    2.217877\n",
       "567    2.245455\n",
       "         ...   \n",
       "30     4.735294\n",
       "25     4.807692\n",
       "515    4.846154\n",
       "251    4.869565\n",
       "53     5.000000\n",
       "Name: rating, Length: 610, dtype: float64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.groupby('userId')['rating'].mean().sort_values()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- ⭐**Bonus**: Use your previous results and filter out users that have rated less than 20 movies! Do your results change?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "userId\n",
       "442    1.275000\n",
       "139    2.144330\n",
       "508    2.145833\n",
       "153    2.217877\n",
       "311    2.339286\n",
       "         ...   \n",
       "30     4.735294\n",
       "25     4.807692\n",
       "515    4.846154\n",
       "251    4.869565\n",
       "53     5.000000\n",
       "Name: rating, Length: 593, dtype: float64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.groupby('userId')['rating'].mean().sort_values().loc[ratings['movieId'] > 20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Plot the distribution of average ratings! How would you characterize the distribution?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='rating', ylabel='Count'>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAELCAYAAAA/cjqaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAfXklEQVR4nO3df3AU5f0H8Pfe5S4hJnfZZCN6wR+g0IpGKSQEFD1/nDpVOnNTMRYmVoaOLUKbMekwYH8EHWSaCniIExrtUHTqtNNa9To6/cO5Ro+pFXoEFYwWiqKiUZKQyx2B5C53t98/gPsSuCR75G73wvN+/eXtPs/e22eWfG6f/SWpqqqCiIiEZDI6ABERGYdFgIhIYCwCREQCYxEgIhIYiwARkcBYBIiIBJan1xe98cYbaGtrgyRJuOyyy7BixQpEo1F4PB50d3ejvLwcDQ0NKCoq0isSEZHwJD3uE+jt7cWvf/1reDweWK1WPP3005g9eza+/PJLFBUVwe12w+v1or+/H3V1ddmOQ0REp+h2JJBIJBCNRmE2mxGNRiHLMl577TU8/vjjAACn04nHH39cUxHo7OzMctqxKYqCnp4eo2OMKNfzAcyYKcyYGRd6RofDkXK5LkWgtLQU3/ve9/DII4/AarXihhtuwA033IBQKARZlgEAsiwjHA7rEYeIiE7RpQj09/cjEAigpaUFhYWFePrpp7Fjxw7N/X0+H3w+HwCgubkZiqJkK6pmeXl5OZFjJLmeD2DGTGHGzBA1oy5FYN++fbj44oths9kAADU1NThw4ADsdjuCwSBkWUYwGEyuP5vL5YLL5Up+zoVDtlw/dMz1fAAzZgozZsaFnnGk6SBdLhFVFAX/+9//EIlEoKoq9u3bh4qKClRVVcHv9wMA/H4/qqur9YhDRESn6HIkMH36dMybNw+rV6+G2WzGlVdeCZfLhcHBQXg8HrS1tUFRFDQ2NuoRh4iITtHt6qDa2lrU1tYOW2axWNDU1KRXBCIiOgvvGCYiEhiLABGRwFgEiIgEpts5ASLKHeGoilAkrrn9JIsZA0Pa29vzzbBZpfOJRjpjESASUCgSR+su7Y9feWjOpXix/WvN7ZfXOGCz8s/LRMDpICIigbEIEBEJjEWAiEhgLAJERAJjESAiEhiLABGRwFgEiIgExiJARCQwFgEiIoGxCBARCYxFgIhIYCwCREQCYxEgIhKYLo/56+zshMfjSX7u6upCbW0tnE4nPB4Puru7UV5ejoaGBhQVFekRiYiIoFMRcDgc2LBhAwAgkUjgJz/5CebOnQuv14vKykq43W54vV54vV7U1dXpEYmIiGDAdNC+fftwySWXoLy8HIFAAE6nEwDgdDoRCAT0jkNEJDTdi8A777yDm266CQAQCoUgyzIAQJZlhMNhveMQEQlN11f/xGIxtLe3Y8mSJWn18/l88Pl8AIDm5mYoipKNeGnJy8vLiRwjyfV8ADNmSl5eHqLmAgRPDGnuk5BUWCwWze1NJlNa7a3WfCiKPCzjRBhHETPqWgTee+89TJ06FSUlJQAAu92OYDAIWZYRDAZhs9lS9nO5XHC5XMnPPT09esQdlaIoOZFjJLmeD2DGTFEUBUf6+tN+XeTQUBpFI5FIq300Ghk2bhNlHC/kjA6HI+VyXaeDzpwKAoCqqir4/X4AgN/vR3V1tZ5xiIiEp1sRiEQi2Lt3L2pqapLL3G439u7di/r6euzduxdut1uvOEREBB2ng/Lz8/GHP/xh2LLi4mI0NTXpFYGIiM7CO4aJiATGIkBEJDAWASIigel6iSgRicFsMuHwsVjy85FIENFobMT29nwzbFZJj2h0FhYBIsq4/mgcL7Z/nfxssVhGvc9geY0DNiv/HBmB00FERAJjESAiEhiLABGRwFgEiIgExiJARCQwFgEiIoHxmiyiHBSOqghF4praHokEMZTIciC6YLEIEOWgUCSu+f0AFosFS67P7ZehUO5iESDKsnR+1Z/GX/akFxYBoixL51f9aQ/NuTRLaYiG44lhIiKBsQgQEQmMRYCISGC6nRM4fvw4WltbcfjwYUiShEceeQQOhwMejwfd3d0oLy9HQ0MDioqK9IpERCQ83YrA9u3bMWvWLPz85z9HLBZDJBLBa6+9hsrKSrjdbni9Xni9XtTV1ekViYhIeLpMB504cQIff/wxbr/9dgBAXl4eLrroIgQCATidTgCA0+lEIBDQIw4REZ2iy5FAV1cXbDYbtm7dis8//xzTpk3D0qVLEQqFIMsyAECWZYTD4ZT9fT4ffD4fAKC5uRmKYvyNMXl5eTmRYyS5ng8QJ+ORSBAWiyWtPiaTSXMfSZLSap/u9jPRXpKkUftbrflQFFnz9rNBlP3xnG1mdGsjiMfjOHToEJYtW4bp06dj+/bt8Hq9mvu7XC64XK7k556eniykTI+iKDmRYyS5ng8QJ2M0Ghv1rVqpJBIJzX0sFkta7dPdfibaj/VmsWg0Yvi+cKHvjw6HI+VyXaaDysrKUFZWhunTpwMA5s2bh0OHDsFutyMYDAIAgsEgbDabHnGIiOgUXYpASUkJysrK0Nl58q7Jffv2YcqUKaiqqoLf7wcA+P1+VFdX6xGHiIhO0e3qoGXLlmHLli2IxWK4+OKLsWLFCqiqCo/Hg7a2NiiKgsbGRr3iEBERdCwCV155JZqbm89Z3tTUpFcEIiI6C+8YJiISGIsAEZHAWASIiATGIkBEJDAWASIigbEIEBEJjEWAiEhgLAJERAJjESAiEhiLABGRwFgEiIgExiJARCQwFgEiIoGxCBARCYxFgIhIYCwCREQCYxEgIhIYiwARkcB0e73kypUrUVBQAJPJBLPZjObmZvT398Pj8aC7uxvl5eVoaGhAUVGRXpGIiISnWxEAgLVr18JmsyU/e71eVFZWwu12w+v1wuv1oq6uTs9IRERCM3Q6KBAIwOl0AgCcTicCgYCRcYiIhKPrkcD69esBAHfeeSdcLhdCoRBkWQYAyLKMcDicsp/P54PP5wMANDc3Q1EUfQKPIi8vLydyjCTX8wHiZDwSCcJisaTVx2Qyae4jSVJa7dPdfibaS5I0an+rNR+KImvefjaIsj+es82Mbm0U69atQ2lpKUKhEJ588kk4HA7NfV0uF1wuV/JzT09PNiKmRVGUnMgxklzPB4iTMRqNYWhoKK0+iURCcx+LxZJW+3S3n4n2Fotl1P7RaMTwfeFC3x9H+pur23RQaWkpAMBut6O6uhoHDx6E3W5HMBgEAASDwWHnC4iIKPt0ORIYHByEqqqYNGkSBgcHsXfvXixatAhVVVXw+/1wu93w+/2orq7WIw4R5RizyYTDx2Jp9bHnm2GzSllKJA5dikAoFMLGjRsBAPF4HAsWLMCsWbNw1VVXwePxoK2tDYqioLGxUY84RJRj+qNxvNj+dVp9ltc4YLPqelrzgqTLCE6ePBkbNmw4Z3lxcTGampr0iEBERCnwjmEiIoGxCBARCYxFgIhIYCwCREQC01wE3n333ZTLd+7cmbEwRESkL81FoLW1NeXy5557LmNhiIhIX2NeInrkyBEAJ28D7+rqgqqqw9ZZrdbspSMioqwaswjU19cn//tnP/vZsHUlJSW4//77M5+KiIh0MWYR+Mtf/gLg5LsAnnjiiawHIiIi/Wg+J8ACQER04dH82Iiuri78+c9/xmeffYbBwcFh6373u99lPBgREWWf5iLwzDPPYPLkyfjhD3+I/Pz8bGYiIiKdaC4CX375JdatWweTifeXERFdKDT/Rb/mmmvw2WefZTEKERHpTfORQHl5OdavX4+5c+eipKRk2LoHHngg07mIiEgHmotAJBLBnDlzEI/HcfTo0WxmIiIinWguAitWrMhmDiIiMoDmInD68RGpTJ48OSNhiIhIX5qLwJmPjzjb6buKx5JIJLBmzRqUlpZizZo16O/vh8fjQXd3N8rLy9HQ0ICioiKtkYiIaJw0F4Gz/9D39fXh5ZdfxjXXXKP5y/7xj3+goqICAwMDAACv14vKykq43W54vV54vV7U1dVp3h4REY3PeV/0X1JSgqVLl+JPf/qTpvZHjx7Fnj17cMcddySXBQIBOJ1OAIDT6UQgEDjfOEREdB40Hwmk0tnZiUgkoqntCy+8gLq6uuRRAACEQiHIsgwAkGUZ4XA4ZV+fzwefzwcAaG5uhqIo44mdEXl5eTmRYyS5ng8QJ+ORSBAWiyWtPiaTSXMfSZLSap/u9jPRXpKkUfunu30AsFrzoShyWn1GI8r+eM42tTZsamqCJEnJz5FIBIcPH8aiRYvG7Nve3g673Y5p06aho6Mj7ZAulwsulyv5uaenJ+1tZJqiKDmRYyS5ng8QJ2M0GsPQ0FBafRKJhOY+Foslrfbpbj8T7S0Wy6j9090+AESjkYzuPxf6/uhwOFIu11wEbr/99mGfCwoKcMUVV+DSSy8ds+/+/fuxe/duvPfee4hGoxgYGMCWLVtgt9sRDAYhyzKCwSBsNpvWOERElAGai8Ctt9563l+yZMkSLFmyBADQ0dGB119/HfX19fjjH/8Iv98Pt9sNv9+P6urq8/4OIiJKn+YiEIvF8Oqrr2LHjh3JX++33HILvv/97yMv7/xOLbjdbng8HrS1tUFRFDQ2Np7XdoiI6Pxo/uv90ksv4ZNPPsHDDz+M8vJydHd345VXXsGJEyewdOlSzV947bXX4tprrwUAFBcXo6mpKe3QRESUGZqLwM6dO7FhwwYUFxcDOHmSYerUqVi1alVaRYCIiHKH5vsEVFXNZg4iIjKA5iOB+fPn47e//S0WLVqUvEzplVdewbx587KZj4iIskhzEairq8Mrr7yCbdu2IRgMorS0FDfddBPuu+++bOYjIqIsGrMI/Pe//8Xu3btRV1eHBx54YNgLZF566SV8+umnmDFjRlZDEhFRdox5TuC1117DzJkzU6677rrr8Oqrr2Y8FBER6WPMIvDZZ59h1qxZKddVVlbi0KFDmc5EREQ6GbMIDAwMIBaLpVwXj8eHPRCOiIgmljGLQEVFBT744IOU6z744ANUVFRkPBQREeljzCJw77334vnnn8euXbuQSCQAnHzi365du/D73/8e9957b9ZDEhFRdox5ddCCBQvQ19eHlpYWDA0NwWazIRwOw2q14v7778eCBQv0yElERFmg6T6BhQsX4vbbb8eBAwfQ39+PoqIizJgxA4WFhdnOR0REWaT5ZrHCwsIRrxIiIqKJ6bzfMUxERBMfiwARkcBYBIiIBMYiQEQkMBYBIiKBnd/LgdMUjUaxdu1axGIxxONxzJs3D7W1tejv74fH40F3dzfKy8vR0NCAoqIiPSIRnbdwVEUoEtfcfiiRxTBE46RLEbBYLFi7di0KCgoQi8XQ1NSEWbNm4T//+Q8qKyvhdrvh9Xrh9XpRV1enRySi8xaKxNG6q1Nz+4fmXJrFNETjo8t0kCRJKCgoAHDyoXPxeBySJCEQCMDpdAIAnE4nAoGAHnGIiOgUXY4EgJPPG1q9ejW++eYb3H333Zg+fTpCoRBkWQYAyLKMcDicsq/P54PP5wMANDc3Q1EUvWKPKC8vLydyjCTX8wETN+ORSBAWi0XzNkwmU1rt0+0jSVLa36F3e0mSRu1/PmNkteZDUeS0+oxmou6P495mRrc2CpPJhA0bNuD48ePYuHEjvvjiC819XS4XXC5X8nNPT082Iqbl9HuWc1Wu5wMmbsZoNIahoSHN20gkEmm1T7ePxWJJ+zv0bm+xWEbtfz5jFI1GMrr/TNT9USuHw5Fyue5XB1100UWYOXMm3n//fdjtdgSDQQBAMBiEzWbTOw4RkdB0KQLhcBjHjx8HcPJKoX379qGiogJVVVXw+/0AAL/fj+rqaj3iEBHRKbpMBwWDQbS0tCCRSEBVVcyfPx9z5szBjBkz4PF40NbWBkVR0NjYqEccIiI6RZcicMUVV+Cpp546Z3lxcTGampr0iEBEFxizyYTDx1K/+jYVe74ZNquUxUQTk24nhomIMqk/GseL7V9rbr+8xgGblX/yzsbHRhARCYxFgIhIYCwCREQCYxEgIhIYiwARkcBYBIiIBMYiQEQkMBYBIiKBsQgQEQmMRYCISGAsAkREAmMRICISGIsAEZHAWASIiATGIkBEJDAWASIigbEIEBEJTJfX7PT09KClpQV9fX2QJAkulwv33HMP+vv74fF40N3djfLycjQ0NKCoqEiPSEREBJ2KgNlsxoMPPohp06ZhYGAAa9aswfXXX4+3334blZWVcLvd8Hq98Hq9qKur0yMSERFBp+kgWZYxbdo0AMCkSZNQUVGB3t5eBAIBOJ1OAIDT6UQgENAjDhERnaL7W5e7urpw6NAhXH311QiFQpBlGcDJQhEOh1P28fl88Pl8AIDm5mYoiqJb3pHk5eXlRI6R5Ho+YOJmPBIJwmKxaN6GyWRKq326fSRJSvs79G4vSdKo/bM9RgBgteZDUeQR10/U/XHc28zo1sYwODiITZs2YenSpSgsLNTcz+VyweVyJT/39PRkI15aFEXJiRwjyfV8wMTNGI3GMDQ0pHkbiUQirfbp9rFYLGl/h97tLRbLqP2zPUYAEI1GRt3fJur+qJXD4Ui5XLerg2KxGDZt2oSbb74ZNTU1AAC73Y5gMAgACAaDsNlsesUhIiLoVARUVUVraysqKiqwcOHC5PKqqir4/X4AgN/vR3V1tR5xiIjoFF2mg/bv348dO3bg8ssvx6pVqwAAixcvhtvthsfjQVtbGxRFQWNjox5xiIjoFF2KwLe//W389a9/TbmuqalJjwhERJQC7xgmIhIYiwARkcB0v0+AKNeEoypCkXjKdUciQUSjsWHLhhJ6pCLSB4sACS8UiaN1V2fKdamub39ozqV6xCLSBaeDiIgExiJARCQwFgEiIoHxnADlvNFO3KZizzfDZpWymIgmIrPJhMPHYiOuP/siAFH2IxYBynmjnbhNZXmNAzYrd20arj8ax4vtX4+4/uyLAETZjzgdREQkMBYBIiKBsQgQEQmMRYCISGAsAkREAmMRICISGIsAEZHALvyLYEk4Y90UdDY+FZREpksR2Lp1K/bs2QO73Y5NmzYBAPr7++HxeNDd3Y3y8nI0NDSgqKhIjzh0gRvrpqCz8amgJDJdpoNuvfVW/OIXvxi2zOv1orKyElu2bEFlZSW8Xq8eUYiI6Ay6FIGZM2ee8ys/EAjA6XQCAJxOJwKBgB5RiIjoDIadGA6FQpBlGQAgyzLC4bBRUYiIhDUhTgz7fD74fD4AQHNzMxRFMTgRkJeXlxM5RpLr+QDtGY9EgrBYLJq3azKZMtZekqRz1mVy+5noI0lS1jONt32qcRzP9jOR6WxnZ7Ra86EoclqZsi0b/64NKwJ2ux3BYBCyLCMYDMJms43Y1uVyweVyJT/39PToEXFUiqLkRI6R5Ho+QHvGaDR2ziseR5NIJDLWPtXrJTO5/Uz0sVgsWc803vapxnE8289EprOdnTEajeTcv6Hx/Lt2OBwplxs2HVRVVQW/3w8A8Pv9qK6uNioKEZGwdDkS2Lx5Mz766CMcO3YMy5cvR21tLdxuNzweD9ra2qAoChobG/WIQkREZ9ClCDz66KMplzc1Nenx9URENAI+NoKISGAsAkREAmMRICISGIsAEZHAWASIiATGIkBEJLAJ8dgIurCEoypCkTiORIKIRsd+7j+f90+UPSwCpLtQJI7WXZ1jPkrgND7vnyh7WARoXE7/qk8Hf9kT5Q4WARqX07/q08Ff9kS5gyeGiYgExiMBIqIUzCYTDh8b+8KF0+z5ZtisUhYTZQeLABFRCv3ROF5s/1pz++U1DtisE+9PKqeDiIgENvHKFhFRDpqo00csAkREGTBRp4+MT0BZle51/JMsZgwMaW/Pa/6Jzk+6Rw4AEDX3w5rhHCwCF7h0r+N/aM6laf2a4TX/ROcn3SMHAKi/eSom52c2h+FF4P3338f27duRSCRwxx13wO12Gx2JiEgYhhaBRCKBbdu24Ve/+hXKysrw2GOPoaqqClOmTMnK96U7NZIrJ26IiLLF0CJw8OBBXHLJJZg8eTIA4MYbb0QgEMhaEUh3aiRXTtwQEWWLofcJ9Pb2oqysLPm5rKwMvb29BiYiIhKLpKqqatSXv/vuu/jggw+wfPlyAMCOHTtw8OBBLFu2bFg7n88Hn88HAGhubtY9JxHRhcrQI4GysjIcPXo0+fno0aOQZfmcdi6XC83NzTlVANasWWN0hFHlej6AGTOFGTND1IyGFoGrrroKX3/9Nbq6uhCLxfDvf/8bVVVVRkYiIhKKoWc9zWYzli1bhvXr1yORSOC2227DZZddZmQkIiKhGH7py+zZszF79myjY6TN5XIZHWFUuZ4PYMZMYcbMEDWjoSeGiYjIWHyUNBGRwAyfDsplW7duxZ49e2C327Fp06Zz1nd0dOCpp57CxRdfDACoqanBokWLdMvX09ODlpYW9PX1QZIkuFwu3HPPPcPaqKqK7du347333kN+fj5WrFiBadOm5VRGo8cxGo1i7dq1iMViiMfjmDdvHmpra4e1MXoctWQ0ehyBk08BWLNmDUpLS8+5ksXoMdSSMRfGcOXKlSgoKIDJZILZbD7nqsiMj6NKI+ro6FA/+eQTtbGxMeX6Dz/8UP3Nb36jc6r/19vbq37yySeqqqrqiRMn1Pr6evXw4cPD2rS3t6vr169XE4mEun//fvWxxx7LuYxGj2MikVAHBgZUVVXVoaEh9bHHHlP3798/rI3R46glo9HjqKqq+vrrr6ubN29OmcPoMTxttIy5MIYrVqxQQ6HQiOszPY6cDhrFzJkzUVRUZHSMEcmynPwFMGnSJFRUVJxzx/Xu3btxyy23QJIkzJgxA8ePH0cwGMypjEaTJAkFBQUAgHg8jng8Dkka/swoo8dRS0ajHT16FHv27MEdd9yRcr3RY6gl40SQ6XHkdNA4HThwAKtWrYIsy3jwwQcNu8S1q6sLhw4dwtVXXz1seW9vLxRFSX4+/WiOVDflZdtIGQHjxzGRSGD16tX45ptvcPfdd2P69OnD1ufCOI6VETB2HF944QXU1dVhYGAg5fpcGMOxMgLG74sAsH79egDAnXfeec4VQZkeRxaBcZg6dSq2bt2KgoIC7NmzBxs2bMCWLVt0zzE4OIhNmzZh6dKlKCwsHLZOTXHxlxG/IEfLmAvjaDKZsGHDBhw/fhwbN27EF198gcsvvzy5PhfGcayMRo5je3s77HY7pk2bho6OjpRtjB5DLRlzYV9ct24dSktLEQqF8OSTT8LhcGDmzJnJ9ZkeR04HjUNhYWHyEH327NmIx+MIh8O6ZojFYti0aRNuvvlm1NTUnLO+rKwMPT09yc8jPZojm8bKmAvjeNpFF12EmTNn4v333x+2PBfG8bSRMho5jvv378fu3buxcuVKbN68GR9++OE5fzyNHkMtGXNhXywtLQUA2O12VFdX4+DBg8PWZ3ocWQTGoa+vL1mVDx48iEQigeLiYt2+X1VVtLa2oqKiAgsXLkzZpqqqCjt27ICqqjhw4AAKCwt1/YenJaPR4xgOh3H8+HEAJ6/C2bdvHyoqKoa1MXoctWQ0chyXLFmC1tZWtLS04NFHH8V1112H+vr6YW2MHkMtGY3eFwcHB5NTVYODg9i7d++woz0g8+PI6aBRbN68GR999BGOHTuG5cuXo7a2FrHYyXeC3nXXXdi5cyfefPNNmM1mWK1WPProo7oe3u7fvx87duzA5ZdfjlWrVgEAFi9enPyVcNddd+E73/kO9uzZg/r6elitVqxYsUK3fFozGj2OwWAQLS0tSCQSUFUV8+fPx5w5c/Dmm28mMxo9jloyGj2OqeTSGI4kl8YwFAph48aNAE5eALBgwQLMmjUrq+PIO4aJiATG6SAiIoGxCBARCYxFgIhIYCwCREQCYxEgIhIYiwBRhjz//PP429/+ZnQMorTwElGi8/D222/jn//8J9atW2d0FKJx4ZEAUQrxeNzoCES64JEA0SkrV67EnXfeiX/961/o7OzEfffdh7fffhuhUAhlZWVYvHgx5s6diy+//BKrV69GLBaD1WqF2WzGCy+8gJaWFpSVleEHP/gBOjo68Oyzz+Lee+/F3//+d5hMJixevBi33XYbAODYsWNoaWnBxx9/DIfDgRtuuAEdHR08siDd8bERRGd45513sGbNGthsNrS3t+OJJ55ASUkJdu7ciWeffRZbtmzBlClT8PDDD485HdTX14cTJ06gtbUVe/fuxdNPP43q6moUFRVh27ZtKCgowPPPP4/u7m6sX79+2OOBifTC6SCiM3z3u9+FoiiwWq2YP38+SktLYTKZcOONN+KSSy4554mOozGbzVi0aBHy8vIwe/ZsFBQUoLOzE4lEArt27UJtbS3y8/MxZcoUOJ3OLP5fEY2MRwJEZzjz17jf78cbb7yB7u5uACef6njs2DHN2youLobZbE5+zs/Px+DgIMLhMOLxOMrKypLrzvxvIj2xCBCl0N3djeeeew5NTU2YMWMGTCYTVq1alfKFHumy2Wwwm804evQoHA4HgJPPhCcyAqeDiFKIRCKQJAk2mw0A8NZbb+Hw4cPJ9SUlJejt7U0+WjwdJpMJc+fOxcsvv4xIJIKvvvoKfr8/Y9mJ0sEjAaIUpkyZgoULF+KXv/wlTCYTbrnlFnzrW99Krr/uuuuSJ4hNJhO2bduW1vZ/9KMfoaWlBT/+8Y/hcDhw00034dNPP830/wbRmHiJKFEOeOmll9DX14ef/vSnRkchwXA6iMgAX331FT7//HOoqoqDBw/irbfewty5c42ORQLidBCRAQYGBvDMM88gGAzCbrdj4cKFqK6uNjoWCYjTQUREAuN0EBGRwFgEiIgExiJARCQwFgEiIoGxCBARCYxFgIhIYP8HjIPTcR4EYSIAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.histplot(ratings.groupby('userId')['rating'].mean().sort_values())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- ⭐**Bonus**: Are users who have given more ratings also stricter in their ratings?\n",
    "\n",
    "    - Derive the correlation between the average rating and the number of ratings per user!\n",
    "    - Draw a scatterplot of the user averages and number of ratings\n",
    "    - Only consider users that have rated at least 10 movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "average = pd.DataFrame(ratings.groupby('userId')['rating'].mean().sort_index())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = pd.DataFrame(ratings.groupby('userId')['rating'].count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "average['count'] = count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rating</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rating</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.198997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>-0.198997</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          rating     count\n",
       "rating  1.000000 -0.198997\n",
       "count  -0.198997  1.000000"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "average.corr()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "--- \n",
    "## Movies table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Are there any duplicated titles?\n",
    "\n",
    "Hint: use the `duplicated` method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.duplicated().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Are there any `movieId`s listed in the movies table that are not in the `ratings` table?\n",
    "\n",
    "- Hint: Use the `isin()` method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True    100836\n",
       "Name: movieId, dtype: int64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings['movieId'].isin(movies['movieId']).value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ⭐ Bonus: Extract the year out of the title column\n",
    "\n",
    "- Use the `pandas.Series.str.extract()` method with a regular expression\n",
    "- Hint: check out the [following regular expression](https://regex101.com/r/dJkxR0/1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [movieId, title, genres]\n",
       "Index: []"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.loc[movies['title'].isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9737</th>\n",
       "      <td>193581</td>\n",
       "      <td>Black Butler: Book of the Atlantic (2017)</td>\n",
       "      <td>Action|Animation|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9738</th>\n",
       "      <td>193583</td>\n",
       "      <td>No Game No Life: Zero (2017)</td>\n",
       "      <td>Animation|Comedy|Fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9739</th>\n",
       "      <td>193585</td>\n",
       "      <td>Flint (2017)</td>\n",
       "      <td>Drama</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9740</th>\n",
       "      <td>193587</td>\n",
       "      <td>Bungo Stray Dogs: Dead Apple (2018)</td>\n",
       "      <td>Action|Animation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9741</th>\n",
       "      <td>193609</td>\n",
       "      <td>Andrew Dice Clay: Dice Rules (1991)</td>\n",
       "      <td>Comedy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9742 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      movieId                                      title  \\\n",
       "0           1                           Toy Story (1995)   \n",
       "1           2                             Jumanji (1995)   \n",
       "2           3                    Grumpier Old Men (1995)   \n",
       "3           4                   Waiting to Exhale (1995)   \n",
       "4           5         Father of the Bride Part II (1995)   \n",
       "...       ...                                        ...   \n",
       "9737   193581  Black Butler: Book of the Atlantic (2017)   \n",
       "9738   193583               No Game No Life: Zero (2017)   \n",
       "9739   193585                               Flint (2017)   \n",
       "9740   193587        Bungo Stray Dogs: Dead Apple (2018)   \n",
       "9741   193609        Andrew Dice Clay: Dice Rules (1991)   \n",
       "\n",
       "                                           genres  \n",
       "0     Adventure|Animation|Children|Comedy|Fantasy  \n",
       "1                      Adventure|Children|Fantasy  \n",
       "2                                  Comedy|Romance  \n",
       "3                            Comedy|Drama|Romance  \n",
       "4                                          Comedy  \n",
       "...                                           ...  \n",
       "9737              Action|Animation|Comedy|Fantasy  \n",
       "9738                     Animation|Comedy|Fantasy  \n",
       "9739                                        Drama  \n",
       "9740                             Action|Animation  \n",
       "9741                                       Comedy  \n",
       "\n",
       "[9742 rows x 3 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies['year'] = movies['title'].str.extract('\\(([0-9]{4})\\)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movieId</th>\n",
       "      <th>title</th>\n",
       "      <th>genres</th>\n",
       "      <th>year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>Adventure|Animation|Children|Comedy|Fantasy</td>\n",
       "      <td>1995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Jumanji (1995)</td>\n",
       "      <td>Adventure|Children|Fantasy</td>\n",
       "      <td>1995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Grumpier Old Men (1995)</td>\n",
       "      <td>Comedy|Romance</td>\n",
       "      <td>1995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Waiting to Exhale (1995)</td>\n",
       "      <td>Comedy|Drama|Romance</td>\n",
       "      <td>1995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Father of the Bride Part II (1995)</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>1995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9737</th>\n",
       "      <td>193581</td>\n",
       "      <td>Black Butler: Book of the Atlantic (2017)</td>\n",
       "      <td>Action|Animation|Comedy|Fantasy</td>\n",
       "      <td>2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9738</th>\n",
       "      <td>193583</td>\n",
       "      <td>No Game No Life: Zero (2017)</td>\n",
       "      <td>Animation|Comedy|Fantasy</td>\n",
       "      <td>2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9739</th>\n",
       "      <td>193585</td>\n",
       "      <td>Flint (2017)</td>\n",
       "      <td>Drama</td>\n",
       "      <td>2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9740</th>\n",
       "      <td>193587</td>\n",
       "      <td>Bungo Stray Dogs: Dead Apple (2018)</td>\n",
       "      <td>Action|Animation</td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9741</th>\n",
       "      <td>193609</td>\n",
       "      <td>Andrew Dice Clay: Dice Rules (1991)</td>\n",
       "      <td>Comedy</td>\n",
       "      <td>1991</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9742 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      movieId                                      title  \\\n",
       "0           1                           Toy Story (1995)   \n",
       "1           2                             Jumanji (1995)   \n",
       "2           3                    Grumpier Old Men (1995)   \n",
       "3           4                   Waiting to Exhale (1995)   \n",
       "4           5         Father of the Bride Part II (1995)   \n",
       "...       ...                                        ...   \n",
       "9737   193581  Black Butler: Book of the Atlantic (2017)   \n",
       "9738   193583               No Game No Life: Zero (2017)   \n",
       "9739   193585                               Flint (2017)   \n",
       "9740   193587        Bungo Stray Dogs: Dead Apple (2018)   \n",
       "9741   193609        Andrew Dice Clay: Dice Rules (1991)   \n",
       "\n",
       "                                           genres  year  \n",
       "0     Adventure|Animation|Children|Comedy|Fantasy  1995  \n",
       "1                      Adventure|Children|Fantasy  1995  \n",
       "2                                  Comedy|Romance  1995  \n",
       "3                            Comedy|Drama|Romance  1995  \n",
       "4                                          Comedy  1995  \n",
       "...                                           ...   ...  \n",
       "9737              Action|Animation|Comedy|Fantasy  2017  \n",
       "9738                     Animation|Comedy|Fantasy  2017  \n",
       "9739                                        Drama  2017  \n",
       "9740                             Action|Animation  2018  \n",
       "9741                                       Comedy  1991  \n",
       "\n",
       "[9742 rows x 4 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "titles = movies['title'].str.partition(' (', True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies['title'] = titles[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "95441"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies[movies['title'].str.fullmatch('Ted')]['movieId'].values[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies.loc[movies['title'].isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "movie = movies[movies['title'].str.contains('Up')]['title'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Once Upon a Time... When We Were Colored',\n",
       "       'Up Close and Personal', 'Air Up There, The',\n",
       "       'Englishman Who Went Up a Hill But Came Down a Mountain, The',\n",
       "       'Denise Calls Up', 'Bringing Up Baby', 'Tie Me Up! Tie Me Down!',\n",
       "       \"Cheech and Chong's Up in Smoke\", 'Once Upon a Time in the West',\n",
       "       'Once Upon a Time in America', 'Pump Up the Volume',\n",
       "       'Honey, I Blew Up the Kid', '52 Pick-Up', '42 Up', 'Hanging Up',\n",
       "       'Up at the Villa', 'Blow-Up', '3 Ninjas Knuckle Up',\n",
       "       'Dancer Upstairs, The', \"What's Up, Doc?\",\n",
       "       \"What's Up, Tiger Lily?\", 'Uptown Girls',\n",
       "       'Once Upon a Time in the Midlands', 'Once Upon a Time in Mexico',\n",
       "       'Once Upon a Time in China', 'Once Upon a Time in China II',\n",
       "       'Once Upon a Time in China III',\n",
       "       'Lagaan: Once Upon a Time in India', '28 Up', '35 Up',\n",
       "       'Upside of Anger, The', 'Break-Up, The', 'Step Up',\n",
       "       'Shut Up & Sing', '49 Up', 'Seven Up!', 'Wake Up, Ron Burgundy',\n",
       "       '21 Up', 'Knocked Up', \"Surf's Up\", \"Shoot 'Em Up\",\n",
       "       'Step Up 2 the Streets', 'Up', 'Fired Up', 'Up in the Air',\n",
       "       \"Mickey's Once Upon a Christmas\", 'Grown Ups',\n",
       "       'Louis C.K.: Chewed Up', 'Upside Down: The Creation Records Story',\n",
       "       'Change-Up, The', 'From Up on Poppy Hill', 'Upstream Color',\n",
       "       'Upside Down', 'Stand Up Guys', 'Grown Ups 2', 'Get on Up',\n",
       "       'Digging Up the Marrow', 'Man Up', 'Keeping Up with the Joneses',\n",
       "       'Rory Scovel Tries Stand-Up for the First Time',\n",
       "       'Pacific Rim: Uprising'], dtype=object)"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies[movies['title'].str.match(\"Grumpy Old Men\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies[movies['title'] == 'Grumpier Old Men']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ⭐ Bonus: Split the genre column into a separate genre table\n",
    "\n",
    "1. Use the `pandas.Series.str.split` method\n",
    "2. Then use the `pandas.Series.explode` method to clean up the list entries\n",
    "3. Which `movieId` has the most genres assigned?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "genres = movies['genres'].str.split('|').explode()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "genres = pd.DataFrame(genres)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "genres['movieId'] = movies['movieId']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "genres.set_index('movieId', inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "genres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "genres.groupby(genres.index).count().sort_values('genres')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies.loc[movies['title'] == \"Rubber\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Fuzzy String Search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Search for \"star wars\" movies in the items table\n",
    "\n",
    "- install the library [thefuzz](https://github.com/seatgeek/thefuzz)\n",
    "- search for the query \"star wars\" in the list of movie titles\n",
    "    - set the `movieId` as index for the dataframe\n",
    "    - extract the `titles` column and use it as input to `extractBests()`\n",
    "- look at the following code snippet and research:\n",
    "    - what is the third entry in the tuples returned?\n",
    "    - what is the second entry in the tuples returned?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "movie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies2 = movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies2.set_index(\"movieId\", drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies2 = movies['title']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Riding in Cars with Boys', 100, 3556),\n",
       " ('Cars', 100, 6212),\n",
       " ('Cars 2', 100, 7639),\n",
       " ('Cars 3', 100, 9509),\n",
       " ('Icarus', 80, 9609)]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from thefuzz import fuzz, process\n",
    "\n",
    "search_query = \"Cars\"\n",
    "match = process.extractBests(\n",
    "    search_query, movies['title'], scorer=fuzz.token_set_ratio\n",
    ")\n",
    "match"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Ratings and movies table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Merge/join the ratings and movies tables together\n",
    "\n",
    "- use the `movieId` as common key\n",
    "- hint: `df.merge(...)`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = ratings.merge(movies, on='movieId')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "#### Count the number of ratings per movie title!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby('movieId')['rating'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- What are the top 10 most rated movies in the dataset?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby('title')['rating'].count().sort_values(ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "--- \n",
    "##### Which movies are most polarizing?\n",
    "\n",
    "- For each movie, calculate the standard deviation divided by the average rating (This is called the [coefficient of variation](https://datacollection.jrc.ec.europa.eu/wordef/coefficient-of-variation))\n",
    "- Look at the top 10 movies that have the highest coefficient of variation!\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby('title')['rating'].std().sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "#### Calculate the average rating per movie title!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby('title')['rating'].mean().sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Make a visualization of the distribution of average ratings per movie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby('title')['rating'].mean().sort_values(ascending=False).plot(kind='hist', figsize=(15,10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Which movie was rated the best? Which movie was rated the worst?\n",
    "    - ⭐**Bonus**: Filter out movies that have been rated by less than 30 users. Do your results change? Why?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print (df.groupby('title')['rating'].mean().sort_values(ascending=False).head(1))\n",
    "print (df.groupby('title')['rating'].mean().sort_values(ascending=False).tail(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df.groupby('title')['rating'].mean().sort_values(ascending=False).loc[df.groupby('title')['rating'].count() > 30].head(1))\n",
    "print(df.groupby('title')['rating'].mean().sort_values(ascending=False).loc[df.groupby('title')['rating'].count() > 30].tail(1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- ⭐**Bonus**: Did the average rating of movies improve over time? \n",
    "\n",
    "    - Use the `year` that you have extracted in a previous ⭐bonus exercises\n",
    "    - Visualize your findings in a nice plot\n",
    "    - Post your plot in slack!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.groupby('year')['rating'].mean().plot(kind='line', figsize=(18,13), color=\"blue\")\n",
    "plt.title('\\nAverage rating of movies over time\\n', fontsize=32)\n",
    "line1 = plt.axvline(65, 0.98,0.02, color='red', linestyle=\"--\", alpha=0.5)\n",
    "plt.xlabel('\\nYear', fontsize=20)\n",
    "plt.ylabel('Average Rating\\n', fontsize=20)\n",
    "plt.savefig('avg_rating_over_time.jpg');"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.7 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "vscode": {
   "interpreter": {
    "hash": "abc7d9422c788b1f325554a74823655484f1a6b478c1481f1f9705007e56709b"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}