2. notebook_pipelines_mlflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b78329da",
   "metadata": {},
   "source": [
    "# Factored Workshops! 🚀\n",
    "_Crearás un modelo capaz de predecir la duración total de un viaje de taxi en Nueva York 🗽_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05b2c90a",
   "metadata": {},
   "source": [
    "# Pipelines en scikit-learn"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a66880dc",
   "metadata": {},
   "source": [
    "El objetivo de esta sección es mostrar cómo se pueden desarrollar flujos de preprocesamiento en sklearn. Vamos a explorar las diferentes operaciones que se le pueden hacer a los diferentes tipos de variables y cómo agrupar todas las operaciones en un solo elemento de sklearn que tenga los métodos .fit() y .transform()."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "979d561f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from typing import List\n",
    "from sklearn import set_config\n",
    "set_config(display='diagram')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9c3774fd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1458644, 13)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\n",
    "    \"https://factored-workshops.s3.amazonaws.com/taxi-trip-duration.csv\"\n",
    ")\n",
    "\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a49e862a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1447855, 13)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Limitar rango de datos\n",
    "tiempo_minimo = 60 # 1 minuto\n",
    "tiempo_maximo = 36000 # 10 horas\n",
    "\n",
    "data = data[\n",
    "    (data[\"trip_duration\"] > tiempo_minimo) &\n",
    "    (data[\"trip_duration\"] < tiempo_maximo)\n",
    "]\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9119c394",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>vendor_id</th>\n",
       "      <th>pickup_datetime</th>\n",
       "      <th>dropoff_datetime</th>\n",
       "      <th>passenger_count</th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "      <th>dropoff_longitude</th>\n",
       "      <th>dropoff_latitude</th>\n",
       "      <th>store_and_fwd_flag</th>\n",
       "      <th>trip_duration</th>\n",
       "      <th>pickup_borough</th>\n",
       "      <th>dropoff_borough</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id2875421</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-03-14 17:24:55</td>\n",
       "      <td>2016-03-14 17:32:30</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.982155</td>\n",
       "      <td>40.767937</td>\n",
       "      <td>-73.964630</td>\n",
       "      <td>40.765602</td>\n",
       "      <td>N</td>\n",
       "      <td>455</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>Manhattan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id2377394</td>\n",
       "      <td>1</td>\n",
       "      <td>2016-06-12 00:43:35</td>\n",
       "      <td>2016-06-12 00:54:38</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.980415</td>\n",
       "      <td>40.738564</td>\n",
       "      <td>-73.999481</td>\n",
       "      <td>40.731152</td>\n",
       "      <td>N</td>\n",
       "      <td>663</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>Brooklyn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id3858529</td>\n",
       "      <td>2</td>\n",
       "      <td>2016-01-19 11:35:24</td>\n",
       "      <td>2016-01-19 12:10:48</td>\n",
       "      <td>1</td>\n",
       "      <td>-73.979027</td>\n",
       "      <td>40.763939</td>\n",
       "      <td>-74.005333</td>\n",
       "      <td>40.710087</td>\n",
       "      <td>N</td>\n",
       "      <td>2124</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>Brooklyn</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  vendor_id      pickup_datetime     dropoff_datetime  \\\n",
       "0  id2875421          2  2016-03-14 17:24:55  2016-03-14 17:32:30   \n",
       "1  id2377394          1  2016-06-12 00:43:35  2016-06-12 00:54:38   \n",
       "2  id3858529          2  2016-01-19 11:35:24  2016-01-19 12:10:48   \n",
       "\n",
       "   passenger_count  pickup_longitude  pickup_latitude  dropoff_longitude  \\\n",
       "0                1        -73.982155        40.767937         -73.964630   \n",
       "1                1        -73.980415        40.738564         -73.999481   \n",
       "2                1        -73.979027        40.763939         -74.005333   \n",
       "\n",
       "   dropoff_latitude store_and_fwd_flag  trip_duration pickup_borough  \\\n",
       "0         40.765602                  N            455      Manhattan   \n",
       "1         40.731152                  N            663      Manhattan   \n",
       "2         40.710087                  N           2124      Manhattan   \n",
       "\n",
       "  dropoff_borough  \n",
       "0       Manhattan  \n",
       "1        Brooklyn  \n",
       "2        Brooklyn  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ceb1003",
   "metadata": {},
   "source": [
    "Es importante siempre separar la variable dependiente—en este caso **trip_duration** del dataframe que vamos a usar para crear las variables independientes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3ee3a2bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "y = data[\"trip_duration\"] # predecir\n",
    "x = data.drop(\n",
    "    [\"id\", \"trip_duration\", \"dropoff_datetime\", \"store_and_fwd_flag\"],\n",
    "    axis=\"columns\"\n",
    ")\n",
    "\n",
    "# Quitamos:\n",
    "# id -> Es un identificador unico y este no brinda nada de informacion\n",
    "# trip_duration -> Variable a predecir\n",
    "# dropoff_datetime -> Se elimina pq para nuevos datos no se tendra este dato\n",
    "# store_and_fwd_flag -> Variable bandera de si almacenar o no el viaje "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9a9f550",
   "metadata": {},
   "source": [
    "**store_and_fwd_flag**\n",
    "\n",
    "Esta bandera indica si el registro de viaje se mantuvo en la memoria del vehículo antes de enviarlo \n",
    "al proveedor porque el vehículo no tenía conexión con el servidor\n",
    "- Y= almacenar y enviar; \n",
    "- N= no es un viaje de almacenamiento y reenvío"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8a674588",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "N    1439947\n",
       "Y       7908\n",
       "Name: store_and_fwd_flag, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['store_and_fwd_flag'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48ae0b40",
   "metadata": {},
   "source": [
    "## División de los datos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "28dfa472",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(x, \n",
    "                                                    y,\n",
    "                                                    random_state=0)\n",
    "# test_size=0.25 -- default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "27dc4340",
   "metadata": {},
   "outputs": [],
   "source": [
    "# help(train_test_split)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "616a2697",
   "metadata": {},
   "source": [
    "## Transformers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3effaffb",
   "metadata": {},
   "source": [
    "`scikit-learn` incluye una gran [lista](https://scikit-learn.org/stable/data_transforms.html) de transformers que nos **permiten limpiar y transformar datos** dependiendo de nuestro objetivo. Sin importar el tipo de transformacion que apliquen, todos los transformers siguen la convención de tener por lo menos dos métodos:\n",
    "\n",
    "- `.fit()` calcula los parámetros necesarios para realizar la transformación a partir de unos datos de entrada. Este método se ejecuta **únicamente en los datos de entrenamiento** para asegurarnos que los parámetros no contienen información de los datos de validación.\n",
    "- `.transform()` aplica la transformación a los datos.\n",
    "\n",
    "Veamos un ejemplo usando `StandardScaler`, un transformer que nos permite **remover la media y escalar** los datos para que **tengan varianza de 1**. Vamos a usarlo para normalizar los coordenadas de inicio del viaje."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d4b9ca43",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.30923835 -0.27478234]\n",
      " [-0.13589444  0.75675283]\n",
      " [ 0.24599071  0.58169127]\n",
      " ...\n",
      " [-0.19727957 -0.13002453]\n",
      " [-0.03955223 -0.08161096]\n",
      " [-0.11150857 -0.79791458]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "transformer = StandardScaler()\n",
    "transformer.fit(\n",
    "    x_train[[\"pickup_longitude\", \"pickup_latitude\"]]\n",
    ")\n",
    "normed_array = transformer.transform(\n",
    "    x_test[[\"pickup_longitude\", \"pickup_latitude\"]]\n",
    ")\n",
    "print(normed_array)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e01b428a",
   "metadata": {},
   "source": [
    "## Custom Transformers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "471c2606",
   "metadata": {},
   "source": [
    "A pesar de que scikit-learn ofrece varias operaciones para transformar datos, frecuentemente necesitamos crear transformaciones que son específicas a nuestro proyecto. Para esto vamos a aprender cómo implementar custom transformers.\n",
    "\n",
    "Todos los transformer custom deben heredar `BaseEstimator` y `TransformerMixin` para que tengan todas las funciones necesarias para conectarse a otros objetos de sklearn. Por convención de sklearn, los objetos usados para transformar datos siempre deben tener los métodos `.fit()` y `.transform()`. Ambos métodos reciben X y y para que se integren sin problemas a pipelines de sklearn.\n",
    "\n",
    "- El método `.fit()` sirve para almacenar cantidades que vamos a usar durante la transformación de los datos y siempre debe retornar self. \n",
    "- El método `.transform()` ejecuta la transformación y retorna los datos transformados.\n",
    "\n",
    "Ahora vamos a replicar el StandardScaler pero esta vez escribiéndolo como un transformer personalizado que no retorne un array sino un DataFrame."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2e6f4af8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class PrimerTransformer(BaseEstimator, TransformerMixin):\n",
    "    def fit(self, X, y=None):\n",
    "        self.mean = X.mean()\n",
    "        self.std = X.std()\n",
    "        return self\n",
    "\n",
    "    def transform(self, X, y=None):\n",
    "        return (X - self.mean) / self.std"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fee3f730",
   "metadata": {},
   "source": [
    "## Checkpoint #1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "dbe2f5ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pickup_longitude</th>\n",
       "      <th>pickup_latitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5949</th>\n",
       "      <td>-0.309238</td>\n",
       "      <td>-0.274782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255492</th>\n",
       "      <td>-0.135894</td>\n",
       "      <td>0.756752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>979171</th>\n",
       "      <td>0.245991</td>\n",
       "      <td>0.581691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121356</th>\n",
       "      <td>0.176797</td>\n",
       "      <td>0.385864</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524382</th>\n",
       "      <td>0.194096</td>\n",
       "      <td>0.182913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>885062</th>\n",
       "      <td>-0.488949</td>\n",
       "      <td>-0.292409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>755237</th>\n",
       "      <td>-0.565470</td>\n",
       "      <td>-1.408094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>790367</th>\n",
       "      <td>-0.197279</td>\n",
       "      <td>-0.130024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106875</th>\n",
       "      <td>-0.039552</td>\n",
       "      <td>-0.081611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260937</th>\n",
       "      <td>-0.111509</td>\n",
       "      <td>-0.797914</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>361964 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        pickup_longitude  pickup_latitude\n",
       "5949           -0.309238        -0.274782\n",
       "255492         -0.135894         0.756752\n",
       "979171          0.245991         0.581691\n",
       "121356          0.176797         0.385864\n",
       "524382          0.194096         0.182913\n",
       "...                  ...              ...\n",
       "885062         -0.488949        -0.292409\n",
       "755237         -0.565470        -1.408094\n",
       "790367         -0.197279        -0.130024\n",
       "106875         -0.039552        -0.081611\n",
       "260937         -0.111509        -0.797914\n",
       "\n",
       "[361964 rows x 2 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pt = PrimerTransformer()\n",
    "\n",
    "pt.fit(x_train[[\"pickup_longitude\", \"pickup_latitude\"]])\n",
    "pt.transform(x_test[[\"pickup_longitude\", \"pickup_latitude\"]])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9befa68",
   "metadata": {},
   "source": [
    "En este [link](https://scikit-learn.org/stable/developers/develop.html) pueden encontrar más detalles de qué papel juegan los métodos __init__, .fit() y .transform()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11b5b992",
   "metadata": {},
   "source": [
    "## Transformer Fechas"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c4a113f",
   "metadata": {},
   "source": [
    "Ahora que sabemos cómo construir objetos para transformar datos, vamos a **crear un transformer para crear variables como el día de la semana y la hora** del momento en el que empieza el servicio.\n",
    "Como vimos la semana pasada en nuestro EDA, esa información puede ser relevante para determinar la duración del viaje.\n",
    "\n",
    "En nuestra transformación no debemos almacenar ningún dato para hacer las transformación entonces dejamos el método .fit() vacío. En .transform() extraemos los datos de fecha que nos interesan y retornamos un dataframe con las nuevas variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "14ff982a",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TransformerFechas(BaseEstimator, TransformerMixin):\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "\n",
    "    def transform(self, X, y=None):\n",
    "        columna_fecha = pd.to_datetime(X[\"pickup_datetime\"])\n",
    "        fecha_df = pd.DataFrame()\n",
    "        # TODO: Crear columnas con dia de la semana y hora de recogida.\n",
    "        fecha_df['weekday'] = columna_fecha.dt.weekday\n",
    "        fecha_df['hour'] = columna_fecha.dt.hour\n",
    "        return fecha_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ee6b86d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['vendor_id', 'pickup_datetime', 'passenger_count', 'pickup_longitude',\n",
       "       'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude',\n",
       "       'pickup_borough', 'dropoff_borough'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6757a2c3",
   "metadata": {},
   "source": [
    "## Checkpoint #2.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "09185b80",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>weekday</th>\n",
       "      <th>hour</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>518949</th>\n",
       "      <td>3</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1128931</th>\n",
       "      <td>6</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>574396</th>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54790</th>\n",
       "      <td>6</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599130</th>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         weekday  hour\n",
       "518949         3    21\n",
       "1128931        6    21\n",
       "574396         1    18\n",
       "54790          6    17\n",
       "599130         0    16"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transformer_fechas = TransformerFechas()\n",
    "fechas_df = transformer_fechas.fit_transform(x_train)\n",
    "fechas_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "25e5d039",
   "metadata": {},
   "outputs": [],
   "source": [
    "# x_train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "05101b57",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "518949     2016-06-02 21:41:39\n",
       "1128931    2016-03-27 21:59:43\n",
       "574396     2016-02-02 18:13:10\n",
       "Name: pickup_datetime, dtype: object"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train[\"pickup_datetime\"].head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d27915e3",
   "metadata": {},
   "source": [
    "## Transformer Distancia"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1f953f8",
   "metadata": {},
   "source": [
    "También queremos crear una feature que nos ayude a medir la distancia entre el punto de origen y el punto de destino usando la longitud y la latitud en los datos. \n",
    "Nuevamente no tenemos que almacenar cantidades en nuestro método .fit() y calculamos la distancia entre los dos puntos usando la [distancia de Haversine](https://es.wikipedia.org/wiki/F%C3%B3rmula_del_semiverseno). \n",
    "\n",
    "En el código ya está implementada la función para calcular la distancia y no es necesario fijarse en los detalles de la implementación. Como pueden ver, nuestros transformers pueden incluir funciones adicionales que nos ayuden a calcular cantidades útiles para realizar la transformación."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "de5de6e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TransformerDistancia(BaseEstimator, TransformerMixin):\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "\n",
    "    def transform(self, X, y=None):\n",
    "        X_init = X[[\"pickup_latitude\", \"pickup_longitude\"]].to_numpy()\n",
    "        X_final = X[[\"dropoff_latitude\", \"dropoff_longitude\"]].to_numpy()\n",
    "\n",
    "        # Distancia de Haversine\n",
    "        # TODO: Calcular la variable distancia usando la funcion\n",
    "        # distancia de Haversine.\n",
    "        distancia = self.distancia_haversine(X_init=X_init,X_final=X_final)\n",
    "        distancia_df = pd.DataFrame()\n",
    "        distancia_df[\"distancia\"] = distancia\n",
    "        return distancia_df\n",
    "    \n",
    "    # DISTANCIA GEOESPACIALES - LATITUD/LONGITUD\n",
    "    def distancia_haversine(self, X_init, X_final):\n",
    "        # Convertir de decimal a radianes\n",
    "        X_init = np.radians(X_init)\n",
    "        X_final = np.radians(X_final)\n",
    "\n",
    "        # Formula Haversine\n",
    "        dlat = X_final[:, 0] - X_init[:, 0] \n",
    "        dlon = X_final[:, 1] - X_init[:, 1]\n",
    "        a = np.sin(dlat / 2) ** 2 + np.cos(X_init[:, 0]) * (\n",
    "            np.cos(X_final[:, 0])) * (\n",
    "            np.sin(dlon / 2) ** 2)\n",
    "        c = 2 * np.arcsin(np.sqrt(a))\n",
    "        r = 6371 \n",
    "        # Radius of earth in kilometers. Use 3956 for miles. \n",
    "        # Determines return value units.\n",
    "        return c * r"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3335452",
   "metadata": {},
   "source": [
    "## Checkpoint #2.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "54b982bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>distancia</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.404355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.390267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.629826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.298386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7.488963</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   distancia\n",
       "0   2.404355\n",
       "1   0.390267\n",
       "2   5.629826\n",
       "3   4.298386\n",
       "4   7.488963"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transformer_dist = TransformerDistancia()\n",
    "distancias_df = transformer_dist.fit_transform(x_train)\n",
    "distancias_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9223ea5b",
   "metadata": {},
   "source": [
    "## Unión de transformers con Pipelines"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efc31206",
   "metadata": {},
   "source": [
    "## **Pipeline Numérico**\n",
    "\n",
    "Ahora vamos a usar los objetos de `Pipeline` y `ColumnTransformer` para unir los transformers que ya hemos creado con otros disponibles en sklearn y lograr todas las transformaciones que queremos realizar a todos los datos."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "669b5419",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6e46102",
   "metadata": {},
   "source": [
    "`ColumnTransformer` nos permite **elegir** las **** sobre las que queremos **aplicar una transformación** cuando nos llegan columnas adicionales. En este caso queremos que al TransformerDistancia llegue únicamente pickup_longitude, pickup_latitude, dropoff_longitude y dropoff_latitude ."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e4c2bbd",
   "metadata": {},
   "source": [
    "Leyendo la [documentación](https://scikit-learn.org/stable/modules/generated/sklearn.compose.ColumnTransformer.html#sklearn.compose.ColumnTransformer) sabemos que **debemos pasar una tupla** con el _nombre del transformer_, la _clase_ que define el transformer y las _columnas_ sobre las que queremos _aplicar la transformación_. \n",
    "\n",
    "`ColumnTransformer` también nos permite definir qué se debe hacer con las **columnas** que **no** estamos **transformando**; en este caso elegimos **pasarlas sin transformarlas** `remainder=\"passthrough\"`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "aecbdc48",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-25ebe402-d045-4582-8b30-4aa746a33a82 {color: black;background-color: white;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 pre{padding: 0;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-toggleable {background-color: white;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.2em 0.3em;box-sizing: border-box;text-align: center;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;margin: 0.25em 0.25em;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-estimator:hover {background-color: #d4ebff;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 2em;bottom: 0;left: 50%;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-item {z-index: 1;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-parallel-item {display: flex;flex-direction: column;position: relative;background-color: white;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-parallel-item:only-child::after {width: 0;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0.2em;box-sizing: border-box;padding-bottom: 0.1em;background-color: white;position: relative;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-label label {font-family: monospace;font-weight: bold;background-color: white;display: inline-block;line-height: 1.2em;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-label-container {position: relative;z-index: 2;text-align: center;}#sk-25ebe402-d045-4582-8b30-4aa746a33a82 div.sk-container {display: inline-block;position: relative;}</style><div id=\"sk-25ebe402-d045-4582-8b30-4aa746a33a82\" class\"sk-top-container\"><div class=\"sk-container\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"182c514d-db47-4bf8-9caf-1b71cc94f483\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"182c514d-db47-4bf8-9caf-1b71cc94f483\">ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(remainder='passthrough',\n",
       "                  transformers=[('transformer_dist', TransformerDistancia(),\n",
       "                                 ['pickup_longitude', 'pickup_latitude',\n",
       "                                  'dropoff_longitude', 'dropoff_latitude'])])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"a4540252-d81f-4df7-9ddd-28a980b0b6d5\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"a4540252-d81f-4df7-9ddd-28a980b0b6d5\">transformer_dist</label><div class=\"sk-toggleable__content\"><pre>['pickup_longitude', 'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude']</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"e6742731-c720-4803-8911-da536086b298\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"e6742731-c720-4803-8911-da536086b298\">TransformerDistancia</label><div class=\"sk-toggleable__content\"><pre>TransformerDistancia()</pre></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"5ef30ead-e21b-45e0-9e3a-50fe04b3c90f\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"5ef30ead-e21b-45e0-9e3a-50fe04b3c90f\">remainder</label><div class=\"sk-toggleable__content\"><pre></pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"4369f491-c802-4fb0-bfc8-5719b2aeb30a\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"4369f491-c802-4fb0-bfc8-5719b2aeb30a\">passthrough</label><div class=\"sk-toggleable__content\"><pre>passthrough</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "ColumnTransformer(remainder='passthrough',\n",
       "                  transformers=[('transformer_dist', TransformerDistancia(),\n",
       "                                 ['pickup_longitude', 'pickup_latitude',\n",
       "                                  'dropoff_longitude', 'dropoff_latitude'])])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "coord_cols = [\n",
    "    \"pickup_longitude\",\n",
    "    \"pickup_latitude\",\n",
    "    \"dropoff_longitude\",\n",
    "    \"dropoff_latitude\"\n",
    "]\n",
    "\n",
    "transformer_coord = ColumnTransformer(\n",
    "    [\n",
    "        (\"transformer_dist\", TransformerDistancia(), coord_cols),\n",
    "    ],\n",
    "    remainder=\"passthrough\" # Las demas variables no sean transformadas\n",
    ")\n",
    "\n",
    "display(transformer_coord)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0a40185",
   "metadata": {},
   "source": [
    "Ahora que tenemos nuestro transformer que realiza transformación usando las coordenadas y deja pasar columnas adicionales, vamos a usarlo para unirlo con la otra variable númerica que tenemos en el dataset `—passenger_count—` y normalizarlas usando `StandardScaler`. Para esto vamos a usar el objeto `Pipeline`."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "910ac6a9",
   "metadata": {},
   "source": [
    "`Pipeline` [(documentación Pipeline)](https://scikit-learn.org/stable/modules/generated/sklearn.pipeline.Pipeline.html#sklearn.pipeline.Pipeline) nos permite concatenar transformaciones de sklearn. En este caso vamos a concatenar el ColumnTransformer que creamos con TransformerDistancia con el StandardScaler. Como passenger_count no estaba entre columnas que selecciona transformer_coord, esa columna pasa directamente a ser normalizada por el StandardScaler."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ccb542ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.25154928 -0.50616698]\n",
      " [-0.73141214  0.25385023]\n",
      " [ 0.51692963 -0.50616698]\n",
      " ...\n",
      " [-0.41058443 -0.50616698]\n",
      " [-0.3461049  -0.50616698]\n",
      " [ 1.33898968  0.25385023]]\n"
     ]
    }
   ],
   "source": [
    "num_cols = [\"passenger_count\"] + coord_cols\n",
    "\n",
    "num_pipeline = Pipeline(\n",
    "    [\n",
    "        (\"transformer_coord\", transformer_coord), # distancia / los 2 puntos\n",
    "        (\"scaler\", StandardScaler())\n",
    "    ]\n",
    ")\n",
    "\n",
    "X_num = num_pipeline.fit_transform(x_train[num_cols], y_train)\n",
    "print(X_num)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc9c7009",
   "metadata": {},
   "source": [
    "Usando la función `display` podemos ver el diagrama del flujo que acabamos de crear. Por `TransformerDistancia` pasan las variables de longitud y latitud y por `passthrough` pasa cualquier columna que le pasemos al ColumnTransformer que no sean longitud o latitud—en este caso passenger_count. El último paso es aplicar un `StandardScaler` a la variable de distancia que sale de TransformerDistancia y a las variables que vienen de passthrough."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f89401a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d {color: black;background-color: white;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d pre{padding: 0;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-toggleable {background-color: white;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.2em 0.3em;box-sizing: border-box;text-align: center;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-estimator {font-family: monospace;background-color: #f0f8ff;margin: 0.25em 0.25em;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-estimator:hover {background-color: #d4ebff;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 2em;bottom: 0;left: 50%;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-item {z-index: 1;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-parallel-item {display: flex;flex-direction: column;position: relative;background-color: white;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-parallel-item:only-child::after {width: 0;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0.2em;box-sizing: border-box;padding-bottom: 0.1em;background-color: white;position: relative;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-label label {font-family: monospace;font-weight: bold;background-color: white;display: inline-block;line-height: 1.2em;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-label-container {position: relative;z-index: 2;text-align: center;}#sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d div.sk-container {display: inline-block;position: relative;}</style><div id=\"sk-94acc9ef-7adb-4724-b369-0b4b10b6d11d\" class\"sk-top-container\"><div class=\"sk-container\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"4678fd90-35c2-460e-8865-773d548f334d\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"4678fd90-35c2-460e-8865-773d548f334d\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[('transformer_coord',\n",
       "                 ColumnTransformer(remainder='passthrough',\n",
       "                                   transformers=[('transformer_dist',\n",
       "                                                  TransformerDistancia(),\n",
       "                                                  ['pickup_longitude',\n",
       "                                                   'pickup_latitude',\n",
       "                                                   'dropoff_longitude',\n",
       "                                                   'dropoff_latitude'])])),\n",
       "                ('scaler', StandardScaler())])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"897a4936-b079-4405-aa83-882d99e1a308\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"897a4936-b079-4405-aa83-882d99e1a308\">transformer_coord: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(remainder='passthrough',\n",
       "                  transformers=[('transformer_dist', TransformerDistancia(),\n",
       "                                 ['pickup_longitude', 'pickup_latitude',\n",
       "                                  'dropoff_longitude', 'dropoff_latitude'])])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"68c02c74-7310-49f9-96c9-dca9af29b097\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"68c02c74-7310-49f9-96c9-dca9af29b097\">transformer_dist</label><div class=\"sk-toggleable__content\"><pre>['pickup_longitude', 'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude']</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"4834b207-d768-45ca-8eb3-90f33dd6f935\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"4834b207-d768-45ca-8eb3-90f33dd6f935\">TransformerDistancia</label><div class=\"sk-toggleable__content\"><pre>TransformerDistancia()</pre></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"ed04c630-9660-44b2-a08b-f0ea2036407d\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"ed04c630-9660-44b2-a08b-f0ea2036407d\">remainder</label><div class=\"sk-toggleable__content\"><pre>['passenger_count']</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"70912e17-454c-47c1-8bae-c6d91ea996fe\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"70912e17-454c-47c1-8bae-c6d91ea996fe\">passthrough</label><div class=\"sk-toggleable__content\"><pre>passthrough</pre></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"9b894c43-4639-4cad-9727-0a76907ee1a1\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"9b894c43-4639-4cad-9727-0a76907ee1a1\">StandardScaler</label><div class=\"sk-toggleable__content\"><pre>StandardScaler()</pre></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "Pipeline(steps=[('transformer_coord',\n",
       "                 ColumnTransformer(remainder='passthrough',\n",
       "                                   transformers=[('transformer_dist',\n",
       "                                                  TransformerDistancia(),\n",
       "                                                  ['pickup_longitude',\n",
       "                                                   'pickup_latitude',\n",
       "                                                   'dropoff_longitude',\n",
       "                                                   'dropoff_latitude'])])),\n",
       "                ('scaler', StandardScaler())])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(num_pipeline)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b2cf57d",
   "metadata": {},
   "source": [
    "## **Pipeline Categórico**\n",
    "Usaremos una lógica similar para las variables categóricas pero esta vez para concatenar el resultado de extraer variables temporales de la fecha de inicio del viaje con un `OneHotEncoder` para las variables categóricas de los datos."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2b701407",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 3. 21.  0.  1.]\n",
      " [ 6. 21.  1.  2.]\n",
      " [ 1. 18.  1.  2.]\n",
      " ...\n",
      " [ 6.  2.  0.  3.]\n",
      " [ 1. 23.  1.  1.]\n",
      " [ 1.  2.  1.  2.]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import OrdinalEncoder\n",
    "\n",
    "cat_cols = [\"vendor_id\", \"pickup_borough\", \"pickup_datetime\"]\n",
    "\n",
    "transformer_fechas = ColumnTransformer(\n",
    "    [\n",
    "        #TODO: Punto 1 de Checkpoint 3\n",
    "        (\"transformer_fechas\",TransformerFechas(),['pickup_datetime'])\n",
    "    ],\n",
    "    remainder=\"passthrough\"\n",
    ")\n",
    "\n",
    "cat_pipeline = Pipeline(\n",
    "    [\n",
    "        #TODO: Punto 2 de Checkpoint 3\n",
    "        (\"transformer_fechas\",transformer_fechas),\n",
    "        (\"OrdinalEncoder\",OrdinalEncoder()) # convertir de letras a numeros\n",
    "    ]\n",
    ")\n",
    "\n",
    "# OrdinalEncoder => crea solo 1 columna con asignacion || Mas recomendable\n",
    "# OneHotEncoder => Crea columnas dependiendo de valores de columna || muchos valores de columnas(k-1)\n",
    "\n",
    "# Cuando hay categorias en orden es recomendable el OrdinalEncoder\n",
    "\n",
    "X_cat = cat_pipeline.fit_transform(x_train[cat_cols])\n",
    "print(X_cat)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb26e4af",
   "metadata": {},
   "source": [
    "## Checkpoint #3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a0263b9c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 {color: black;background-color: white;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 pre{padding: 0;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-toggleable {background-color: white;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.2em 0.3em;box-sizing: border-box;text-align: center;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;margin: 0.25em 0.25em;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-estimator:hover {background-color: #d4ebff;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 2em;bottom: 0;left: 50%;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-item {z-index: 1;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-parallel-item {display: flex;flex-direction: column;position: relative;background-color: white;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-parallel-item:only-child::after {width: 0;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0.2em;box-sizing: border-box;padding-bottom: 0.1em;background-color: white;position: relative;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-label label {font-family: monospace;font-weight: bold;background-color: white;display: inline-block;line-height: 1.2em;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-label-container {position: relative;z-index: 2;text-align: center;}#sk-169db5d9-e8f0-4e6a-bdec-d73831978a94 div.sk-container {display: inline-block;position: relative;}</style><div id=\"sk-169db5d9-e8f0-4e6a-bdec-d73831978a94\" class\"sk-top-container\"><div class=\"sk-container\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"7213bb0a-e849-49a7-b2dc-06bc0763c510\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"7213bb0a-e849-49a7-b2dc-06bc0763c510\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[('transformer_fechas',\n",
       "                 ColumnTransformer(remainder='passthrough',\n",
       "                                   transformers=[('transformer_fechas',\n",
       "                                                  TransformerFechas(),\n",
       "                                                  ['pickup_datetime'])])),\n",
       "                ('OrdinalEncoder', OrdinalEncoder())])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"308e2e27-8044-476d-87b3-ba5bbab1e10a\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"308e2e27-8044-476d-87b3-ba5bbab1e10a\">transformer_fechas: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(remainder='passthrough',\n",
       "                  transformers=[('transformer_fechas', TransformerFechas(),\n",
       "                                 ['pickup_datetime'])])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"e5fee2a5-926e-4d48-b745-2f052c437eba\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"e5fee2a5-926e-4d48-b745-2f052c437eba\">transformer_fechas</label><div class=\"sk-toggleable__content\"><pre>['pickup_datetime']</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"116b5c2b-4a65-4228-819e-4260d2da98e9\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"116b5c2b-4a65-4228-819e-4260d2da98e9\">TransformerFechas</label><div class=\"sk-toggleable__content\"><pre>TransformerFechas()</pre></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"924b8df4-99ac-482c-9b29-ca2741162ad9\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"924b8df4-99ac-482c-9b29-ca2741162ad9\">remainder</label><div class=\"sk-toggleable__content\"><pre>['vendor_id', 'pickup_borough']</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"a4c70d65-44ac-4bfd-9108-b4ec0c06cab1\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"a4c70d65-44ac-4bfd-9108-b4ec0c06cab1\">passthrough</label><div class=\"sk-toggleable__content\"><pre>passthrough</pre></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"5ed106bd-663b-4b16-a724-3540a9021fce\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"5ed106bd-663b-4b16-a724-3540a9021fce\">OrdinalEncoder</label><div class=\"sk-toggleable__content\"><pre>OrdinalEncoder()</pre></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "Pipeline(steps=[('transformer_fechas',\n",
       "                 ColumnTransformer(remainder='passthrough',\n",
       "                                   transformers=[('transformer_fechas',\n",
       "                                                  TransformerFechas(),\n",
       "                                                  ['pickup_datetime'])])),\n",
       "                ('OrdinalEncoder', OrdinalEncoder())])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(cat_pipeline)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b537332c",
   "metadata": {},
   "source": [
    "## Unión de Pipelines"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b87f8fa",
   "metadata": {},
   "source": [
    "Por último, usaremos nuevamente el `ColumnTransformer` para determinar cuáles son las variables numéricas y las variables catégoricas en nuestros datos. \n",
    "Definiendo varios elementos en la lista que le pasamos al ColumnTransformer le hacemos saber a sklearn que queremos diferentes transformaciones para las columnas y al final queremos unirlas para que todas queden en un solo numpy array. En nuestro caso, unimos las columnas que resultan del preprocesamiento de las categóricas y del de las numéricas."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d034a6c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1085891, 6)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.pipeline import FeatureUnion\n",
    "\n",
    "full_pipeline = ColumnTransformer(\n",
    "    [\n",
    "        (\"num_pipeline\", num_pipeline, num_cols),\n",
    "        (\"cat_pipeline\", cat_pipeline, cat_cols)\n",
    "    ]\n",
    ")\n",
    "\n",
    "X_transformed = full_pipeline.fit_transform(x_train, y_train)\n",
    "print(X_transformed.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d53840ea",
   "metadata": {},
   "source": [
    "En este diagrama podemos ver que ejecutamos las transformaciones por separado para las variables numéricas y las categóricas y al final las unimos para tener los datos en un array que le pasaremos al modelo al momento de entrenar."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "91a9d418",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-167f824d-f858-4de7-a881-01b09d704932 {color: black;background-color: white;}#sk-167f824d-f858-4de7-a881-01b09d704932 pre{padding: 0;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-toggleable {background-color: white;}#sk-167f824d-f858-4de7-a881-01b09d704932 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.2em 0.3em;box-sizing: border-box;text-align: center;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-167f824d-f858-4de7-a881-01b09d704932 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-167f824d-f858-4de7-a881-01b09d704932 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;margin: 0.25em 0.25em;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-estimator:hover {background-color: #d4ebff;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 2em;bottom: 0;left: 50%;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-item {z-index: 1;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-parallel-item {display: flex;flex-direction: column;position: relative;background-color: white;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-parallel-item:only-child::after {width: 0;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0.2em;box-sizing: border-box;padding-bottom: 0.1em;background-color: white;position: relative;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-label label {font-family: monospace;font-weight: bold;background-color: white;display: inline-block;line-height: 1.2em;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-label-container {position: relative;z-index: 2;text-align: center;}#sk-167f824d-f858-4de7-a881-01b09d704932 div.sk-container {display: inline-block;position: relative;}</style><div id=\"sk-167f824d-f858-4de7-a881-01b09d704932\" class\"sk-top-container\"><div class=\"sk-container\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"53a46f61-ec7d-474b-ba0b-34569aacf87f\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"53a46f61-ec7d-474b-ba0b-34569aacf87f\">ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(transformers=[('num_pipeline',\n",
       "                                 Pipeline(steps=[('transformer_coord',\n",
       "                                                  ColumnTransformer(remainder='passthrough',\n",
       "                                                                    transformers=[('transformer_dist',\n",
       "                                                                                   TransformerDistancia(),\n",
       "                                                                                   ['pickup_longitude',\n",
       "                                                                                    'pickup_latitude',\n",
       "                                                                                    'dropoff_longitude',\n",
       "                                                                                    'dropoff_latitude'])])),\n",
       "                                                 ('scaler', StandardScaler())]),\n",
       "                                 ['passenger_count', 'pickup_longitude',\n",
       "                                  'pickup_latitude', 'dropoff_longitude',\n",
       "                                  'dropoff_latitude']),\n",
       "                                ('cat_pipeline',\n",
       "                                 Pipeline(steps=[('transformer_fechas',\n",
       "                                                  ColumnTransformer(remainder='passthrough',\n",
       "                                                                    transformers=[('transformer_fechas',\n",
       "                                                                                   TransformerFechas(),\n",
       "                                                                                   ['pickup_datetime'])])),\n",
       "                                                 ('OrdinalEncoder',\n",
       "                                                  OrdinalEncoder())]),\n",
       "                                 ['vendor_id', 'pickup_borough',\n",
       "                                  'pickup_datetime'])])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"53f8a7b5-6bd4-4b36-82fc-6ad8adb25fdc\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"53f8a7b5-6bd4-4b36-82fc-6ad8adb25fdc\">num_pipeline</label><div class=\"sk-toggleable__content\"><pre>['passenger_count', 'pickup_longitude', 'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude']</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"405357f8-07fe-43d9-a42e-a6c80285cb54\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"405357f8-07fe-43d9-a42e-a6c80285cb54\">transformer_coord: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(remainder='passthrough',\n",
       "                  transformers=[('transformer_dist', TransformerDistancia(),\n",
       "                                 ['pickup_longitude', 'pickup_latitude',\n",
       "                                  'dropoff_longitude', 'dropoff_latitude'])])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"ea9e1f2e-5e3f-41d1-bdbd-cf1a4f34f804\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"ea9e1f2e-5e3f-41d1-bdbd-cf1a4f34f804\">transformer_dist</label><div class=\"sk-toggleable__content\"><pre>['pickup_longitude', 'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude']</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"b9735114-71ad-4020-a276-01f64f0a4e34\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"b9735114-71ad-4020-a276-01f64f0a4e34\">TransformerDistancia</label><div class=\"sk-toggleable__content\"><pre>TransformerDistancia()</pre></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"19cc0d1e-7635-45c9-98ea-62c8dfdc5a49\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"19cc0d1e-7635-45c9-98ea-62c8dfdc5a49\">remainder</label><div class=\"sk-toggleable__content\"><pre>['passenger_count']</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"eb831e18-a1ae-488a-9d93-f2850f212ff9\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"eb831e18-a1ae-488a-9d93-f2850f212ff9\">passthrough</label><div class=\"sk-toggleable__content\"><pre>passthrough</pre></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"72019720-88ad-495c-a390-50c83ab6ab01\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"72019720-88ad-495c-a390-50c83ab6ab01\">StandardScaler</label><div class=\"sk-toggleable__content\"><pre>StandardScaler()</pre></div></div></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"67293443-829e-4028-8784-e016da333699\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"67293443-829e-4028-8784-e016da333699\">cat_pipeline</label><div class=\"sk-toggleable__content\"><pre>['vendor_id', 'pickup_borough', 'pickup_datetime']</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"58a672ef-f6b4-4a9a-b307-3686fa09240f\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"58a672ef-f6b4-4a9a-b307-3686fa09240f\">transformer_fechas: ColumnTransformer</label><div class=\"sk-toggleable__content\"><pre>ColumnTransformer(remainder='passthrough',\n",
       "                  transformers=[('transformer_fechas', TransformerFechas(),\n",
       "                                 ['pickup_datetime'])])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"890ecd16-5a61-4bce-9bd3-88e178fdb410\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"890ecd16-5a61-4bce-9bd3-88e178fdb410\">transformer_fechas</label><div class=\"sk-toggleable__content\"><pre>['pickup_datetime']</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"aa474e75-3074-4f56-99c7-6d7a244381ca\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"aa474e75-3074-4f56-99c7-6d7a244381ca\">TransformerFechas</label><div class=\"sk-toggleable__content\"><pre>TransformerFechas()</pre></div></div></div></div></div></div><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"12363f37-e573-459a-b1c4-f8026fb9e8d1\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"12363f37-e573-459a-b1c4-f8026fb9e8d1\">remainder</label><div class=\"sk-toggleable__content\"><pre>['vendor_id', 'pickup_borough']</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"4c329d6d-aa1f-452c-b236-fde73521175c\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"4c329d6d-aa1f-452c-b236-fde73521175c\">passthrough</label><div class=\"sk-toggleable__content\"><pre>passthrough</pre></div></div></div></div></div></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"3fc45f6a-426d-4c85-8ca5-6c7ea166db5e\" type=\"checkbox\" ><label class=\"sk-toggleable__label\" for=\"3fc45f6a-426d-4c85-8ca5-6c7ea166db5e\">OrdinalEncoder</label><div class=\"sk-toggleable__content\"><pre>OrdinalEncoder()</pre></div></div></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "ColumnTransformer(transformers=[('num_pipeline',\n",
       "                                 Pipeline(steps=[('transformer_coord',\n",
       "                                                  ColumnTransformer(remainder='passthrough',\n",
       "                                                                    transformers=[('transformer_dist',\n",
       "                                                                                   TransformerDistancia(),\n",
       "                                                                                   ['pickup_longitude',\n",
       "                                                                                    'pickup_latitude',\n",
       "                                                                                    'dropoff_longitude',\n",
       "                                                                                    'dropoff_latitude'])])),\n",
       "                                                 ('scaler', StandardScaler())]),\n",
       "                                 ['passenger_count', 'pickup_longitude',\n",
       "                                  'pickup_latitude', 'dropoff_longitude',\n",
       "                                  'dropoff_latitude']),\n",
       "                                ('cat_pipeline',\n",
       "                                 Pipeline(steps=[('transformer_fechas',\n",
       "                                                  ColumnTransformer(remainder='passthrough',\n",
       "                                                                    transformers=[('transformer_fechas',\n",
       "                                                                                   TransformerFechas(),\n",
       "                                                                                   ['pickup_datetime'])])),\n",
       "                                                 ('OrdinalEncoder',\n",
       "                                                  OrdinalEncoder())]),\n",
       "                                 ['vendor_id', 'pickup_borough',\n",
       "                                  'pickup_datetime'])])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(full_pipeline)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c99a8df",
   "metadata": {},
   "source": [
    "## Guardar Pipeline a Disco"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "022aaba2",
   "metadata": {},
   "source": [
    "Además de tener una clara separación entre los métodos `.fit()` y `.transform()`, la ventaja de escribir todo nuestro proceso como un solo pipeline (`full_pipeline`) es que podemos guardar en disco el objeto que usamos para preprocesar. \n",
    "\n",
    "Esto es útil para tener todas las transformaciones definidas en un objeto al momento que queramos hacer reproducibles nuestras transformaciones para, por ejemplo, desplegar nuestro modelo.\n",
    "\n",
    "**Creamos** un **contexto** usando `open()` y usamos la función **dill.dump()** para **guardar** nuestro **flujo de preprocesamiento en disco**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "a761bc86",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Se guardo el pipeline !!\n"
     ]
    }
   ],
   "source": [
    "import dill\n",
    "dill.settings['recurse'] = True\n",
    "\n",
    "with open(\"preprocesser.pkl\", \"wb\") as f:\n",
    "    dill.dump(full_pipeline, f)  # Guardar el flujo\n",
    "    \n",
    "print(\"Se guardo el pipeline !!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2191a374",
   "metadata": {},
   "source": [
    "Para cargar nuestro flujo, usamos la función `dill.load()`. Además verificamos que el resultado de las transformaciones con el flujo que teníamos en el notebook y el que cargamos desde el disco es idéntico."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da40ce8a",
   "metadata": {},
   "source": [
    "## Checkpoint #4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "9f1184a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "with open(\"preprocesser.pkl\", \"rb\") as f:\n",
    "    loaded_pipeline = dill.load(f) # Cargar el flujo\n",
    "    \n",
    "X_loaded = loaded_pipeline.transform(x_train)\n",
    "\n",
    "print((X_loaded == X_transformed).all())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d72a9d3f",
   "metadata": {},
   "source": [
    "# MLflow\n",
    "\n",
    "Veremos cómo usar MLflow para guardar métricas y parámetros de todos los modelos que corramos."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62e2f539",
   "metadata": {},
   "source": [
    "## Transformación de Datos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2be27ad9",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"preprocesser.pkl\", \"rb\") as f:\n",
    "    preprocessor = dill.load(f)\n",
    "\n",
    "X_train = preprocessor.transform(x_train)\n",
    "X_test = preprocessor.transform(x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "146f1adb",
   "metadata": {},
   "source": [
    "## Entrenar Modelos"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "737a93be",
   "metadata": {},
   "source": [
    "### Baseline con DummyRegressor"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3fcf922",
   "metadata": {},
   "source": [
    "Una opción que nos ofrece sklearn es entrenar `modelos dummy`. Estos son modelos que predicen lo mismo para todos los casos (en este caso el promedio) y nos sirven como una medida de cuál es el rendimiento mínimo para un problema. Si por alguna razón nuestro modelo tiene peor desempeño que el modelo dummy, tenemos que revisar nuestro modelo porque probablemente tenemos errores en la implementación.\n",
    "En este caso vamos a usar el `DummyRegressor` que siempre predice la media de los datos de entrenamiento. La función `evaluar_predicciones` es una ayuda para poder calcular varias métricas de un modelo usando metricas incluídas de scikit-learn."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "cf811863",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_absolute_error, mean_absolute_percentage_error, mean_squared_error, r2_score, mean_squared_log_error\n",
    "\n",
    "def evaluar_predicciones(y_pred, y_true):\n",
    "    mae = mean_absolute_error(y_pred=y_pred, y_true=y_true)\n",
    "    mape = mean_absolute_percentage_error(y_pred=y_pred, y_true=y_true)\n",
    "    rmse = mean_squared_error(y_pred=y_pred, y_true=y_true, squared=False)\n",
    "    print(f\"MAE: {mae:.2f}\")\n",
    "    print(f\"MAPE: {mape}\")\n",
    "    print(f\"RMSE: {rmse}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eec937cb",
   "metadata": {},
   "source": [
    "### DummyRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "51da41b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAIN\n",
      "MAE: 468.61\n",
      "MAPE: 0.95585624685749\n",
      "RMSE: 683.1658889831872\n",
      "VALIDATION | TEST\n",
      "MAE: 468.42\n",
      "MAPE: 0.9567650882594899\n",
      "RMSE: 680.0030431781618\n"
     ]
    }
   ],
   "source": [
    "from sklearn.dummy import DummyRegressor\n",
    "\n",
    "dummy_model = DummyRegressor(strategy=\"mean\")\n",
    "dummy_model.fit(X_train, y_train)\n",
    "y_train_dummy = dummy_model.predict(X_train)\n",
    "y_test_dummy = dummy_model.predict(X_test)\n",
    "\n",
    "print(\"TRAIN\")\n",
    "evaluar_predicciones(y_pred=y_train_dummy, y_true=y_train)\n",
    "\n",
    "print(\"VALIDATION | TEST\")\n",
    "evaluar_predicciones(y_pred=y_test_dummy, y_true=y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97e1d2f8",
   "metadata": {},
   "source": [
    "### Baseline con regresión lineal"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f438cb9",
   "metadata": {},
   "source": [
    "Otra buena medida es siempre empezar con un modelo lineal. Así tenemos una medida de cómo se desempeña un modelo sencillo.\n",
    "\n",
    "Algo para notar es que sin importar el modelo, los modelos de sklearn siempre siguen el mismo proceso. \n",
    "1. Inicializar modelo con la clase de sklearn. \n",
    "2. Ejecutar función .fit(X_train, y_train). \n",
    "3. Ejecutar función .predict(X) para generar las predicciones.\n",
    "\n",
    "Esta consistencia en el proceso simplifica el uso de diferentes modelos."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44bc2250",
   "metadata": {},
   "source": [
    "**Checkpoint #1**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "e7b050c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAIN\n",
      "MAE: 296.09\n",
      "MAPE: 0.5533021393418179\n",
      "RMSE: 487.05043254440744\n",
      "VALIDATION\n",
      "MAE: 295.55\n",
      "MAPE: 0.554068139553267\n",
      "RMSE: 529.0958979710383\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "linear_model = LinearRegression()\n",
    "linear_model.fit(X_train, y_train)\n",
    "y_train_linear = linear_model.predict(X_train)\n",
    "y_test_linear = linear_model.predict(X_test)\n",
    "\n",
    "print(\"TRAIN\")\n",
    "evaluar_predicciones(y_pred=y_train_linear, y_true=y_train)\n",
    "\n",
    "print(\"VALIDATION\")\n",
    "evaluar_predicciones(y_pred=y_test_linear, y_true=y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "839990c3",
   "metadata": {},
   "source": [
    "### ¿Cómo comparar modelos más allá de un print? MLFlow 🙌"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d31aeb0",
   "metadata": {},
   "source": [
    "[MLflow](https://www.mlflow.org/) es una librería open-source que nos ayuda a **manejar** todo el **ciclo de modelos de machine learning**. En este caso vamos a ver cómo podemos **usar MLflow** para **guardar resultados** del desempeño **de modelos** para **determinar** cuál modelo es **el mejor**.\n",
    "\n",
    "Vamos a correr los mismos modelos que acabamos de comparar pero esta vez vamos a usar MLflow para ver los resultados."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "a0d0ddc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import mlflow\n",
    "mlflow.sklearn.autolog()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3e9bdc5",
   "metadata": {},
   "source": [
    "`mlflow.sklearn.autolog()` automáticamente nos ayuda a **guardar varias métricas y parámetros del modelo**. \n",
    "Sin embargo, no incluye valores para datos de validación. \n",
    "\n",
    "Para guardar métricas que no se incluyen en `.autolog` vamos a usar la función `mlflow.log_metric` _dentro del context manager de run_.\n",
    "\n",
    "Vamos primero a usar el modelo dummy y vamos a poner run_name=\"dummy\" para poder verlo con el nombre correcto en MLflow."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "790e8e1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "with mlflow.start_run(run_name=\"dummy\") as run:\n",
    "    dummy_model.fit(X_train, y_train)\n",
    "    y_pred_val = dummy_model.predict(X_test)\n",
    "    val_mae = mean_absolute_error(y_pred=y_pred_val, y_true=y_test)\n",
    "    val_rmse = mean_squared_error(y_pred=y_pred_val, y_true=y_test, \n",
    "                                  squared=False)\n",
    "    val_mape = mean_absolute_percentage_error(y_pred=y_pred_val, y_true=y_test)\n",
    "    val_r2 = r2_score(y_pred=y_pred_val, y_true=y_test)\n",
    "\n",
    "    mlflow.log_metric(\"val_mae\", val_mae)\n",
    "    mlflow.log_metric(\"val_rmse\", val_rmse)\n",
    "    mlflow.log_metric(\"val_mape\", val_mape)\n",
    "    mlflow.log_metric(\"val_r2\", val_r2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "613622c5",
   "metadata": {},
   "source": [
    "Una vez termine de correr esta celda debemos **abrir la interfaz de MLflow** para ver los resultados. Deberíamos encontrar una nueva **carpeta** llamada **mlruns** en la ubicación donde corrió este notebook. \n",
    "Una vez estemos en la carpeta donde está mlruns, debemos ir a un terminal a esa ubicación y ejecutamos el comando"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04d24629",
   "metadata": {},
   "source": [
    "> mlflow ui"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "436d4d7c",
   "metadata": {},
   "source": [
    "Ahí nos debería salir el mensaje de que está iniciando un servidor y dar la ruta para acceder. (normalmente la ruta es http://127.0.0.1:5000). Copiamos esa dirección en un navegador y ahí ya deberíamos ver la interfaz de MLflow."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1eba59d5",
   "metadata": {},
   "source": [
    "En la interfaz de MLflow estarán los modelos que corramos usando MLflow y podemos filtrar u ordenar por métricas o parámetros que consideramos importantes. \n",
    "\n",
    "En general, con la interfaz de MLflow tenemos una base de datos de modelos que podemos analizar fácilmente.\n",
    "\n",
    "Además, si hacemos click un alguna de las runs (click en la columna Start Time) podemos ver que tenemos los datos más detallados y los artifacts de cada run. Los artifacts pueden ser todo tipo de archivos (por ejemplo imágenes y CSVs) que nos ayudan a guardar lo relevante de cada caso como el modelo o el ambiente de conda desde el que corrimos el modelo."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8461f42d",
   "metadata": {},
   "source": [
    "Ahora corramos nuevamente la regresión lineal para que quede registrada en MLflow. \n",
    "En este caso vamos a guardar nuestro preprocesador que quede asociado a cada run usando `.log_artifact.`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "150a3af3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def log_metrics_mlflow(y_pred_val, y_test,name_modelo):\n",
    "    print(\"log_metrics_mlflow >>\", name_modelo)\n",
    "    \n",
    "    val_mae = mean_absolute_error(y_pred=y_pred_val, y_true=y_test)\n",
    "    val_rmse = mean_squared_error(y_pred=y_pred_val, y_true=y_test, \n",
    "                                  squared=False)\n",
    "    val_mape = mean_absolute_percentage_error(y_pred=y_pred_val, y_true=y_test)\n",
    "    val_r2 = r2_score(y_pred=y_pred_val, y_true=y_test)\n",
    "\n",
    "    mlflow.log_metric(\"val_mae\", val_mae)\n",
    "    mlflow.log_metric(\"val_rmse\", val_rmse)\n",
    "    mlflow.log_metric(\"val_mape\", val_mape)\n",
    "    mlflow.log_metric(\"val_r2\", val_r2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "43f3ab47",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "log_metrics_mlflow >> linear_regression\n"
     ]
    }
   ],
   "source": [
    "with mlflow.start_run(run_name=\"linear_regression\") as run:\n",
    "    #TODO: Hacer fit a un modelo lineal\n",
    "    linear_model.fit(X_train, y_train)\n",
    "    y_pred_val = linear_model.predict(X_test)\n",
    "    log_metrics_mlflow(y_pred_val, y_test,\"linear_regression\")\n",
    "    mlflow.log_artifact(\"lr_model.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "003618f4",
   "metadata": {},
   "source": [
    "Ahora que tenemos la regresión lineal y el modelo dummy en MLflow, vamos a probar con un modelo más complejo como el Random Forest. Por ahora vamos a dejar los parámetros por defecto que trae la clase `RandomForestRegressor` excepto `n_jobs` que lo vamos a poner con el parámetro `n_jobs=2`. Poner n_jobs=2 hace que el modelo corra en paralelo en 2 procesadores.\n",
    "\n",
    "Entrenar el random forest puede tomar algunos minutos así que es un buen momento para resolver cualquier duda o tomar un descanso y estirar las piernas :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8020ab0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "with mlflow.start_run(run_name=\"random_forest\") as run:\n",
    "    #TODO: Hacer fit a un modelo random forest\n",
    "    rf_model = RandomForestRegressor(n_jobs=2)\n",
    "    rf_model.fit(X_train, y_train)\n",
    "    y_pred_val = rf_model.predict(X_test)\n",
    "    log_metrics_mlflow(y_pred_val, y_test,\"random_forest\")\n",
    "    mlflow.log_artifact(\"rf_model.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a206cd22",
   "metadata": {},
   "source": [
    "Con esas herramientas puedes buscar [más modelos de regresión de sklearn](https://scikit-learn.org/stable/supervised_learning.html) o mejorar los hiperparámetros de los modelos que usamos para intentar mejorar las métricas que obtuvimos hasta el momento. Incluso librerías como [XGBoost](https://xgboost.readthedocs.io/en/latest/python/python_api.html#module-xgboost.sklearn) o [LightGBM](https://lightgbm.readthedocs.io/en/latest/Python-API.html#scikit-learn-api) ofrecen la opción de crear modelos con la API de sklearn. A pesar de que cada modelo puede usar técnicas muy diferentes, el código va a ser casi idéntico gracias a la consistencia en la API de scikit-learn."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7bdfef3",
   "metadata": {},
   "source": [
    "# Extras"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20ba283a",
   "metadata": {},
   "source": [
    "### Transformer Velocidades"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd2c8351",
   "metadata": {},
   "source": [
    "Vamos a crear un transformer para calcular la velocidad promedio en cada zona (borough) de Nueva York y asignarle esa velocidad a cada viaje.\n",
    "Para calcular la velocidad necesitamos el tiempo que se demora el viaje.\n",
    "\n",
    "Como esta es nuestra variable dependiente, debemos tener en cuenta que solo podemos usar los datos de train porque, en principio, no hemos observado los datos de validation. \n",
    "\n",
    "Al calcular las velocidades para cada zona con los datos que llegan a .fit() evitamos el data leakage porque garantizamos que no estamos usando los tiempos de viaje de validación."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b424c9cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TransformerVelocidad(BaseEstimator, TransformerMixin):\n",
    "    def fit(self, X, y):\n",
    "        X_init = X[[\"pickup_latitude\", \"pickup_longitude\"]].to_numpy()\n",
    "        X_final = X[[\"dropoff_latitude\", \"dropoff_longitude\"]].to_numpy()\n",
    "\n",
    "        # Distancia de Haversine\n",
    "        distancia = self.distancia_haversine(X_init=X_init, X_final=X_final)\n",
    "        \n",
    "        velocidad_df = pd.DataFrame()\n",
    "        tiempo_en_horas = y.to_numpy() / 3600\n",
    "        velocidad_df[\"velocidad\"] = distancia / tiempo_en_horas\n",
    "        velocidad_df[\"pickup_borough\"] = X[\"pickup_borough\"]\n",
    "        velocidad_borough = velocidad_df.groupby(\"pickup_borough\")[\"velocidad\"].mean()\n",
    "        self.velocidad_borough = velocidad_borough.to_dict()\n",
    "        return self\n",
    "\n",
    "    def transform(self, X, y=None):\n",
    "        velocidad_df = pd.DataFrame()\n",
    "        velocidad_df[\"velocidad\"] = X[\"pickup_borough\"].map(self.velocidad_borough)\n",
    "        return velocidad_df\n",
    "    \n",
    "    def distancia_haversine(self, X_init, X_final):\n",
    "        # Convertir de decimal a radianes\n",
    "        X_init = np.radians(X_init)\n",
    "        X_final = np.radians(X_final)\n",
    "\n",
    "        # Formula de Haversine\n",
    "        dlat = X_final[:, 0] - X_init[:, 0] \n",
    "        dlon = X_final[:, 1] - X_init[:, 1]\n",
    "        a = (np.sin(dlat / 2) ** 2) + np.cos(X_init[:, 0]) * np.cos(X_final[:, 0]) * (np.sin(dlon / 2) ** 2)\n",
    "        c = 2 * np.arcsin(np.sqrt(a))\n",
    "        r = 6371 # Radio de la tierra en kilómetros\n",
    "        return c * r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb1e8ac7",
   "metadata": {},
   "outputs": [],
   "source": [
    "transformer_velocidades = TransformerVelocidad()\n",
    "velocidades_df = transformer_velocidades.fit_transform(x_train, y_train)\n",
    "velocidades_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee88e265",
   "metadata": {},
   "outputs": [],
   "source": [
    "# conda install xgboost lightgbm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8fadc1ec",
   "metadata": {},
   "source": [
    "### LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60181839",
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightgbm as lgb\n",
    "from lightgbm import LGBMRegressor\n",
    "\n",
    "mlflow.lightgbm.autolog()\n",
    "with mlflow.start_run(run_name=\"lgbm\") as run:\n",
    "    lgbm_model = LGBMRegressor()\n",
    "    lgbm_model.fit(X_train, y_train)\n",
    "    y_pred_val = lgbm_model.predict(X_test)\n",
    "    log_metrics_mlflow(y_pred_val, y_test,\"LGBMRegressor\")\n",
    "    mlflow.log_artifact(\"lgbm_model.pkl\")\n",
    "    \n",
    "    with open(\"lgbm_model.pkl\", \"wb\") as f:\n",
    "        dill.dump(lgbm_model, f)\n",
    "    mlflow.log_artifact(\"lgbm_model.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e283db7",
   "metadata": {},
   "source": [
    "### XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01d643df",
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBRegressor\n",
    "\n",
    "mlflow.xgboost.autolog()\n",
    "with mlflow.start_run(run_name=\"xgboost\") as run:\n",
    "    xgb_model = XGBRegressor()\n",
    "    xgb_model.fit(X_train, y_train)\n",
    "    y_pred_val = xgb_model.predict(X_test)\n",
    "    log_metrics_mlflow(y_pred_val, y_test,\"LGBMRegressor\")\n",
    "    mlflow.log_artifact(\"xgb_model.pkl\")\n",
    "    \n",
    "    with open(\"xgb_model.pkl\", \"wb\") as f:\n",
    "        dill.dump(xgb_model, f)\n",
    "    mlflow.log_artifact(\"xgb_model.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4987555c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}