Car_Price_Train_Pipeline.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPezgfDhMGwlpuHv80h5TnP"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "ea41a3a48e03438bb86adf7c675891ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f4cb5ff7291d4f4abce72965261058e2",
              "IPY_MODEL_0165404c30ce4fd2a27bfa9b6d08ceb9",
              "IPY_MODEL_c9260d37d2f94668b743a4d1c958fa9c"
            ],
            "layout": "IPY_MODEL_9ca680b5118b458a8499544cb5204cf7"
          }
        },
        "f4cb5ff7291d4f4abce72965261058e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e94142f6119a4cfe8d8b4e8ed4550ea5",
            "placeholder": "​",
            "style": "IPY_MODEL_e58a762c432f4b44a5fa5a184b077894",
            "value": "Model export complete: 100%"
          }
        },
        "0165404c30ce4fd2a27bfa9b6d08ceb9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_91a97bfe8d3a46daa717aede99458a78",
            "max": 6,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b1869bceaca54faf87cfefe353dc166e",
            "value": 6
          }
        },
        "c9260d37d2f94668b743a4d1c958fa9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f70631e73d574b06ab88a8de03bb1fca",
            "placeholder": "​",
            "style": "IPY_MODEL_486784a2397a494a91ab915537ecffb4",
            "value": " 6/6 [00:44&lt;00:00,  8.74s/it]"
          }
        },
        "9ca680b5118b458a8499544cb5204cf7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e94142f6119a4cfe8d8b4e8ed4550ea5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e58a762c432f4b44a5fa5a184b077894": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "91a97bfe8d3a46daa717aede99458a78": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b1869bceaca54faf87cfefe353dc166e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f70631e73d574b06ab88a8de03bb1fca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "486784a2397a494a91ab915537ecffb4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Training Pipeline"
      ],
      "metadata": {
        "id": "Kn145OT5BI9p"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Installing Hopsworks (feature store)"
      ],
      "metadata": {
        "id": "P21bnP3RBnAY"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Krii7HEsA461",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "070112f5-a70a-4a8e-e98e-30cb5c64ab66"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K     |████████████████████████████████| 120 kB 14.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 50 kB 5.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 132 kB 52.1 MB/s \n",
            "\u001b[K     |████████████████████████████████| 45 kB 3.3 MB/s \n",
            "\u001b[K     |████████████████████████████████| 68 kB 6.8 MB/s \n",
            "\u001b[K     |████████████████████████████████| 43 kB 2.1 MB/s \n",
            "\u001b[K     |████████████████████████████████| 4.9 MB 42.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 42 kB 1.7 MB/s \n",
            "\u001b[K     |████████████████████████████████| 2.8 MB 45.5 MB/s \n",
            "\u001b[K     |████████████████████████████████| 2.3 MB 39.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 4.1 MB 46.5 MB/s \n",
            "\u001b[K     |████████████████████████████████| 109 kB 57.7 MB/s \n",
            "\u001b[K     |████████████████████████████████| 67 kB 6.7 MB/s \n",
            "\u001b[K     |████████████████████████████████| 140 kB 46.7 MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.6 MB 44.5 MB/s \n",
            "\u001b[K     |████████████████████████████████| 127 kB 75.0 MB/s \n",
            "\u001b[K     |████████████████████████████████| 500 kB 70.4 MB/s \n",
            "\u001b[K     |████████████████████████████████| 9.9 MB 59.0 MB/s \n",
            "\u001b[K     |████████████████████████████████| 79 kB 8.6 MB/s \n",
            "\u001b[K     |████████████████████████████████| 59 kB 6.4 MB/s \n",
            "\u001b[K     |████████████████████████████████| 57 kB 5.4 MB/s \n",
            "\u001b[K     |████████████████████████████████| 2.3 MB 28.0 MB/s \n",
            "\u001b[?25h  Building wheel for hopsworks (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for hsfs (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for avro (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for hsml (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for pyhopshive (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for thrift (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for twofish (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ],
      "source": [
        "!pip install -U hopsworks --quiet"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Importing ML libraries \n",
        "from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error, make_scorer\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.ensemble import RandomForestRegressor\n",
        "# Importing pandas, numpy and hopsworks\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import hopsworks"
      ],
      "metadata": {
        "id": "H2346dy3BIGV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Creating a feaure view in Hopsworks"
      ],
      "metadata": {
        "id": "PLogDnkjnyBq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "project = hopsworks.login()\n",
        "fs = project.get_feature_store()\n",
        "\n",
        "try: \n",
        "    feature_view = fs.get_feature_view(name=\"car_prices\", version=1)\n",
        "except:\n",
        "    car_fg  = fs.get_feature_group(name=\"car_prices\", version=1)\n",
        "    query = car_fg .select_all()\n",
        "    feature_view = fs.create_feature_view(name=\"car_prices\",\n",
        "                                      version=1,\n",
        "                                      description=\"Read from car price dataset\",\n",
        "                                      labels=[\"selling_price\"],\n",
        "                                      query=query)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_DJKaURTBXqY",
        "outputId": "e7fabddb-4839-4519-a360-5f8a6a304e20"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Connection closed.\n",
            "Connected. Call `.close()` to terminate connection gracefully.\n",
            "\n",
            "Logged in to project, explore it here https://c.app.hopsworks.ai:443/p/3342\n",
            "Connected. Call `.close()` to terminate connection gracefully.\n",
            "Feature view created successfully, explore it at \n",
            "https://c.app.hopsworks.ai:443/p/3342/fs/3289/fv/car_prices/version/1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Splitting the data into train and test sets\n",
        "X_train, X_test, y_train, y_test = feature_view.train_test_split(0.15)"
      ],
      "metadata": {
        "id": "zzYQwO4dCP5k",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4bbc5964-45c1-443f-b9e6-17a266f6223d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "VersionWarning: Incremented version to `1`.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Dropping the high cardinality feature name from X_train and X_test\n",
        "X_train.drop(\"full_name\", axis = 1, inplace = True)\n",
        "X_test.drop(\"full_name\", axis = 1, inplace = True)"
      ],
      "metadata": {
        "id": "ZhxvO0AWCeyr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = RandomForestRegressor()\n",
        "model.fit(X_train,y_train)\n",
        "model.score(X_test,y_test)\n",
        "mae = mean_absolute_error(y_test, model.predict(X_test))"
      ],
      "metadata": {
        "id": "Vhgk9AaJiduK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ca40cb06-739d-40a8-b6d1-c2dd8fbc832a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"The mean absolute error in lakhs is:\", mae)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "42ECt9m-kbKX",
        "outputId": "0fe11a6d-e98e-4b42-bb49-9e43a1ccc48d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The mean absolute error in lakhs is: 1.3666149552148883\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Saving the model in the model registry of Hopsworks"
      ],
      "metadata": {
        "id": "SpGwLfVVoJHw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from hsml.schema import Schema\n",
        "from hsml.model_schema import ModelSchema\n",
        "import os\n",
        "import joblib\n",
        "import hopsworks\n",
        "import shutil\n",
        "\n",
        "project =  hopsworks.login()\n",
        "mr = project.get_model_registry()\n",
        "\n",
        "# The 'car prices' directory will be saved to the model registry\n",
        "model_dir=\"car_prices_model\"\n",
        "if os.path.isdir(model_dir) == False:\n",
        "    os.mkdir(model_dir)\n",
        "joblib.dump(model, model_dir + \"/car_prices_model.pkl\")\n",
        "\n",
        "input_example = X_train.sample()\n",
        "input_schema = Schema(X_train)\n",
        "output_schema = Schema(y_train)\n",
        "model_schema = ModelSchema(input_schema, output_schema)\n",
        "\n",
        "car_prices_model = mr.python.create_model(\n",
        "    version=1,\n",
        "    name=\"car_prices\", \n",
        "    metrics={\"Mean Absolute Error\" : mae},\n",
        "    model_schema=model_schema,\n",
        "    input_example=input_example, \n",
        "    description=\"Car Price Predictor\")\n",
        "\n",
        "car_prices_model.save(model_dir)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170,
          "referenced_widgets": [
            "ea41a3a48e03438bb86adf7c675891ae",
            "f4cb5ff7291d4f4abce72965261058e2",
            "0165404c30ce4fd2a27bfa9b6d08ceb9",
            "c9260d37d2f94668b743a4d1c958fa9c",
            "9ca680b5118b458a8499544cb5204cf7",
            "e94142f6119a4cfe8d8b4e8ed4550ea5",
            "e58a762c432f4b44a5fa5a184b077894",
            "91a97bfe8d3a46daa717aede99458a78",
            "b1869bceaca54faf87cfefe353dc166e",
            "f70631e73d574b06ab88a8de03bb1fca",
            "486784a2397a494a91ab915537ecffb4"
          ]
        },
        "id": "h8xy8S69lw-H",
        "outputId": "bd560e9a-bf67-4c54-dcfd-23ce67a3ca89"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Connection closed.\n",
            "Connected. Call `.close()` to terminate connection gracefully.\n",
            "\n",
            "Logged in to project, explore it here https://c.app.hopsworks.ai:443/p/3342\n",
            "Connected. Call `.close()` to terminate connection gracefully.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/6 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ea41a3a48e03438bb86adf7c675891ae"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model created, explore it at https://c.app.hopsworks.ai:443/p/3342/models/car_prices/1\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Model(name: 'car_prices', version: 1)"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    }
  ]
}