Stacking_Practice.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This script was based on the framework from Human Analog, https://www.kaggle.com/humananalog/xgboost-lasso\n",
    "# and was then trained using stacking methodology, which is also called a Super Learner Ensemble. \n",
    "# \n",
    "# The basic idea can be understood in three steps as listed below. \n",
    "# These words are borrowed from h2o documentation. http://docs.h20.ai\n",
    "#\n",
    "# 1. Set up the emsemble:\n",
    "#    a. Specify a list of L base algorithms\n",
    "#    b. Specify a metalearning algorithm\n",
    "# 2. Train the ensemble:\n",
    "#    a. Train each of the L base learner with k-fold cross-validation and collect the cross-validated predicted values from \n",
    "#       each of the L base learner\n",
    "#    b. Use the collected predicted values as the input, labels of the training data as the output for the metalearning layer\n",
    "#       (lavel one) to train the layer one.\n",
    "# 3. Predict on the new data:\n",
    "#    a. First generate predictions from base learner\n",
    "#    b. Feed those predictions into metalearner to generate ensemble prediction\n",
    "#    c. submit to kaggle. You are done!\n",
    "#\n",
    "# Here I will only test 2-layer stacking, more layers can be done accordingly....\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import KFold, cross_val_score, train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import mean_squared_error, make_scorer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load the data.\n",
    "train_df = pd.read_csv(\"train.csv\")\n",
    "test_df = pd.read_csv(\"test.csv\")\n",
    "\n",
    "\n",
    "# There are a few houses with more than 4000 sq ft living area that are\n",
    "# outliers, so we drop them from the training data. (There is also one in\n",
    "# the test set but we obviously can't drop that one.)\n",
    "train_df.drop(train_df[train_df[\"GrLivArea\"] > 4000].index, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The test example with ID 666 has GarageArea, GarageCars, and GarageType \n",
    "# but none of the other fields, so use the mode and median to fill them in.\n",
    "test_df.loc[666, \"GarageQual\"] = \"TA\"\n",
    "test_df.loc[666, \"GarageCond\"] = \"TA\"\n",
    "test_df.loc[666, \"GarageFinish\"] = \"Unf\"\n",
    "test_df.loc[666, \"GarageYrBlt\"] = \"1980\"\n",
    "\n",
    "# The test example 1116 only has GarageType but no other information. We'll \n",
    "# assume it does not have a garage.\n",
    "test_df.loc[1116, \"GarageType\"] = np.nan\n",
    "\n",
    "# For imputing missing values: fill in missing LotFrontage values by the median\n",
    "# LotFrontage of the neighborhood.\n",
    "lot_frontage_by_neighborhood = train_df[\"LotFrontage\"].groupby(train_df[\"Neighborhood\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Used to convert categorical features into ordinal numbers.\n",
    "# (There's probably an easier way to do this, but it works.)\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()\n",
    "\n",
    "\n",
    "def factorize(df, factor_df, column, fill_na=None):\n",
    "    factor_df[column] = df[column]\n",
    "    if fill_na is not None:\n",
    "        factor_df[column].fillna(fill_na, inplace=True)\n",
    "    le.fit(factor_df[column].unique())\n",
    "    factor_df[column] = le.transform(factor_df[column])\n",
    "    return factor_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Combine all the (numerical) features into one big DataFrame. We don't add \n",
    "# the one-hot encoded variables here yet, that happens later on.\n",
    "def munge(df):\n",
    "    all_df = pd.DataFrame(index = df.index)\n",
    "   \n",
    "    all_df[\"LotFrontage\"] = df[\"LotFrontage\"]   \n",
    "    for key, group in lot_frontage_by_neighborhood:\n",
    "        idx = (df[\"Neighborhood\"] == key) & (df[\"LotFrontage\"].isnull())\n",
    "        all_df.loc[idx, \"LotFrontage\"] = group.median()    \n",
    "\n",
    "    all_df[\"LotArea\"] = df[\"LotArea\"]\n",
    "\n",
    "    all_df[\"MasVnrArea\"] = df[\"MasVnrArea\"]\n",
    "    all_df[\"MasVnrArea\"].fillna(0, inplace=True)\n",
    "   \n",
    "    all_df[\"BsmtFinSF1\"] = df[\"BsmtFinSF1\"]\n",
    "    all_df[\"BsmtFinSF1\"].fillna(0, inplace=True)\n",
    "\n",
    "    all_df[\"BsmtFinSF2\"] = df[\"BsmtFinSF2\"]\n",
    "    all_df[\"BsmtFinSF2\"].fillna(0, inplace=True)\n",
    "\n",
    "    all_df[\"BsmtUnfSF\"] = df[\"BsmtUnfSF\"]\n",
    "    all_df[\"BsmtUnfSF\"].fillna(0, inplace=True)\n",
    "\n",
    "    all_df[\"TotalBsmtSF\"] = df[\"TotalBsmtSF\"]\n",
    "    all_df[\"TotalBsmtSF\"].fillna(0, inplace=True)\n",
    "\n",
    "    all_df[\"1stFlrSF\"] = df[\"1stFlrSF\"]\n",
    "    all_df[\"2ndFlrSF\"] = df[\"2ndFlrSF\"]\n",
    "    all_df[\"GrLivArea\"] = df[\"GrLivArea\"]\n",
    "    \n",
    "    all_df[\"GarageArea\"] = df[\"GarageArea\"]\n",
    "    all_df[\"GarageArea\"].fillna(0, inplace=True)\n",
    "\n",
    "    all_df[\"WoodDeckSF\"] = df[\"WoodDeckSF\"]\n",
    "    all_df[\"OpenPorchSF\"] = df[\"OpenPorchSF\"]\n",
    "    all_df[\"EnclosedPorch\"] = df[\"EnclosedPorch\"]\n",
    "    all_df[\"3SsnPorch\"] = df[\"3SsnPorch\"]\n",
    "    all_df[\"ScreenPorch\"] = df[\"ScreenPorch\"]\n",
    "    \n",
    "    all_df[\"BsmtFullBath\"] = df[\"BsmtFullBath\"]\n",
    "    all_df[\"BsmtFullBath\"].fillna(0, inplace=True)\n",
    "\n",
    "    all_df[\"BsmtHalfBath\"] = df[\"BsmtHalfBath\"]\n",
    "    all_df[\"BsmtHalfBath\"].fillna(0, inplace=True)\n",
    "\n",
    "    all_df[\"FullBath\"] = df[\"FullBath\"] \n",
    "    all_df[\"HalfBath\"] = df[\"HalfBath\"] \n",
    "    all_df[\"BedroomAbvGr\"] = df[\"BedroomAbvGr\"] \n",
    "    all_df[\"KitchenAbvGr\"] = df[\"KitchenAbvGr\"] \n",
    "    all_df[\"TotRmsAbvGrd\"] = df[\"TotRmsAbvGrd\"] \n",
    "    all_df[\"Fireplaces\"] = df[\"Fireplaces\"] \n",
    "\n",
    "    all_df[\"GarageCars\"] = df[\"GarageCars\"]\n",
    "    all_df[\"GarageCars\"].fillna(0, inplace=True)\n",
    "\n",
    "    all_df[\"CentralAir\"] = (df[\"CentralAir\"] == \"Y\") * 1.0\n",
    "   \n",
    "    all_df[\"OverallQual\"] = df[\"OverallQual\"]\n",
    "    all_df[\"OverallCond\"] = df[\"OverallCond\"]\n",
    "\n",
    "    # Quality measurements are stored as text but we can convert them to \n",
    "    # numbers where a higher number means higher quality.\n",
    "\n",
    "    qual_dict = {None: 0, \"Po\": 1, \"Fa\": 2, \"TA\": 3, \"Gd\": 4, \"Ex\": 5}\n",
    "    all_df[\"ExterQual\"] = df[\"ExterQual\"].map(qual_dict).astype(int)\n",
    "    all_df[\"ExterCond\"] = df[\"ExterCond\"].map(qual_dict).astype(int)\n",
    "    all_df[\"BsmtQual\"] = df[\"BsmtQual\"].map(qual_dict).astype(int)\n",
    "    all_df[\"BsmtCond\"] = df[\"BsmtCond\"].map(qual_dict).astype(int)\n",
    "    all_df[\"HeatingQC\"] = df[\"HeatingQC\"].map(qual_dict).astype(int)\n",
    "    all_df[\"KitchenQual\"] = df[\"KitchenQual\"].map(qual_dict).astype(int)\n",
    "    all_df[\"FireplaceQu\"] = df[\"FireplaceQu\"].map(qual_dict).astype(int)\n",
    "    all_df[\"GarageQual\"] = df[\"GarageQual\"].map(qual_dict).astype(int)\n",
    "    all_df[\"GarageCond\"] = df[\"GarageCond\"].map(qual_dict).astype(int)\n",
    "\n",
    "    all_df[\"BsmtExposure\"] = df[\"BsmtExposure\"].map(\n",
    "        {None: 0, \"No\": 1, \"Mn\": 2, \"Av\": 3, \"Gd\": 4}).astype(int)\n",
    "\n",
    "    bsmt_fin_dict = {None: 0, \"Unf\": 1, \"LwQ\": 2, \"Rec\": 3, \"BLQ\": 4, \"ALQ\": 5, \"GLQ\": 6}\n",
    "    all_df[\"BsmtFinType1\"] = df[\"BsmtFinType1\"].map(bsmt_fin_dict).astype(int)\n",
    "    all_df[\"BsmtFinType2\"] = df[\"BsmtFinType2\"].map(bsmt_fin_dict).astype(int)\n",
    "\n",
    "    all_df[\"Functional\"] = df[\"Functional\"].map(\n",
    "        {None: 0, \"Sal\": 1, \"Sev\": 2, \"Maj2\": 3, \"Maj1\": 4, \n",
    "         \"Mod\": 5, \"Min2\": 6, \"Min1\": 7, \"Typ\": 8}).astype(int)\n",
    "\n",
    "    all_df[\"GarageFinish\"] = df[\"GarageFinish\"].map(\n",
    "        {None: 0, \"Unf\": 1, \"RFn\": 2, \"Fin\": 3}).astype(int)\n",
    "\n",
    "    all_df[\"Fence\"] = df[\"Fence\"].map(\n",
    "        {None: 0, \"MnWw\": 1, \"GdWo\": 2, \"MnPrv\": 3, \"GdPrv\": 4}).astype(int)\n",
    "\n",
    "    all_df[\"YearBuilt\"] = df[\"YearBuilt\"]\n",
    "    all_df[\"YearRemodAdd\"] = df[\"YearRemodAdd\"]\n",
    "\n",
    "    all_df[\"GarageYrBlt\"] = df[\"GarageYrBlt\"]\n",
    "    all_df[\"GarageYrBlt\"].fillna(0.0, inplace=True)\n",
    "\n",
    "    all_df[\"MoSold\"] = df[\"MoSold\"]\n",
    "    all_df[\"YrSold\"] = df[\"YrSold\"]\n",
    "    \n",
    "    all_df[\"LowQualFinSF\"] = df[\"LowQualFinSF\"]\n",
    "    all_df[\"MiscVal\"] = df[\"MiscVal\"]\n",
    "\n",
    "    all_df[\"PoolQC\"] = df[\"PoolQC\"].map(qual_dict).astype(int)\n",
    "\n",
    "    all_df[\"PoolArea\"] = df[\"PoolArea\"]\n",
    "    all_df[\"PoolArea\"].fillna(0, inplace=True)\n",
    "    \n",
    "    # Add categorical features as numbers too. It seems to help a bit.\n",
    "    all_df = factorize(df, all_df, \"MSSubClass\")\n",
    "    all_df = factorize(df, all_df, \"MSZoning\", \"RL\")\n",
    "    all_df = factorize(df, all_df, \"LotConfig\")\n",
    "    all_df = factorize(df, all_df, \"Neighborhood\")\n",
    "    all_df = factorize(df, all_df, \"Condition1\")\n",
    "    all_df = factorize(df, all_df, \"BldgType\")\n",
    "    all_df = factorize(df, all_df, \"HouseStyle\")\n",
    "    all_df = factorize(df, all_df, \"RoofStyle\")\n",
    "    all_df = factorize(df, all_df, \"Exterior1st\", \"Other\")\n",
    "    all_df = factorize(df, all_df, \"Exterior2nd\", \"Other\")\n",
    "    all_df = factorize(df, all_df, \"MasVnrType\", \"None\")\n",
    "    all_df = factorize(df, all_df, \"Foundation\")\n",
    "    all_df = factorize(df, all_df, \"SaleType\", \"Oth\")\n",
    "    all_df = factorize(df, all_df, \"SaleCondition\")\n",
    "\n",
    "    # IR2 and IR3 don't appear that often, so just make a distinction\n",
    "    # between regular and irregular.\n",
    "    all_df[\"IsRegularLotShape\"] = (df[\"LotShape\"] == \"Reg\") * 1\n",
    "\n",
    "    # Most properties are level; bin the other possibilities together\n",
    "    # as \"not level\".\n",
    "    all_df[\"IsLandLevel\"] = (df[\"LandContour\"] == \"Lvl\") * 1\n",
    "\n",
    "    # Most land slopes are gentle; treat the others as \"not gentle\".\n",
    "    all_df[\"IsLandSlopeGentle\"] = (df[\"LandSlope\"] == \"Gtl\") * 1\n",
    "\n",
    "    # Most properties use standard circuit breakers.\n",
    "    all_df[\"IsElectricalSBrkr\"] = (df[\"Electrical\"] == \"SBrkr\") * 1\n",
    "\n",
    "    # About 2/3rd have an attached garage.\n",
    "    all_df[\"IsGarageDetached\"] = (df[\"GarageType\"] == \"Detchd\") * 1\n",
    "\n",
    "    # Most have a paved drive. Treat dirt/gravel and partial pavement\n",
    "    # as \"not paved\".\n",
    "    all_df[\"IsPavedDrive\"] = (df[\"PavedDrive\"] == \"Y\") * 1\n",
    "\n",
    "    # The only interesting \"misc. feature\" is the presence of a shed.\n",
    "    all_df[\"HasShed\"] = (df[\"MiscFeature\"] == \"Shed\") * 1.  \n",
    "\n",
    "    # If YearRemodAdd != YearBuilt, then a remodeling took place at some point.\n",
    "    all_df[\"Remodeled\"] = (all_df[\"YearRemodAdd\"] != all_df[\"YearBuilt\"]) * 1\n",
    "    \n",
    "    # Did a remodeling happen in the year the house was sold?\n",
    "    all_df[\"RecentRemodel\"] = (all_df[\"YearRemodAdd\"] == all_df[\"YrSold\"]) * 1\n",
    "    \n",
    "    # Was this house sold in the year it was built?\n",
    "    all_df[\"VeryNewHouse\"] = (all_df[\"YearBuilt\"] == all_df[\"YrSold\"]) * 1\n",
    "\n",
    "    all_df[\"Has2ndFloor\"] = (all_df[\"2ndFlrSF\"] == 0) * 1\n",
    "    all_df[\"HasMasVnr\"] = (all_df[\"MasVnrArea\"] == 0) * 1\n",
    "    all_df[\"HasWoodDeck\"] = (all_df[\"WoodDeckSF\"] == 0) * 1\n",
    "    all_df[\"HasOpenPorch\"] = (all_df[\"OpenPorchSF\"] == 0) * 1\n",
    "    all_df[\"HasEnclosedPorch\"] = (all_df[\"EnclosedPorch\"] == 0) * 1\n",
    "    all_df[\"Has3SsnPorch\"] = (all_df[\"3SsnPorch\"] == 0) * 1\n",
    "    all_df[\"HasScreenPorch\"] = (all_df[\"ScreenPorch\"] == 0) * 1\n",
    "\n",
    "    # These features actually lower the score a little.\n",
    "    # all_df[\"HasBasement\"] = df[\"BsmtQual\"].isnull() * 1\n",
    "    # all_df[\"HasGarage\"] = df[\"GarageQual\"].isnull() * 1\n",
    "    # all_df[\"HasFireplace\"] = df[\"FireplaceQu\"].isnull() * 1\n",
    "    # all_df[\"HasFence\"] = df[\"Fence\"].isnull() * 1\n",
    "\n",
    "    # Months with the largest number of deals may be significant.\n",
    "    all_df[\"HighSeason\"] = df[\"MoSold\"].replace( \n",
    "        {1: 0, 2: 0, 3: 0, 4: 1, 5: 1, 6: 1, 7: 1, 8: 0, 9: 0, 10: 0, 11: 0, 12: 0})\n",
    "\n",
    "    all_df[\"NewerDwelling\"] = df[\"MSSubClass\"].replace(\n",
    "        {20: 1, 30: 0, 40: 0, 45: 0,50: 0, 60: 1, 70: 0, 75: 0, 80: 0, 85: 0,\n",
    "         90: 0, 120: 1, 150: 0, 160: 0, 180: 0, 190: 0})   \n",
    "    \n",
    "    all_df.loc[df.Neighborhood == 'NridgHt', \"Neighborhood_Good\"] = 1\n",
    "    all_df.loc[df.Neighborhood == 'Crawfor', \"Neighborhood_Good\"] = 1\n",
    "    all_df.loc[df.Neighborhood == 'StoneBr', \"Neighborhood_Good\"] = 1\n",
    "    all_df.loc[df.Neighborhood == 'Somerst', \"Neighborhood_Good\"] = 1\n",
    "    all_df.loc[df.Neighborhood == 'NoRidge', \"Neighborhood_Good\"] = 1\n",
    "    all_df[\"Neighborhood_Good\"].fillna(0, inplace=True)\n",
    "\n",
    "    all_df[\"SaleCondition_PriceDown\"] = df.SaleCondition.replace(\n",
    "        {'Abnorml': 1, 'Alloca': 1, 'AdjLand': 1, 'Family': 1, 'Normal': 0, 'Partial': 0})\n",
    "\n",
    "    # House completed before sale or not\n",
    "    all_df[\"BoughtOffPlan\"] = df.SaleCondition.replace(\n",
    "        {\"Abnorml\" : 0, \"Alloca\" : 0, \"AdjLand\" : 0, \"Family\" : 0, \"Normal\" : 0, \"Partial\" : 1})\n",
    "    \n",
    "    all_df[\"BadHeating\"] = df.HeatingQC.replace(\n",
    "        {'Ex': 0, 'Gd': 0, 'TA': 0, 'Fa': 1, 'Po': 1})\n",
    "\n",
    "    area_cols = ['LotFrontage', 'LotArea', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF',\n",
    "                 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'GrLivArea', 'GarageArea', 'WoodDeckSF', \n",
    "                 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'LowQualFinSF', 'PoolArea' ]\n",
    "    all_df[\"TotalArea\"] = all_df[area_cols].sum(axis=1)\n",
    "\n",
    "    all_df[\"TotalArea1st2nd\"] = all_df[\"1stFlrSF\"] + all_df[\"2ndFlrSF\"]\n",
    "\n",
    "    all_df[\"Age\"] = 2010 - all_df[\"YearBuilt\"]\n",
    "    all_df[\"TimeSinceSold\"] = 2010 - all_df[\"YrSold\"]\n",
    "\n",
    "    all_df[\"SeasonSold\"] = all_df[\"MoSold\"].map({12:0, 1:0, 2:0, 3:1, 4:1, 5:1, \n",
    "                                                  6:2, 7:2, 8:2, 9:3, 10:3, 11:3}).astype(int)\n",
    "    \n",
    "    all_df[\"YearsSinceRemodel\"] = all_df[\"YrSold\"] - all_df[\"YearRemodAdd\"]\n",
    "    \n",
    "    # Simplifications of existing features into bad/average/good.\n",
    "    all_df[\"SimplOverallQual\"] = all_df.OverallQual.replace(\n",
    "        {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2, 6 : 2, 7 : 3, 8 : 3, 9 : 3, 10 : 3})\n",
    "    all_df[\"SimplOverallCond\"] = all_df.OverallCond.replace(\n",
    "        {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2, 6 : 2, 7 : 3, 8 : 3, 9 : 3, 10 : 3})\n",
    "    all_df[\"SimplPoolQC\"] = all_df.PoolQC.replace(\n",
    "        {1 : 1, 2 : 1, 3 : 2, 4 : 2})\n",
    "    all_df[\"SimplGarageCond\"] = all_df.GarageCond.replace(\n",
    "        {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})\n",
    "    all_df[\"SimplGarageQual\"] = all_df.GarageQual.replace(\n",
    "        {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})\n",
    "    all_df[\"SimplFireplaceQu\"] = all_df.FireplaceQu.replace(\n",
    "        {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})\n",
    "    all_df[\"SimplFireplaceQu\"] = all_df.FireplaceQu.replace(\n",
    "        {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})\n",
    "    all_df[\"SimplFunctional\"] = all_df.Functional.replace(\n",
    "        {1 : 1, 2 : 1, 3 : 2, 4 : 2, 5 : 3, 6 : 3, 7 : 3, 8 : 4})\n",
    "    all_df[\"SimplKitchenQual\"] = all_df.KitchenQual.replace(\n",
    "        {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})\n",
    "    all_df[\"SimplHeatingQC\"] = all_df.HeatingQC.replace(\n",
    "        {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})\n",
    "    all_df[\"SimplBsmtFinType1\"] = all_df.BsmtFinType1.replace(\n",
    "        {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2, 6 : 2})\n",
    "    all_df[\"SimplBsmtFinType2\"] = all_df.BsmtFinType2.replace(\n",
    "        {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2, 6 : 2})\n",
    "    all_df[\"SimplBsmtCond\"] = all_df.BsmtCond.replace(\n",
    "        {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})\n",
    "    all_df[\"SimplBsmtQual\"] = all_df.BsmtQual.replace(\n",
    "        {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})\n",
    "    all_df[\"SimplExterCond\"] = all_df.ExterCond.replace(\n",
    "        {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})\n",
    "    all_df[\"SimplExterQual\"] = all_df.ExterQual.replace(\n",
    "        {1 : 1, 2 : 1, 3 : 1, 4 : 2, 5 : 2})\n",
    "            \n",
    "    # Bin by neighborhood (a little arbitrarily). Values were computed by: \n",
    "    # train_df[\"SalePrice\"].groupby(train_df[\"Neighborhood\"]).median().sort_values()\n",
    "    neighborhood_map = {\n",
    "        \"MeadowV\" : 0,  #  88000\n",
    "        \"IDOTRR\" : 1,   # 103000\n",
    "        \"BrDale\" : 1,   # 106000\n",
    "        \"OldTown\" : 1,  # 119000\n",
    "        \"Edwards\" : 1,  # 119500\n",
    "        \"BrkSide\" : 1,  # 124300\n",
    "        \"Sawyer\" : 1,   # 135000\n",
    "        \"Blueste\" : 1,  # 137500\n",
    "        \"SWISU\" : 2,    # 139500\n",
    "        \"NAmes\" : 2,    # 140000\n",
    "        \"NPkVill\" : 2,  # 146000\n",
    "        \"Mitchel\" : 2,  # 153500\n",
    "        \"SawyerW\" : 2,  # 179900\n",
    "        \"Gilbert\" : 2,  # 181000\n",
    "        \"NWAmes\" : 2,   # 182900\n",
    "        \"Blmngtn\" : 2,  # 191000\n",
    "        \"CollgCr\" : 2,  # 197200\n",
    "        \"ClearCr\" : 3,  # 200250\n",
    "        \"Crawfor\" : 3,  # 200624\n",
    "        \"Veenker\" : 3,  # 218000\n",
    "        \"Somerst\" : 3,  # 225500\n",
    "        \"Timber\" : 3,   # 228475\n",
    "        \"StoneBr\" : 4,  # 278000\n",
    "        \"NoRidge\" : 4,  # 290000\n",
    "        \"NridgHt\" : 4,  # 315000\n",
    "    }\n",
    "\n",
    "    all_df[\"NeighborhoodBin\"] = df[\"Neighborhood\"].map(neighborhood_map)\n",
    "    return all_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1456, 111)\n",
      "(1459, 111)\n"
     ]
    }
   ],
   "source": [
    "train_df_munged = munge(train_df)\n",
    "test_df_munged = munge(test_df)\n",
    "\n",
    "print(train_df_munged.shape)\n",
    "print(test_df_munged.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Copy NeighborhoodBin into a temporary DataFrame because we want to use the\n",
    "# unscaled version later on (to one-hot encode it). \n",
    "neighborhood_bin_train = pd.DataFrame(index = train_df.index)\n",
    "neighborhood_bin_train[\"NeighborhoodBin\"] = train_df_munged[\"NeighborhoodBin\"]\n",
    "neighborhood_bin_test = pd.DataFrame(index = test_df.index)\n",
    "neighborhood_bin_test[\"NeighborhoodBin\"] = test_df_munged[\"NeighborhoodBin\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "################################################################################\n",
    "numeric_features = train_df_munged.dtypes[train_df_munged.dtypes != \"object\"].index\n",
    "\n",
    "# Transform the skewed numeric features by taking log(feature + 1).\n",
    "# This will make the features more normal.\n",
    "from scipy.stats import skew\n",
    "\n",
    "skewed = train_df_munged[numeric_features].apply(lambda x: skew(x.dropna().astype(float)))\n",
    "skewed = skewed[skewed > 0.75]\n",
    "skewed = skewed.index\n",
    "\n",
    "# use numpy log1p to normalize the data log1p = log(1+x)\n",
    "train_df_munged[skewed] = np.log1p(train_df_munged[skewed])\n",
    "test_df_munged[skewed] = np.log1p(test_df_munged[skewed])\n",
    "\n",
    "# Additional processing: scale the data.   \n",
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(train_df_munged[numeric_features])\n",
    "\n",
    "scaled = scaler.transform(train_df_munged[numeric_features])\n",
    "for i, col in enumerate(numeric_features):\n",
    "    train_df_munged[col] = scaled[:, i]\n",
    "\n",
    "scaled = scaler.transform(test_df_munged[numeric_features])\n",
    "for i, col in enumerate(numeric_features):\n",
    "    test_df_munged[col] = scaled[:, i]\n",
    "################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Convert categorical features using one-hot encoding.\n",
    "def onehot(onehot_df, df, column_name, fill_na, drop_name):\n",
    "    onehot_df[column_name] = df[column_name]\n",
    "    if fill_na is not None:\n",
    "        onehot_df[column_name].fillna(fill_na, inplace=True)\n",
    "\n",
    "    dummies = pd.get_dummies(onehot_df[column_name], prefix=\"_\" + column_name)\n",
    "    \n",
    "    # Dropping one of the columns actually made the results slightly worse.\n",
    "    # if drop_name is not None:\n",
    "    #     dummies.drop([\"_\" + column_name + \"_\" + drop_name], axis=1, inplace=True)\n",
    "\n",
    "    onehot_df = onehot_df.join(dummies)\n",
    "    onehot_df = onehot_df.drop([column_name], axis=1)\n",
    "    return onehot_df\n",
    "\n",
    "def munge_onehot(df):\n",
    "    onehot_df = pd.DataFrame(index = df.index)\n",
    "\n",
    "    onehot_df = onehot(onehot_df, df, \"MSSubClass\", None, \"40\")\n",
    "    onehot_df = onehot(onehot_df, df, \"MSZoning\", \"RL\", \"RH\")\n",
    "    onehot_df = onehot(onehot_df, df, \"LotConfig\", None, \"FR3\")\n",
    "    onehot_df = onehot(onehot_df, df, \"Neighborhood\", None, \"OldTown\")\n",
    "    onehot_df = onehot(onehot_df, df, \"Condition1\", None, \"RRNe\")\n",
    "    onehot_df = onehot(onehot_df, df, \"BldgType\", None, \"2fmCon\")\n",
    "    onehot_df = onehot(onehot_df, df, \"HouseStyle\", None, \"1.5Unf\")\n",
    "    onehot_df = onehot(onehot_df, df, \"RoofStyle\", None, \"Shed\")\n",
    "    onehot_df = onehot(onehot_df, df, \"Exterior1st\", \"VinylSd\", \"CBlock\")\n",
    "    onehot_df = onehot(onehot_df, df, \"Exterior2nd\", \"VinylSd\", \"CBlock\")\n",
    "    onehot_df = onehot(onehot_df, df, \"Foundation\", None, \"Wood\")\n",
    "    onehot_df = onehot(onehot_df, df, \"SaleType\", \"WD\", \"Oth\")\n",
    "    onehot_df = onehot(onehot_df, df, \"SaleCondition\", \"Normal\", \"AdjLand\")\n",
    "\n",
    "    # Fill in missing MasVnrType for rows that do have a MasVnrArea.\n",
    "    temp_df = df[[\"MasVnrType\", \"MasVnrArea\"]].copy()\n",
    "    idx = (df[\"MasVnrArea\"] != 0) & ((df[\"MasVnrType\"] == \"None\") | (df[\"MasVnrType\"].isnull()))\n",
    "    temp_df.loc[idx, \"MasVnrType\"] = \"BrkFace\"\n",
    "    onehot_df = onehot(onehot_df, temp_df, \"MasVnrType\", \"None\", \"BrkCmn\")\n",
    "\n",
    "    # Also add the booleans from calc_df as dummy variables.\n",
    "    onehot_df = onehot(onehot_df, df, \"LotShape\", None, \"IR3\")\n",
    "    onehot_df = onehot(onehot_df, df, \"LandContour\", None, \"Low\")\n",
    "    onehot_df = onehot(onehot_df, df, \"LandSlope\", None, \"Sev\")\n",
    "    onehot_df = onehot(onehot_df, df, \"Electrical\", \"SBrkr\", \"FuseP\")\n",
    "    onehot_df = onehot(onehot_df, df, \"GarageType\", \"None\", \"CarPort\")\n",
    "    onehot_df = onehot(onehot_df, df, \"PavedDrive\", None, \"P\")\n",
    "    onehot_df = onehot(onehot_df, df, \"MiscFeature\", \"None\", \"Othr\")\n",
    "\n",
    "    # Features we can probably ignore (but want to include anyway to see\n",
    "    # if they make any positive difference).\n",
    "    # Definitely ignoring Utilities: all records are \"AllPub\", except for\n",
    "    # one \"NoSeWa\" in the train set and 2 NA in the test set.\n",
    "    onehot_df = onehot(onehot_df, df, \"Street\", None, \"Grvl\")\n",
    "    onehot_df = onehot(onehot_df, df, \"Alley\", \"None\", \"Grvl\")\n",
    "    onehot_df = onehot(onehot_df, df, \"Condition2\", None, \"PosA\")\n",
    "    onehot_df = onehot(onehot_df, df, \"RoofMatl\", None, \"WdShake\")\n",
    "    onehot_df = onehot(onehot_df, df, \"Heating\", None, \"Wall\")\n",
    "\n",
    "    # I have these as numerical variables too.\n",
    "    onehot_df = onehot(onehot_df, df, \"ExterQual\", \"None\", \"Ex\")\n",
    "    onehot_df = onehot(onehot_df, df, \"ExterCond\", \"None\", \"Ex\")\n",
    "    onehot_df = onehot(onehot_df, df, \"BsmtQual\", \"None\", \"Ex\")\n",
    "    onehot_df = onehot(onehot_df, df, \"BsmtCond\", \"None\", \"Ex\")\n",
    "    onehot_df = onehot(onehot_df, df, \"HeatingQC\", \"None\", \"Ex\")\n",
    "    onehot_df = onehot(onehot_df, df, \"KitchenQual\", \"TA\", \"Ex\")\n",
    "    onehot_df = onehot(onehot_df, df, \"FireplaceQu\", \"None\", \"Ex\")\n",
    "    onehot_df = onehot(onehot_df, df, \"GarageQual\", \"None\", \"Ex\")\n",
    "    onehot_df = onehot(onehot_df, df, \"GarageCond\", \"None\", \"Ex\")\n",
    "    onehot_df = onehot(onehot_df, df, \"PoolQC\", \"None\", \"Ex\")\n",
    "    onehot_df = onehot(onehot_df, df, \"BsmtExposure\", \"None\", \"Gd\")\n",
    "    onehot_df = onehot(onehot_df, df, \"BsmtFinType1\", \"None\", \"GLQ\")\n",
    "    onehot_df = onehot(onehot_df, df, \"BsmtFinType2\", \"None\", \"GLQ\")\n",
    "    onehot_df = onehot(onehot_df, df, \"Functional\", \"Typ\", \"Typ\")\n",
    "    onehot_df = onehot(onehot_df, df, \"GarageFinish\", \"None\", \"Fin\")\n",
    "    onehot_df = onehot(onehot_df, df, \"Fence\", \"None\", \"MnPrv\")\n",
    "    onehot_df = onehot(onehot_df, df, \"MoSold\", None, None)\n",
    "    \n",
    "    # Divide up the years between 1871 and 2010 in slices of 20 years.\n",
    "    year_map = pd.concat(pd.Series(\"YearBin\" + str(i+1), index=range(1871+i*20,1891+i*20)) for i in range(0, 7))\n",
    "\n",
    "    yearbin_df = pd.DataFrame(index = df.index)\n",
    "    yearbin_df[\"GarageYrBltBin\"] = df.GarageYrBlt.map(year_map)\n",
    "    yearbin_df[\"GarageYrBltBin\"].fillna(\"NoGarage\", inplace=True)\n",
    "\n",
    "    yearbin_df[\"YearBuiltBin\"] = df.YearBuilt.map(year_map)\n",
    "    yearbin_df[\"YearRemodAddBin\"] = df.YearRemodAdd.map(year_map)\n",
    "    \n",
    "    onehot_df = onehot(onehot_df, yearbin_df, \"GarageYrBltBin\", None, None)\n",
    "    onehot_df = onehot(onehot_df, yearbin_df, \"YearBuiltBin\", None, None)\n",
    "    onehot_df = onehot(onehot_df, yearbin_df, \"YearRemodAddBin\", None, None)\n",
    "\n",
    "    return onehot_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Add the one-hot encoded categorical features.\n",
    "onehot_df = munge_onehot(train_df)\n",
    "onehot_df = onehot(onehot_df, neighborhood_bin_train, \"NeighborhoodBin\", None, None)\n",
    "train_df_munged = train_df_munged.join(onehot_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Training set size:', (1456, 403))\n",
      "('Test set size:', (1459, 403))\n"
     ]
    }
   ],
   "source": [
    "# These onehot columns are missing in the test data, so drop them from the\n",
    "# training data or we might overfit on them.\n",
    "drop_cols = [\n",
    "                \"_Exterior1st_ImStucc\", \"_Exterior1st_Stone\",\n",
    "                \"_Exterior2nd_Other\",\"_HouseStyle_2.5Fin\", \n",
    "            \n",
    "                \"_RoofMatl_Membran\", \"_RoofMatl_Metal\", \"_RoofMatl_Roll\",\n",
    "                \"_Condition2_RRAe\", \"_Condition2_RRAn\", \"_Condition2_RRNn\",\n",
    "                \"_Heating_Floor\", \"_Heating_OthW\",\n",
    "\n",
    "                \"_Electrical_Mix\", \n",
    "                \"_MiscFeature_TenC\",\n",
    "                \"_GarageQual_Ex\", \"_PoolQC_Fa\"\n",
    "            ]\n",
    "\n",
    "train_df_munged.drop(drop_cols, axis=1, inplace=True)\n",
    "\n",
    "onehot_df = munge_onehot(test_df)\n",
    "onehot_df = onehot(onehot_df, neighborhood_bin_test, \"NeighborhoodBin\", None, None)\n",
    "test_df_munged = test_df_munged.join(onehot_df)\n",
    "\n",
    "# This column is missing in the training data. There is only one example with\n",
    "# this value in the test set. So just drop it.\n",
    "test_df_munged.drop([\"_MSSubClass_150\"], axis=1, inplace=True)\n",
    "\n",
    "# Drop these columns. They are either not very helpful or they cause overfitting.\n",
    "drop_cols = [\n",
    "    \"_Condition2_PosN\",    # only two are not zero\n",
    "    \"_MSZoning_C (all)\",\n",
    "    \"_MSSubClass_160\",\n",
    "]\n",
    "train_df_munged.drop(drop_cols, axis=1, inplace=True)\n",
    "test_df_munged.drop(drop_cols, axis=1, inplace=True)\n",
    "\n",
    "################################################################################\n",
    "\n",
    "# We take the log here because the error metric is between the log of the\n",
    "# SalePrice and the log of the predicted price. That does mean we need to \n",
    "# exp() the prediction to get an actual sale price.\n",
    "label_df = pd.DataFrame(index = train_df_munged.index, columns=[\"SalePrice\"])\n",
    "label_df[\"SalePrice\"] = np.log1p(train_df[\"SalePrice\"])\n",
    "\n",
    "print(\"Training set size:\", train_df_munged.shape)\n",
    "print(\"Test set size:\", test_df_munged.shape)\n",
    "\n",
    "################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/conebeam/anaconda2/lib/python2.7/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.model_selection import KFold, cross_val_score\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.linear_model import LinearRegression, RidgeCV, LassoCV, Lasso, Ridge\n",
    "from sklearn.ensemble import RandomForestRegressor, AdaBoostRegressor, GradientBoostingRegressor, BaggingRegressor\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "# The error metric: RMSE on the log of the sale prices.\n",
    "def rmse(y_true, y_pred):\n",
    "    return np.sqrt(mean_squared_error(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# From here starts the stacking training process\n",
    "# first of all, we need to define the base learner and meta learner\n",
    "# Base learner: GBM, RF, DT, Lasso, Ridge, MLP, xgboost\n",
    "# meta learner: GBM\n",
    "\n",
    "# number of CV folds (to generate level-one data for stacking)\n",
    "nfolds = 5\n",
    "\n",
    "# Learn each base learner using cross validatoin:\n",
    "# 1.Lasso \n",
    "bs_lasso = Lasso(alpha=0.0046, max_iter=10000)\n",
    "bs_lasso.fit(train_df_munged, label_df)\n",
    "#print(lasso.alpha_)\n",
    "# Best alpha is 0.0046\n",
    "bs_lass_pred = bs_lasso.predict(train_df_munged)\n",
    "real_lass_pred = bs_lasso.predict(test_df_munged)\n",
    "\n",
    "\n",
    "# 2.Ridge \n",
    "# alphas = np.arange(0,1,0.001)\n",
    "bs_ridge = Ridge(alpha=0.999, max_iter=10000)\n",
    "bs_ridge.fit(train_df_munged, label_df)\n",
    "# print(ridge.alpha_)\n",
    "# Best alpha is 0.999\n",
    "bs_ridge_pred = bs_ridge.predict(train_df_munged)\n",
    "real_ridge_pred = bs_ridge.predict(test_df_munged)\n",
    "\n",
    "\n",
    "# 3.Xgboost\n",
    "bs_xgb = xgb.XGBRegressor(\n",
    "                 colsample_bytree=0.2,\n",
    "                 gamma=0.0,\n",
    "                 learning_rate=0.01,\n",
    "                 max_depth=4,\n",
    "                 min_child_weight=1.5,\n",
    "                 n_estimators=7200,                                                                  \n",
    "                 reg_alpha=0.9,\n",
    "                 reg_lambda=0.6,\n",
    "                 subsample=0.2,\n",
    "                 seed=42,\n",
    "                 silent=1)\n",
    "\n",
    "bs_xgb.fit(train_df_munged, label_df)\n",
    "bs_xgb_pred = bs_xgb.predict(train_df_munged)\n",
    "real_xgb_pred = bs_xgb.predict(test_df_munged)\n",
    "\n",
    "\n",
    "# 4.RF\n",
    "bs_rf = RandomForestRegressor(n_estimators=500, n_jobs=-1)\n",
    "bs_rf.fit(train_df_munged, label_df)\n",
    "bs_rf_pred = bs_rf.predict(train_df_munged)\n",
    "real_rf_pred = bs_rf.predict(test_df_munged)\n",
    "\n",
    "\n",
    "# 5.GB\n",
    "bs_gb = GradientBoostingRegressor(n_estimators=500)\n",
    "bs_gb.fit(train_df_munged, label_df)\n",
    "bs_gb_pred = bs_gb.predict(train_df_munged)\n",
    "real_gb_pred = bs_gb.predict(test_df_munged)\n",
    "\n",
    "\n",
    "# 6.MLP\n",
    "bs_mlp = MLPRegressor()\n",
    "bs_mlp.fit(train_df_munged, label_df)\n",
    "bs_mlp_pred = bs_mlp.predict(train_df_munged)\n",
    "real_mlp_pred = bs_mlp.predict(test_df_munged)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# From here starts the stacking training process\n",
    "# First of all, we need to define the base learner and meta learner\n",
    "# Base learner candidates: GBM, RF, DT, Lasso, Ridge, MLP, xgboost\n",
    "# meta learner candidates: GBM, Lasso, xgboost, NN\n",
    "\n",
    "\n",
    "np.random.seed(42) #for reproduction\n",
    "\n",
    "# print (\"Preparing models.\")\n",
    "\n",
    "n_trees = 1000\n",
    "n_folds = 5 # 10-fold cross-validation\n",
    "\n",
    "# level 0: base learners\n",
    "clfs = [\n",
    "        Lasso(alpha = 0.00046, max_iter=50000),\n",
    "        Ridge(alpha = 0.99, max_iter=50000),\n",
    "        MLPRegressor(),\n",
    "        RandomForestRegressor(n_estimators = n_trees, n_jobs=-1),\n",
    "        GradientBoostingRegressor(n_estimators = n_trees),\n",
    "        xgb.XGBRegressor(gamma=0.0, learning_rate=0.01, max_depth=4, min_child_weight=1.5,\n",
    "                         n_estimators=7200, reg_alpha=0.9, reg_lambda=0.6, subsample=0.5, seed=42, silent=1)\n",
    "        ]\n",
    "\n",
    "# Ready for cross validation\n",
    "skf = KFold(n_splits = n_folds)\n",
    "\n",
    "# Pre-allocate the data\n",
    "trainsize = len(label_df)\n",
    "testsize = len(test_df_munged)\n",
    "blend_train = np.zeros((trainsize, len(clfs))) # Number of training data x Number of regressors\n",
    "blend_test = np.zeros((testsize, len(clfs))) # Number of testing data x Number of regressors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training regressor 0, Lasso(alpha=0.00046, copy_X=True, fit_intercept=True, max_iter=50000,\n",
      "   normalize=False, positive=False, precompute=False, random_state=None,\n",
      "   selection='cyclic', tol=0.0001, warm_start=False)\n",
      "Fold [0] norm. RMSE = 0.10488\n",
      "Fold [1] norm. RMSE = 0.11302\n",
      "Fold [2] norm. RMSE = 0.11821\n",
      "Fold [3] norm. RMSE = 0.10225\n",
      "Fold [4] norm. RMSE = 0.11225\n",
      "\n",
      "Training regressor 1, Ridge(alpha=0.99, copy_X=True, fit_intercept=True, max_iter=50000,\n",
      "   normalize=False, random_state=None, solver='auto', tol=0.001)\n",
      "Fold [0] norm. RMSE = 0.11651\n",
      "Fold [1] norm. RMSE = 0.12456\n",
      "Fold [2] norm. RMSE = 0.13540\n",
      "Fold [3] norm. RMSE = 0.10822\n",
      "Fold [4] norm. RMSE = 0.11852\n",
      "\n",
      "Training regressor 2, MLPRegressor(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,\n",
      "       beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
      "       hidden_layer_sizes=(100,), learning_rate='constant',\n",
      "       learning_rate_init=0.001, max_iter=200, momentum=0.9,\n",
      "       nesterovs_momentum=True, power_t=0.5, random_state=None,\n",
      "       shuffle=True, solver='adam', tol=0.0001, validation_fraction=0.1,\n",
      "       verbose=False, warm_start=False)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/conebeam/anaconda2/lib/python2.7/site-packages/sklearn/neural_network/multilayer_perceptron.py:1266: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold [0] norm. RMSE = 0.23747\n",
      "Fold [1] norm. RMSE = 0.88519\n",
      "Fold [2] norm. RMSE = 0.82974\n",
      "Fold [3] norm. RMSE = 0.76048\n",
      "Fold [4] norm. RMSE = 0.66050\n",
      "\n",
      "Training regressor 3, RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
      "           max_features='auto', max_leaf_nodes=None,\n",
      "           min_impurity_split=1e-07, min_samples_leaf=1,\n",
      "           min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
      "           n_estimators=1000, n_jobs=-1, oob_score=False,\n",
      "           random_state=None, verbose=0, warm_start=False)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/conebeam/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:19: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold [0] norm. RMSE = 0.12744\n",
      "Fold [1] norm. RMSE = 0.13673\n",
      "Fold [2] norm. RMSE = 0.14291\n",
      "Fold [3] norm. RMSE = 0.12999\n",
      "Fold [4] norm. RMSE = 0.13444\n",
      "\n",
      "Training regressor 4, GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,\n",
      "             learning_rate=0.1, loss='ls', max_depth=3, max_features=None,\n",
      "             max_leaf_nodes=None, min_impurity_split=1e-07,\n",
      "             min_samples_leaf=1, min_samples_split=2,\n",
      "             min_weight_fraction_leaf=0.0, n_estimators=1000,\n",
      "             presort='auto', random_state=None, subsample=1.0, verbose=0,\n",
      "             warm_start=False)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/conebeam/anaconda2/lib/python2.7/site-packages/sklearn/utils/validation.py:526: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold [0] norm. RMSE = 0.11305\n",
      "Fold [1] norm. RMSE = 0.12760\n",
      "Fold [2] norm. RMSE = 0.12684\n",
      "Fold [3] norm. RMSE = 0.10984\n",
      "Fold [4] norm. RMSE = 0.12372\n",
      "\n",
      "Training regressor 5, XGBRegressor(base_score=0.5, colsample_bylevel=1, colsample_bytree=1,\n",
      "       gamma=0.0, learning_rate=0.01, max_delta_step=0, max_depth=4,\n",
      "       min_child_weight=1.5, missing=None, n_estimators=7200, nthread=-1,\n",
      "       objective='reg:linear', reg_alpha=0.9, reg_lambda=0.6,\n",
      "       scale_pos_weight=1, seed=42, silent=1, subsample=0.5)\n",
      "Fold [0] norm. RMSE = 0.10923\n",
      "Fold [1] norm. RMSE = 0.11770\n",
      "Fold [2] norm. RMSE = 0.12185\n",
      "Fold [3] norm. RMSE = 0.10751\n",
      "Fold [4] norm. RMSE = 0.11603\n"
     ]
    }
   ],
   "source": [
    "cv_results = np.zeros((len(clfs), n_folds))  # Number of regressors x Number of folds\n",
    "# print('size of cv_result is', np.shape(cv_results))\n",
    "\n",
    "# For each regressor, we train the number of fold times (=len(skf))\n",
    "for j, clf in enumerate(clfs):\n",
    "    print ('\\nTraining regressor %s, %s' % (j, clf))\n",
    "    \n",
    "    # Number of testing data x Number of folds , we will take the mean of the predictions later\n",
    "    blend_test_j = np.zeros((testsize, n_folds))\n",
    "    \n",
    "    for i, (train_index, cv_index) in enumerate(skf.split(train_df_munged, label_df)):\n",
    "\n",
    "        # This is the training and validation set\n",
    "        X_train = train_df_munged.iloc[train_index]\n",
    "        Y_train = label_df.iloc[train_index]\n",
    "        X_cv = train_df_munged.iloc[cv_index]\n",
    "        Y_cv = label_df.iloc[cv_index]\n",
    "            \n",
    "        clf.fit(X_train, Y_train)\n",
    "\n",
    "        # This output will be the basis for our blended regressor to train against,\n",
    "        # which is also the output of our regression\n",
    "        one_result = clf.predict(X_cv)\n",
    "        # print (np.shape(one_result))\n",
    "        blend_train[cv_index, j] = one_result.reshape((one_result.shape[0],))\n",
    "        score_rmse = rmse(Y_cv, one_result)    \n",
    "        print ('Fold [%s] norm. RMSE = %0.5f' % (i, score_rmse)) \n",
    "        one_predict = clf.predict(test_df_munged)\n",
    "        blend_test_j[:, i] = one_predict.reshape((one_predict.shape[0],))\n",
    "    # Take the mean of the predictions of the cross validation set\n",
    "    blend_test[:, j] = blend_test_j.mean(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold [0] norm. RMSE = 0.11414\n",
      "Fold [1] norm. RMSE = 0.11514\n",
      "Fold [2] norm. RMSE = 0.11753\n",
      "Fold [3] norm. RMSE = 0.10586\n",
      "Fold [4] norm. RMSE = 0.11318\n"
     ]
    }
   ],
   "source": [
    "# Level one: meta learner, let's try xgb first\n",
    "ml_xgb = xgb.XGBRegressor(gamma=0.0, learning_rate=0.01, max_depth=4, min_child_weight=1.5,\n",
    "                         n_estimators=7200, reg_alpha=0.9, reg_lambda=0.6, subsample=0.5, seed=42, silent=1)\n",
    "\n",
    "   \n",
    "# Number of testing data x Number of folds , we will take the mean of the predictions later\n",
    "test_j = np.zeros((testsize, n_folds))\n",
    "    \n",
    "# Ready for cross validation\n",
    "kf = KFold(n_splits = n_folds)\n",
    "\n",
    "for i, (train_index, cv_index) in enumerate(kf.split(blend_train, label_df)):\n",
    "\n",
    "    # This is the training and validation set\n",
    "    X_train = blend_train[train_index]\n",
    "    Y_train = label_df.iloc[train_index]\n",
    "    X_cv = blend_train[cv_index]\n",
    "    Y_cv = label_df.iloc[cv_index]\n",
    "            \n",
    "    ml_xgb.fit(X_train, Y_train)\n",
    "\n",
    "    # This output will be the basis for our blended regressor to train against,\n",
    "    # which is also the output of our regression\n",
    "    one_result = ml_xgb.predict(X_cv)\n",
    "    # print (np.shape(one_result))\n",
    "    one_result = one_result.reshape((one_result.shape[0],))\n",
    "    score_rmse = rmse(Y_cv, one_result)    \n",
    "    print ('Fold [%s] norm. RMSE = %0.5f' % (i, score_rmse)) \n",
    "    one_predict = ml_xgb.predict(blend_test)\n",
    "    test_j[:, i] = one_predict.reshape((one_predict.shape[0],))\n",
    "        \n",
    "# Take the mean of the predictions of the cross validation set\n",
    "preds = test_j.mean(1)\n",
    "\n",
    "preds = np.expm1(preds)\n",
    "\n",
    "pred_df = pd.DataFrame(preds, index=test_df[\"Id\"], columns=[\"SalePrice\"])\n",
    "pred_df.to_csv('stacking_test_xgb.csv', header=True, index_label='Id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold [0] norm. RMSE = 0.11122\n",
      "Fold [1] norm. RMSE = 0.09412\n",
      "Fold [2] norm. RMSE = 0.09809\n",
      "Fold [3] norm. RMSE = 0.11774\n",
      "Fold [4] norm. RMSE = 0.13747\n",
      "Fold [5] norm. RMSE = 0.09522\n",
      "Fold [6] norm. RMSE = 0.11075\n",
      "Fold [7] norm. RMSE = 0.09153\n",
      "Fold [8] norm. RMSE = 0.09041\n",
      "Fold [9] norm. RMSE = 0.12367\n"
     ]
    }
   ],
   "source": [
    "# Level one: meta learner, let's try lasso next\n",
    "\n",
    "#Use LassoCV to find best alpha\n",
    "#alphas = [0.0001, 0.005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0, 50.0, 100.0]\n",
    "#mlf = LassoCV(alphas = alphas, cv=10)\n",
    "#mlf.fit(blend_train, label_df)\n",
    "#print(mlf.alpha_)\n",
    "\n",
    "mlf = Lasso(alpha=0.0001, max_iter=5000)\n",
    "nfolds = 10\n",
    "# Number of testing data x Number of folds , we will take the mean of the predictions later\n",
    "test_j = np.zeros((testsize, nfolds))\n",
    "    \n",
    "# Ready for cross validation\n",
    "kf = KFold(n_splits = nfolds)\n",
    "\n",
    "for i, (train_index, cv_index) in enumerate(kf.split(blend_train, label_df)):\n",
    "\n",
    "    # This is the training and validation set\n",
    "    X_train = blend_train[train_index]\n",
    "    Y_train = label_df.iloc[train_index]\n",
    "    X_cv = blend_train[cv_index]\n",
    "    Y_cv = label_df.iloc[cv_index]\n",
    "            \n",
    "    mlf.fit(X_train, Y_train)\n",
    "\n",
    "    # This output will be the basis for our blended regressor to train against,\n",
    "    # which is also the output of our regression\n",
    "    one_result = mlf.predict(X_cv)\n",
    "    # print (np.shape(one_result))\n",
    "    one_result = one_result.reshape((one_result.shape[0],))\n",
    "    score_rmse = rmse(Y_cv, one_result)    \n",
    "    print ('Fold [%s] norm. RMSE = %0.5f' % (i, score_rmse)) \n",
    "    one_predict = mlf.predict(blend_test)\n",
    "    test_j[:, i] = one_predict.reshape((one_predict.shape[0],))\n",
    "        \n",
    "# Take the mean of the predictions of the cross validation set\n",
    "preds = test_j.mean(1)\n",
    "\n",
    "preds = np.expm1(preds)\n",
    "\n",
    "pred_df = pd.DataFrame(preds, index=test_df[\"Id\"], columns=[\"SalePrice\"])\n",
    "pred_df.to_csv('stacking_test_lasso.csv', header=True, index_label='Id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Use LassoCV to find best alpha\n",
    "alphas = [0.0001, 0.005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0, 50.0, 100.0]\n",
    "mlf = LassoCV(alphas = alphas, cv=10)\n",
    "mlf.fit(blend_train, label_df)\n",
    "\n",
    "preds = mlf.predict(blend_test)\n",
    "preds = np.expm1(preds)\n",
    "\n",
    "pred_df = pd.DataFrame(preds, index=test_df[\"Id\"], columns=[\"SalePrice\"])\n",
    "pred_df.to_csv('stacking_test_lasso.csv', header=True, index_label='Id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# keras input are numpy arrays, need to convert label_df from pandas series to numpy arrays\n",
    "blend_train = blend_train.astype('float32')\n",
    "label_df = np.array(label_df)\n",
    "label_df = label_df.astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "1456/1456 [==============================] - 0s - loss: 33.4477 - acc: 0.0000e+00      \n",
      "Epoch 2/100\n",
      "1456/1456 [==============================] - 0s - loss: 3.5791 - acc: 0.0000e+00      \n",
      "Epoch 3/100\n",
      "1456/1456 [==============================] - 0s - loss: 1.0612 - acc: 0.0000e+00     \n",
      "Epoch 4/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.2793 - acc: 0.0000e+00     \n",
      "Epoch 5/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0781 - acc: 0.0000e+00     \n",
      "Epoch 6/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0365 - acc: 0.0000e+00     \n",
      "Epoch 7/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0297 - acc: 0.0000e+00     \n",
      "Epoch 8/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0249 - acc: 0.0000e+00     \n",
      "Epoch 9/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0236 - acc: 0.0000e+00     \n",
      "Epoch 10/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0231 - acc: 0.0000e+00     \n",
      "Epoch 11/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0227 - acc: 0.0000e+00     \n",
      "Epoch 12/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0225 - acc: 0.0000e+00     \n",
      "Epoch 13/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0223 - acc: 0.0000e+00     \n",
      "Epoch 14/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0228 - acc: 0.0000e+00     \n",
      "Epoch 15/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0223 - acc: 0.0000e+00     \n",
      "Epoch 16/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0218 - acc: 0.0000e+00     \n",
      "Epoch 17/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0216 - acc: 0.0000e+00     \n",
      "Epoch 18/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0216 - acc: 0.0000e+00     \n",
      "Epoch 19/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0213 - acc: 0.0000e+00     \n",
      "Epoch 20/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0206 - acc: 0.0000e+00     \n",
      "Epoch 21/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0203 - acc: 0.0000e+00     \n",
      "Epoch 22/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0201 - acc: 0.0000e+00     \n",
      "Epoch 23/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0199 - acc: 0.0000e+00     \n",
      "Epoch 24/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0198 - acc: 0.0000e+00     \n",
      "Epoch 25/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0196 - acc: 0.0000e+00     \n",
      "Epoch 26/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0197 - acc: 0.0000e+00     \n",
      "Epoch 27/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0194 - acc: 0.0000e+00     \n",
      "Epoch 28/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0192 - acc: 0.0000e+00     \n",
      "Epoch 29/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0188 - acc: 0.0000e+00     \n",
      "Epoch 30/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0189 - acc: 0.0000e+00     \n",
      "Epoch 31/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0184 - acc: 0.0000e+00     \n",
      "Epoch 32/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0180 - acc: 0.0000e+00     \n",
      "Epoch 33/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0176 - acc: 0.0000e+00     \n",
      "Epoch 34/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0173 - acc: 0.0000e+00     \n",
      "Epoch 35/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0171 - acc: 0.0000e+00     \n",
      "Epoch 36/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0171 - acc: 0.0000e+00     \n",
      "Epoch 37/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0168 - acc: 0.0000e+00     \n",
      "Epoch 38/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0166 - acc: 0.0000e+00     \n",
      "Epoch 39/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0161 - acc: 0.0000e+00     \n",
      "Epoch 40/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0161 - acc: 0.0000e+00     \n",
      "Epoch 41/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0159 - acc: 0.0000e+00     \n",
      "Epoch 42/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0156 - acc: 0.0000e+00     \n",
      "Epoch 43/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0155 - acc: 0.0000e+00     \n",
      "Epoch 44/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0154 - acc: 0.0000e+00     \n",
      "Epoch 45/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0151 - acc: 0.0000e+00     \n",
      "Epoch 46/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0150 - acc: 0.0000e+00     \n",
      "Epoch 47/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0147 - acc: 0.0000e+00     \n",
      "Epoch 48/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0145 - acc: 0.0000e+00     \n",
      "Epoch 49/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0144 - acc: 0.0000e+00     \n",
      "Epoch 50/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0144 - acc: 0.0000e+00     \n",
      "Epoch 51/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0141 - acc: 0.0000e+00     \n",
      "Epoch 52/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0142 - acc: 0.0000e+00     \n",
      "Epoch 53/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0139 - acc: 0.0000e+00     \n",
      "Epoch 54/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0138 - acc: 0.0000e+00     \n",
      "Epoch 55/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0138 - acc: 0.0000e+00     \n",
      "Epoch 56/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0143 - acc: 0.0000e+00     \n",
      "Epoch 57/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0135 - acc: 0.0000e+00     \n",
      "Epoch 58/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0134 - acc: 0.0000e+00     \n",
      "Epoch 59/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0133 - acc: 0.0000e+00     \n",
      "Epoch 60/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0135 - acc: 0.0000e+00     \n",
      "Epoch 61/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0131 - acc: 0.0000e+00     \n",
      "Epoch 62/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0132 - acc: 0.0000e+00     \n",
      "Epoch 63/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0136 - acc: 0.0000e+00     \n",
      "Epoch 64/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0133 - acc: 0.0000e+00     \n",
      "Epoch 65/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0129 - acc: 0.0000e+00     \n",
      "Epoch 66/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0128 - acc: 0.0000e+00     \n",
      "Epoch 67/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0128 - acc: 0.0000e+00     \n",
      "Epoch 68/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0129 - acc: 0.0000e+00     \n",
      "Epoch 69/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0126 - acc: 0.0000e+00     \n",
      "Epoch 70/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0127 - acc: 0.0000e+00     \n",
      "Epoch 71/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0129 - acc: 0.0000e+00     \n",
      "Epoch 72/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0126 - acc: 0.0000e+00     \n",
      "Epoch 73/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0125 - acc: 0.0000e+00     \n",
      "Epoch 74/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0126 - acc: 0.0000e+00     \n",
      "Epoch 75/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0127 - acc: 0.0000e+00     \n",
      "Epoch 76/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0125 - acc: 0.0000e+00     \n",
      "Epoch 77/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0126 - acc: 0.0000e+00     \n",
      "Epoch 78/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0125 - acc: 0.0000e+00     \n",
      "Epoch 79/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0124 - acc: 0.0000e+00     \n",
      "Epoch 80/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0125 - acc: 0.0000e+00     \n",
      "Epoch 81/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0124 - acc: 0.0000e+00     \n",
      "Epoch 82/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0128 - acc: 0.0000e+00     \n",
      "Epoch 83/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1456/1456 [==============================] - 0s - loss: 0.0124 - acc: 0.0000e+00     \n",
      "Epoch 84/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0123 - acc: 0.0000e+00     \n",
      "Epoch 85/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0124 - acc: 0.0000e+00     \n",
      "Epoch 86/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0124 - acc: 0.0000e+00     \n",
      "Epoch 87/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0123 - acc: 0.0000e+00     \n",
      "Epoch 88/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0124 - acc: 0.0000e+00     \n",
      "Epoch 89/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0124 - acc: 0.0000e+00     \n",
      "Epoch 90/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0124 - acc: 0.0000e+00     \n",
      "Epoch 91/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0124 - acc: 0.0000e+00     \n",
      "Epoch 92/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0123 - acc: 0.0000e+00     \n",
      "Epoch 93/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0122 - acc: 0.0000e+00     \n",
      "Epoch 94/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0125 - acc: 0.0000e+00     \n",
      "Epoch 95/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0126 - acc: 0.0000e+00     \n",
      "Epoch 96/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0123 - acc: 0.0000e+00     \n",
      "Epoch 97/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0124 - acc: 0.0000e+00     \n",
      "Epoch 98/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0125 - acc: 0.0000e+00     \n",
      "Epoch 99/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0125 - acc: 0.0000e+00     \n",
      "Epoch 100/100\n",
      "1456/1456 [==============================] - 0s - loss: 0.0129 - acc: 0.0000e+00     \n"
     ]
    }
   ],
   "source": [
    "# Let's try to use neural network as layer one\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(1000, input_shape=(6,)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(100))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(1))\n",
    "model.add(Activation('linear'))\n",
    "\n",
    "model.compile(loss = 'mse', optimizer = 'adam')\n",
    "\n",
    "model.fit(blend_train, label_df, epochs=100, batch_size=112)\n",
    "pred = model.predict(blend_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "preds = model.predict(blend_test)\n",
    "preds = np.expm1(preds)\n",
    "\n",
    "pred_df = pd.DataFrame(preds, index=test_df[\"Id\"], columns=[\"SalePrice\"])\n",
    "pred_df.to_csv('stacking_test_nn.csv', header=True, index_label='Id')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}