03-other-features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "import os, sys, gc, warnings, psutil, random\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 80 entries, 0 to 79\n",
      "Data columns (total 35 columns):\n",
      " #   Column            Non-Null Count  Dtype   \n",
      "---  ------            --------------  -----   \n",
      " 0   id                80 non-null     category\n",
      " 1   item_id           80 non-null     category\n",
      " 2   dept_id           80 non-null     category\n",
      " 3   cat_id            80 non-null     category\n",
      " 4   store_id          80 non-null     category\n",
      " 5   state_id          80 non-null     category\n",
      " 6   d                 80 non-null     int16   \n",
      " 7   sales             40 non-null     float64 \n",
      " 8   day               40 non-null     float64 \n",
      " 9   release           80 non-null     int16   \n",
      " 10  sell_price        80 non-null     float16 \n",
      " 11  price_max         80 non-null     float16 \n",
      " 12  price_min         80 non-null     float16 \n",
      " 13  price_std         80 non-null     float16 \n",
      " 14  price_mean        80 non-null     float16 \n",
      " 15  price_norm        80 non-null     float16 \n",
      " 16  price_nunique     80 non-null     float16 \n",
      " 17  item_nunique      80 non-null     int16   \n",
      " 18  price_momentum    80 non-null     float16 \n",
      " 19  price_momentum_m  80 non-null     float16 \n",
      " 20  price_momentum_y  80 non-null     float16 \n",
      " 21  event_name_1      40 non-null     category\n",
      " 22  event_type_1      40 non-null     category\n",
      " 23  event_name_2      0 non-null      category\n",
      " 24  event_type_2      0 non-null      category\n",
      " 25  snap_CA           80 non-null     category\n",
      " 26  snap_TX           80 non-null     category\n",
      " 27  snap_WI           80 non-null     category\n",
      " 28  tm_d              80 non-null     int8    \n",
      " 29  tm_w              80 non-null     int8    \n",
      " 30  tm_m              80 non-null     int8    \n",
      " 31  tm_y              80 non-null     int8    \n",
      " 32  tm_wm             80 non-null     int8    \n",
      " 33  tm_dw             80 non-null     int8    \n",
      " 34  tm_w_end          80 non-null     int8    \n",
      "dtypes: category(13), float16(10), float64(2), int16(3), int8(7)\n",
      "memory usage: 1.6 MB\n"
     ]
    }
   ],
   "source": [
    "\n",
    "grid_df = pd.concat([pd.read_pickle('./output/grid_part_1.pkl'),\n",
    "                     pd.read_pickle('./output/grid_part_2.pkl').iloc[:,2:],\n",
    "                     pd.read_pickle('./output/grid_part_3.pkl').iloc[:,2:]],\n",
    "                     axis=1)\n",
    "\n",
    "# \n",
    "grid_df = grid_df.iloc[:1000, :]\n",
    "\n",
    "keep_id = np.array_split(list(grid_df['id'].unique()), 20)[0]\n",
    "grid_df = grid_df[grid_df['id'].isin(keep_id)].reset_index(drop=True)\n",
    "\n",
    "# Let's \"inspect\" our grid DataFrame\n",
    "grid_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_df.drop(['day'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Input data must be 2 dimensional and non empty.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-4ff4cb0adbc6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m \u001b[0;31m# Make baseline model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 69\u001b[0;31m \u001b[0mbaseline_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmake_fast_test\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrid_df\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-7-4ff4cb0adbc6>\u001b[0m in \u001b[0;36mmake_fast_test\u001b[0;34m(df)\u001b[0m\n\u001b[1;32m     61\u001b[0m                             \u001b[0mtrain_data\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m                             \u001b[0mvalid_sets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtrain_data\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mvalid_data\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m                             \u001b[0mverbose_eval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m500\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     64\u001b[0m                         )\n\u001b[1;32m     65\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.virtualenvs/kaggle/lib/python3.7/site-packages/lightgbm/engine.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(params, train_set, num_boost_round, valid_sets, valid_names, fobj, feval, init_model, feature_name, categorical_feature, early_stopping_rounds, evals_result, verbose_eval, learning_rates, keep_training_booster, callbacks)\u001b[0m\n\u001b[1;32m    230\u001b[0m             \u001b[0mbooster\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_train_data_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_data_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    231\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mvalid_set\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname_valid_set\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreduced_valid_sets\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname_valid_sets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 232\u001b[0;31m             \u001b[0mbooster\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_valid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalid_set\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname_valid_set\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    233\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m         \u001b[0mtrain_set\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reverse_update_params\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.virtualenvs/kaggle/lib/python3.7/site-packages/lightgbm/basic.py\u001b[0m in \u001b[0;36madd_valid\u001b[0;34m(self, data, name)\u001b[0m\n\u001b[1;32m   1893\u001b[0m         _safe_call(_LIB.LGBM_BoosterAddValidData(\n\u001b[1;32m   1894\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1895\u001b[0;31m             data.construct().handle))\n\u001b[0m\u001b[1;32m   1896\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalid_sets\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1897\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname_valid_sets\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.virtualenvs/kaggle/lib/python3.7/site-packages/lightgbm/basic.py\u001b[0m in \u001b[0;36mconstruct\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1051\u001b[0m                                     \u001b[0mweight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroup\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1052\u001b[0m                                     \u001b[0minit_score\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minit_score\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredictor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_predictor\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1053\u001b[0;31m                                     silent=self.silent, feature_name=self.feature_name, params=self.params)\n\u001b[0m\u001b[1;32m   1054\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1055\u001b[0m                     \u001b[0;31m# construct subset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.virtualenvs/kaggle/lib/python3.7/site-packages/lightgbm/basic.py\u001b[0m in \u001b[0;36m_lazy_init\u001b[0;34m(self, data, label, reference, weight, group, init_score, predictor, silent, feature_name, categorical_feature, params)\u001b[0m\n\u001b[1;32m    828\u001b[0m                                                                                              \u001b[0mfeature_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    829\u001b[0m                                                                                              \u001b[0mcategorical_feature\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 830\u001b[0;31m                                                                                              self.pandas_categorical)\n\u001b[0m\u001b[1;32m    831\u001b[0m         \u001b[0mlabel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_label_from_pandas\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    832\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.virtualenvs/kaggle/lib/python3.7/site-packages/lightgbm/basic.py\u001b[0m in \u001b[0;36m_data_from_pandas\u001b[0;34m(data, feature_name, categorical_feature, pandas_categorical)\u001b[0m\n\u001b[1;32m    313\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    314\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 315\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Input data must be 2 dimensional and non empty.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    316\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mfeature_name\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'auto'\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfeature_name\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    317\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrename\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Input data must be 2 dimensional and non empty."
     ]
    }
   ],
   "source": [
    "########################### Baseline model\n",
    "#################################################################################\n",
    "\n",
    "# We will need some global VARS for future\n",
    "\n",
    "SEED = 42  # Our random seed for everything\n",
    "random.seed(SEED)  # to make all tests \"deterministic\"\n",
    "np.random.seed(SEED)\n",
    "N_CORES = psutil.cpu_count()  # Available CPU cores\n",
    "\n",
    "TARGET = 'sales'  # Our Target\n",
    "END_TRAIN = 1913  # And we will use last 28 days as validation\n",
    "\n",
    "# Drop some items from \"TEST\" set part (1914...)\n",
    "grid_df = grid_df[grid_df['d'] <= END_TRAIN].reset_index(drop=True)\n",
    "\n",
    "# Features that we want to exclude from training\n",
    "remove_features = ['id', 'd', TARGET]\n",
    "\n",
    "# Our baseline model serves\n",
    "# to do fast checks of\n",
    "# new features performance\n",
    "\n",
    "# We will use LightGBM for our tests\n",
    "import lightgbm as lgb\n",
    "lgb_params = {\n",
    "    'boosting_type': 'gbdt',  # Standart boosting type\n",
    "    'objective': 'regression',  # Standart loss for RMSE\n",
    "    'metric': ['rmse'],  # as we will use rmse as metric \"proxy\"\n",
    "    'subsample': 0.8,\n",
    "    'subsample_freq': 1,\n",
    "    'learning_rate': 0.05,  # 0.5 is \"fast enough\" for us\n",
    "    'num_leaves': 2**7 - 1,  # We will need model only for fast check\n",
    "    'min_data_in_leaf': 2**8 -\n",
    "    1,  # So we want it to train faster even with drop in generalization \n",
    "    'feature_fraction': 0.8,\n",
    "    'n_estimators':\n",
    "    5000,  # We don't want to limit training (you can change 5000 to any big enough number)\n",
    "    'early_stopping_rounds':\n",
    "    30,  # We will stop training almost immediately (if it stops improving) \n",
    "    'seed': SEED,\n",
    "    'verbose': -1,\n",
    "    'device': 'gpu',\n",
    "    'gpu_platform_id': -1,\n",
    "    'gpu_device_id': -1\n",
    "}\n",
    "\n",
    "\n",
    "## RMSE\n",
    "def rmse(y, y_pred):\n",
    "    return np.sqrt(np.mean(np.square(y - y_pred)))\n",
    "\n",
    "\n",
    "# Small function to make fast features tests\n",
    "# estimator = make_fast_test(grid_df)\n",
    "# it will return lgb booster for future analisys\n",
    "def make_fast_test(df):\n",
    "\n",
    "    features_columns = [col for col in list(df) if col not in remove_features]\n",
    "\n",
    "    tr_x, tr_y = df[df['d'] <= (END_TRAIN - 28)][features_columns], df[\n",
    "        df['d'] <= (END_TRAIN - 28)][TARGET]\n",
    "    vl_x, v_y = df[df['d'] > (END_TRAIN - 28)][features_columns], df[\n",
    "        df['d'] > (END_TRAIN - 28)][TARGET]\n",
    "\n",
    "    train_data = lgb.Dataset(tr_x, label=tr_y)\n",
    "    valid_data = lgb.Dataset(vl_x, label=v_y)\n",
    "\n",
    "    estimator = lgb.train(\n",
    "        lgb_params,\n",
    "        train_data,\n",
    "        valid_sets=[train_data, valid_data],\n",
    "        verbose_eval=500,\n",
    "    )\n",
    "\n",
    "    return estimator\n",
    "\n",
    "\n",
    "# Make baseline model\n",
    "baseline_model = make_fast_test(grid_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "     \n",
    "########################### Some more info about lags here:\n",
    "\n",
    "\n",
    "# Small helper to make lags creation faster\n",
    "from multiprocessing import Pool                # Multiprocess Runs\n",
    "\n",
    "## Multiprocessing Run.\n",
    "# :t_split - int of lags days                   # type: int\n",
    "# :func - Function to apply on each split       # type: python function\n",
    "# This function is NOT 'bulletproof', be carefull and pass only correct types of variables.\n",
    "## Multiprocess Runs\n",
    "def df_parallelize_run(func, t_split):\n",
    "    num_cores = np.min([N_CORES,len(t_split)])\n",
    "    pool = Pool(num_cores)\n",
    "    df = pd.concat(pool.map(func, t_split), axis=1)\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "    return df\n",
    "\n",
    "def make_normal_lag(lag_day):\n",
    "    lag_df = grid_df[['id','d',TARGET]] # not good to use df from \"global space\"\n",
    "    col_name = 'sales_lag_'+str(lag_day)\n",
    "    lag_df[col_name] = lag_df.groupby(['id'])[TARGET].transform(lambda x: x.shift(lag_day)).astype(np.float16)\n",
    "    return lag_df[[col_name]]\n",
    "\n",
    "# Launch parallel lag creation\n",
    "# and \"append\" to our grid\n",
    "LAGS_SPLIT = [col for col in range(1,1+7)]\n",
    "grid_df = pd.concat([grid_df, df_parallelize_run(make_normal_lag,LAGS_SPLIT)], axis=1)\n",
    "\n",
    "# Make features test\n",
    "test_model = make_fast_test(grid_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Permutation importance Test\n",
    "\n",
    "# Let's creat validation dataset and features\n",
    "features_columns = [col for col in list(grid_df) if col not in remove_features]\n",
    "validation_df = grid_df[grid_df['d']>(END_TRAIN-28)].reset_index(drop=True)\n",
    "\n",
    "# Make normal prediction with our model and save score\n",
    "validation_df['preds'] = test_model.predict(validation_df[features_columns])\n",
    "base_score = rmse(validation_df[TARGET], validation_df['preds'])\n",
    "print('Standart RMSE', base_score)\n",
    "\n",
    "\n",
    "# Now we are looping over all our numerical features\n",
    "for col in features_columns:\n",
    "    \n",
    "    # We will make validation set copy to restore\n",
    "    # features states on each run\n",
    "    temp_df = validation_df.copy()\n",
    "    \n",
    "    # Error here appears if we have \"categorical\" features and can't \n",
    "    # do np.random.permutation without disrupt categories\n",
    "    # so we need to check if feature is numerical\n",
    "    if temp_df[col].dtypes.name != 'category':\n",
    "        temp_df[col] = np.random.permutation(temp_df[col].values)\n",
    "        temp_df['preds'] = test_model.predict(temp_df[features_columns])\n",
    "        cur_score = rmse(temp_df[TARGET], temp_df['preds'])\n",
    "        \n",
    "        # If our current rmse score is less than base score\n",
    "        # it means that feature most probably is a bad one\n",
    "        # and our model is learning on noise\n",
    "        print(col, np.round(cur_score - base_score, 4))\n",
    "\n",
    "# Remove Temp data\n",
    "del temp_df, validation_df\n",
    "\n",
    "# Remove test features\n",
    "# As we will compare performance with baseline model for now\n",
    "keep_cols = [col for col in list(grid_df) if 'sales_lag_' not in col]\n",
    "grid_df = grid_df[keep_cols]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "from eli5 documentation (seems it's perfect explanation)\n",
    "\n",
    "The idea is the following: feature importance can be measured by looking at how much the score (accuracy, mse, rmse, mae, etc. - any score we’re interested in) decreases when a feature is not available.\n",
    "\n",
    "To do that one can remove feature from the dataset, re-train the estimator and check the score. But it requires re-training an estimator for each feature, which can be computationally intensive. Also, it shows what may be important within a dataset, not what is important within a concrete trained model.\n",
    "\n",
    "To avoid re-training the estimator we can remove a feature only from the test part of the dataset, and compute score without using this feature. It doesn’t work as-is, because estimators expect feature to be present. So instead of removing a feature we can **replace it with random noise** - feature column is still there, but it no longer contains useful information. This method works if noise is drawn from the **same distribution as original feature values** (as otherwise estimator may fail). The simplest way to get such noise is to shuffle values for a feature, i.e. use other examples’ feature values - this is how permutation importance is computed.\n",
    "\n",
    "---\n",
    "\n",
    "It's not good when feature remove (replaced by noise) but we have better score. Simple and easy. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Lets test far away Lags (7 days with 56 days shift)\n",
    "########################### and check permutation importance\n",
    "#################################################################################\n",
    "\n",
    "LAGS_SPLIT = [col for col in range(56,56+7)]\n",
    "grid_df = pd.concat([grid_df, df_parallelize_run(make_normal_lag,LAGS_SPLIT)], axis=1)\n",
    "test_model = make_fast_test(grid_df)\n",
    "\n",
    "features_columns = [col for col in list(grid_df) if col not in remove_features]\n",
    "validation_df = grid_df[grid_df['d']>(END_TRAIN-28)].reset_index(drop=True)\n",
    "validation_df['preds'] = test_model.predict(validation_df[features_columns])\n",
    "base_score = rmse(validation_df[TARGET], validation_df['preds'])\n",
    "print('Standart RMSE', base_score)\n",
    "\n",
    "for col in features_columns:\n",
    "    temp_df = validation_df.copy()\n",
    "    if temp_df[col].dtypes.name != 'category':\n",
    "        temp_df[col] = np.random.permutation(temp_df[col].values)\n",
    "        temp_df['preds'] = test_model.predict(temp_df[features_columns])\n",
    "        cur_score = rmse(temp_df[TARGET], temp_df['preds'])\n",
    "        print(col, np.round(cur_score - base_score, 4))\n",
    "\n",
    "del temp_df, validation_df\n",
    "        \n",
    "# Remove test features\n",
    "# As we will compare performance with baseline model for now\n",
    "keep_cols = [col for col in list(grid_df) if 'sales_lag_' not in col]\n",
    "grid_df = grid_df[keep_cols]\n",
    "\n",
    "\n",
    "# Results:\n",
    "## Lags with 56 days shift (far away past) are not as important\n",
    "## as nearest past lags\n",
    "## and at some point will be just noise for our model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "def make_pca(df, pca_col, n_days):\n",
    "    print('PCA:', pca_col, n_days)\n",
    "    \n",
    "    # We don't need any other columns to make pca\n",
    "    pca_df = df[[pca_col,'d',TARGET]]\n",
    "    \n",
    "    # If we are doing pca for other series \"levels\" \n",
    "    # we need to agg first\n",
    "    if pca_col != 'id':\n",
    "        merge_base = pca_df[[pca_col,'d']]\n",
    "        pca_df = pca_df.groupby([pca_col,'d'])[TARGET].agg(['sum']).reset_index()\n",
    "        pca_df[TARGET] = pca_df['sum']\n",
    "        del pca_df['sum']\n",
    "    \n",
    "    # Min/Max scaling\n",
    "    pca_df[TARGET] = pca_df[TARGET]/pca_df[TARGET].max()\n",
    "    \n",
    "    # Making \"lag\" in old way (not parallel)\n",
    "    LAG_DAYS = [col for col in range(1,n_days+1)]\n",
    "    format_s = '{}_pca_'+pca_col+str(n_days)+'_{}'\n",
    "    pca_df = pca_df.assign(**{\n",
    "            format_s.format(col, l): pca_df.groupby([pca_col])[col].transform(lambda x: x.shift(l))\n",
    "            for l in LAG_DAYS\n",
    "            for col in [TARGET]\n",
    "        })\n",
    "    \n",
    "    pca_columns = list(pca_df)[3:]\n",
    "    pca_df[pca_columns] = pca_df[pca_columns].fillna(0)\n",
    "    pca = PCA(random_state=SEED)\n",
    "    \n",
    "    # You can use fit_transform here\n",
    "    pca.fit(pca_df[pca_columns])\n",
    "    pca_df[pca_columns] = pca.transform(pca_df[pca_columns])\n",
    "    \n",
    "    print(pca.explained_variance_ratio_)\n",
    "    \n",
    "    # we will keep only 3 most \"valuable\" columns/dimensions \n",
    "    keep_cols = pca_columns[:3]\n",
    "    print('Columns to keep:', keep_cols)\n",
    "    \n",
    "    # If we are doing pca for other series \"levels\"\n",
    "    # we need merge back our results to merge_base df\n",
    "    # and only than return resulted df\n",
    "    # I'll skip that step here\n",
    "    \n",
    "    return pca_df[keep_cols]\n",
    "\n",
    "\n",
    "# Make PCA\n",
    "grid_df = pd.concat([grid_df, make_pca(grid_df,'id',7)], axis=1)\n",
    "\n",
    "# Make features test\n",
    "test_model = make_fast_test(grid_df)\n",
    "\n",
    "# Remove test features\n",
    "# As we will compare performance with baseline model for now\n",
    "keep_cols = [col for col in list(grid_df) if '_pca_' not in col]\n",
    "grid_df = grid_df[keep_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Mean/std target encoding\n",
    "#################################################################################\n",
    "\n",
    "# We will use these three columns for test\n",
    "# (in combination with store_id)\n",
    "icols = ['item_id','cat_id','dept_id']\n",
    "\n",
    "# But we can use any other column or even multiple groups\n",
    "# like these ones\n",
    "#            'state_id',\n",
    "#            'store_id',\n",
    "#            'cat_id',\n",
    "#            'dept_id',\n",
    "#            ['state_id', 'cat_id'],\n",
    "#            ['state_id', 'dept_id'],\n",
    "#            ['store_id', 'cat_id'],\n",
    "#            ['store_id', 'dept_id'],\n",
    "#            'item_id',\n",
    "#            ['item_id', 'state_id'],\n",
    "#            ['item_id', 'store_id']\n",
    "\n",
    "# There are several ways to do \"mean\" encoding\n",
    "## K-fold scheme\n",
    "## LOO (leave one out)\n",
    "## Smoothed/regularized \n",
    "## Expanding mean\n",
    "## etc \n",
    "\n",
    "# You can test as many options as you want\n",
    "# and decide what to use\n",
    "# Because of memory issues you can't \n",
    "# use many features.\n",
    "\n",
    "# We will use simple target encoding\n",
    "# by std and mean agg\n",
    "for col in icols:\n",
    "    print('Encoding', col)\n",
    "    temp_df = grid_df[grid_df['d']<=(1913-28)] # to be sure we don't have leakage in our validation set\n",
    "    \n",
    "    temp_df = temp_df.groupby([col,'store_id']).agg({TARGET: ['std','mean']})\n",
    "    joiner = '_'+col+'_encoding_'\n",
    "    temp_df.columns = [joiner.join(col).strip() for col in temp_df.columns.values]\n",
    "    temp_df = temp_df.reset_index()\n",
    "    grid_df = grid_df.merge(temp_df, on=[col,'store_id'], how='left')\n",
    "    del temp_df\n",
    "\n",
    "# Make features test\n",
    "test_model = make_fast_test(grid_df)\n",
    "\n",
    "# Remove test features\n",
    "keep_cols = [col for col in list(grid_df) if '_encoding_' not in col]\n",
    "grid_df = grid_df[keep_cols]\n",
    "\n",
    "# Bad thing that for some items  \n",
    "# we are using past and future values.\n",
    "# But we are looking for \"categorical\" similiarity\n",
    "# on a \"long run\". So future here is not a big problem."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Last non O sale\n",
    "#################################################################################\n",
    "\n",
    "def find_last_sale(df,n_day):\n",
    "    \n",
    "    # Limit initial df\n",
    "    ls_df = df[['id','d',TARGET]]\n",
    "    \n",
    "    # Convert target to binary\n",
    "    ls_df['non_zero'] = (ls_df[TARGET]>0).astype(np.int8)\n",
    "    \n",
    "    # Make lags to prevent any leakage\n",
    "    ls_df['non_zero_lag'] = ls_df.groupby(['id'])['non_zero'].transform(lambda x: x.shift(n_day).rolling(2000,1).sum()).fillna(-1)\n",
    "\n",
    "    temp_df = ls_df[['id','d','non_zero_lag']].drop_duplicates(subset=['id','non_zero_lag'])\n",
    "    temp_df.columns = ['id','d_min','non_zero_lag']\n",
    "\n",
    "    ls_df = ls_df.merge(temp_df, on=['id','non_zero_lag'], how='left')\n",
    "    ls_df['last_sale'] = ls_df['d'] - ls_df['d_min']\n",
    "\n",
    "    return ls_df[['last_sale']]\n",
    "\n",
    "\n",
    "# Find last non zero\n",
    "# Need some \"dances\" to fit in memory limit with groupers\n",
    "grid_df = pd.concat([grid_df, find_last_sale(grid_df,1)], axis=1)\n",
    "\n",
    "# Make features test\n",
    "test_model = make_fast_test(grid_df)\n",
    "\n",
    "# Remove test features\n",
    "keep_cols = [col for col in list(grid_df) if 'last_sale' not in col]\n",
    "grid_df = grid_df[keep_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Apply on grid_df\n",
    "#################################################################################\n",
    "# lets read grid from \n",
    "# https://www.kaggle.com/kyakovlev/m5-simple-fe\n",
    "# to be sure that our grids are aligned by index\n",
    "grid_df = pd.read_pickle('./output/grid_part_1.pkl')\n",
    "grid_df[TARGET][grid_df['d']>(1913-28)] = np.nan\n",
    "base_cols = list(grid_df)\n",
    "\n",
    "icols =  [\n",
    "            ['state_id'],\n",
    "            ['store_id'],\n",
    "            ['cat_id'],\n",
    "            ['dept_id'],\n",
    "            ['state_id', 'cat_id'],\n",
    "            ['state_id', 'dept_id'],\n",
    "            ['store_id', 'cat_id'],\n",
    "            ['store_id', 'dept_id'],\n",
    "            ['item_id'],\n",
    "            ['item_id', 'state_id'],\n",
    "            ['item_id', 'store_id']\n",
    "            ]\n",
    "\n",
    "for col in icols:\n",
    "    print('Encoding', col)\n",
    "    col_name = '_'+'_'.join(col)+'_'\n",
    "    grid_df['enc'+col_name+'mean'] = grid_df.groupby(col)[TARGET].transform('mean').astype(np.float16)\n",
    "    grid_df['enc'+col_name+'std'] = grid_df.groupby(col)[TARGET].transform('std').astype(np.float16)\n",
    "\n",
    "keep_cols = [col for col in list(grid_df) if col not in base_cols]\n",
    "grid_df = grid_df[['id','d']+keep_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#################################################################################\n",
    "print('Save Mean/Std encoding')\n",
    "grid_df.to_pickle('./output/mean_encoding_df.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Final list of new features\n",
    "#################################################################################\n",
    "grid_df.info()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}