Neural_Network_Hyperparameter_Tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b8fda16d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing the necessary packages\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import keras\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c494eb06",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "56bd4842",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load pima indians dataset\n",
    "dataset = np.loadtxt(\"C:/Users/Moin Dalvi/Documents/Data Science Material/Data Science/Python/Datasets/pima-indians-diabetes.data.csv\", delimiter=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "996f9a8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = dataset[:,0:8]\n",
    "y = dataset[:,8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "906fe977",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Standardization\n",
    "a = StandardScaler()\n",
    "a.fit(X)\n",
    "X_standardized = a.transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1b89c3d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>7.680000e+02</td>\n",
       "      <td>7.680000e+02</td>\n",
       "      <td>7.680000e+02</td>\n",
       "      <td>7.680000e+02</td>\n",
       "      <td>7.680000e+02</td>\n",
       "      <td>7.680000e+02</td>\n",
       "      <td>7.680000e+02</td>\n",
       "      <td>7.680000e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-7.748432e-17</td>\n",
       "      <td>3.614007e-18</td>\n",
       "      <td>-1.327244e-17</td>\n",
       "      <td>7.762888e-17</td>\n",
       "      <td>-5.493291e-17</td>\n",
       "      <td>2.972738e-15</td>\n",
       "      <td>1.924387e-15</td>\n",
       "      <td>2.192980e-16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.000652e+00</td>\n",
       "      <td>1.000652e+00</td>\n",
       "      <td>1.000652e+00</td>\n",
       "      <td>1.000652e+00</td>\n",
       "      <td>1.000652e+00</td>\n",
       "      <td>1.000652e+00</td>\n",
       "      <td>1.000652e+00</td>\n",
       "      <td>1.000652e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-1.141852e+00</td>\n",
       "      <td>-3.783654e+00</td>\n",
       "      <td>-3.572597e+00</td>\n",
       "      <td>-1.288212e+00</td>\n",
       "      <td>-6.928906e-01</td>\n",
       "      <td>-4.060474e+00</td>\n",
       "      <td>-1.189553e+00</td>\n",
       "      <td>-1.041549e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-8.448851e-01</td>\n",
       "      <td>-6.852363e-01</td>\n",
       "      <td>-3.673367e-01</td>\n",
       "      <td>-1.288212e+00</td>\n",
       "      <td>-6.928906e-01</td>\n",
       "      <td>-5.955785e-01</td>\n",
       "      <td>-6.889685e-01</td>\n",
       "      <td>-7.862862e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-2.509521e-01</td>\n",
       "      <td>-1.218877e-01</td>\n",
       "      <td>1.496408e-01</td>\n",
       "      <td>1.545332e-01</td>\n",
       "      <td>-4.280622e-01</td>\n",
       "      <td>9.419788e-04</td>\n",
       "      <td>-3.001282e-01</td>\n",
       "      <td>-3.608474e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>6.399473e-01</td>\n",
       "      <td>6.057709e-01</td>\n",
       "      <td>5.632228e-01</td>\n",
       "      <td>7.190857e-01</td>\n",
       "      <td>4.120079e-01</td>\n",
       "      <td>5.847705e-01</td>\n",
       "      <td>4.662269e-01</td>\n",
       "      <td>6.602056e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.906578e+00</td>\n",
       "      <td>2.444478e+00</td>\n",
       "      <td>2.734528e+00</td>\n",
       "      <td>4.921866e+00</td>\n",
       "      <td>6.652839e+00</td>\n",
       "      <td>4.455807e+00</td>\n",
       "      <td>5.883565e+00</td>\n",
       "      <td>4.063716e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0             1             2             3             4  \\\n",
       "count  7.680000e+02  7.680000e+02  7.680000e+02  7.680000e+02  7.680000e+02   \n",
       "mean  -7.748432e-17  3.614007e-18 -1.327244e-17  7.762888e-17 -5.493291e-17   \n",
       "std    1.000652e+00  1.000652e+00  1.000652e+00  1.000652e+00  1.000652e+00   \n",
       "min   -1.141852e+00 -3.783654e+00 -3.572597e+00 -1.288212e+00 -6.928906e-01   \n",
       "25%   -8.448851e-01 -6.852363e-01 -3.673367e-01 -1.288212e+00 -6.928906e-01   \n",
       "50%   -2.509521e-01 -1.218877e-01  1.496408e-01  1.545332e-01 -4.280622e-01   \n",
       "75%    6.399473e-01  6.057709e-01  5.632228e-01  7.190857e-01  4.120079e-01   \n",
       "max    3.906578e+00  2.444478e+00  2.734528e+00  4.921866e+00  6.652839e+00   \n",
       "\n",
       "                  5             6             7  \n",
       "count  7.680000e+02  7.680000e+02  7.680000e+02  \n",
       "mean   2.972738e-15  1.924387e-15  2.192980e-16  \n",
       "std    1.000652e+00  1.000652e+00  1.000652e+00  \n",
       "min   -4.060474e+00 -1.189553e+00 -1.041549e+00  \n",
       "25%   -5.955785e-01 -6.889685e-01 -7.862862e-01  \n",
       "50%    9.419788e-04 -3.001282e-01 -3.608474e-01  \n",
       "75%    5.847705e-01  4.662269e-01  6.602056e-01  \n",
       "max    4.455807e+00  5.883565e+00  4.063716e+00  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(X_standardized).describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2aed3144",
   "metadata": {},
   "source": [
    "#### Tuning of Hyperparameters :- Batch Size and Epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0eedde78",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing the necessary packages\n",
    "from sklearn.model_selection import GridSearchCV, KFold\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c52c2645",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create model\n",
    "def create_model():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(12, input_dim=8, kernel_initializer='uniform', activation='relu'))\n",
    "    model.add(Dense(8, kernel_initializer='uniform', activation='relu'))\n",
    "    model.add(Dense(1, kernel_initializer='uniform', activation='sigmoid'))\n",
    "    \n",
    "    adam=Adam(lr=0.01)\n",
    "    model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9bd9ef88",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\AppData\\Local\\Temp\\ipykernel_12640\\659144266.py:2: DeprecationWarning: KerasClassifier is deprecated, use Sci-Keras (https://github.com/adriangb/scikeras) instead. See https://www.adriangb.com/scikeras/stable/migration.html for help migrating.\n",
      "  model = KerasClassifier(build_fn = create_model,verbose = 0)\n",
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 9 candidates, totalling 45 fits\n",
      "[CV 1/5; 1/9] START batch_size=10, epochs=10....................................\n",
      "[CV 1/5; 1/9] END .....batch_size=10, epochs=10;, score=0.747 total time=   2.3s\n",
      "[CV 2/5; 1/9] START batch_size=10, epochs=10....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/5; 1/9] END .....batch_size=10, epochs=10;, score=0.721 total time=   1.6s\n",
      "[CV 3/5; 1/9] START batch_size=10, epochs=10....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/5; 1/9] END .....batch_size=10, epochs=10;, score=0.779 total time=   1.6s\n",
      "[CV 4/5; 1/9] START batch_size=10, epochs=10....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 4/5; 1/9] END .....batch_size=10, epochs=10;, score=0.830 total time=   1.6s\n",
      "[CV 5/5; 1/9] START batch_size=10, epochs=10....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 5/5; 1/9] END .....batch_size=10, epochs=10;, score=0.784 total time=   1.7s\n",
      "[CV 1/5; 2/9] START batch_size=10, epochs=50....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/5; 2/9] END .....batch_size=10, epochs=50;, score=0.766 total time=   5.7s\n",
      "[CV 2/5; 2/9] START batch_size=10, epochs=50....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/5; 2/9] END .....batch_size=10, epochs=50;, score=0.734 total time=   5.3s\n",
      "[CV 3/5; 2/9] START batch_size=10, epochs=50....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/5; 2/9] END .....batch_size=10, epochs=50;, score=0.760 total time=   4.7s\n",
      "[CV 4/5; 2/9] START batch_size=10, epochs=50....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 4/5; 2/9] END .....batch_size=10, epochs=50;, score=0.837 total time=   5.1s\n",
      "[CV 5/5; 2/9] START batch_size=10, epochs=50....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 5/5; 2/9] END .....batch_size=10, epochs=50;, score=0.765 total time=   5.0s\n",
      "[CV 1/5; 3/9] START batch_size=10, epochs=100...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/5; 3/9] END ....batch_size=10, epochs=100;, score=0.695 total time=   9.5s\n",
      "[CV 2/5; 3/9] START batch_size=10, epochs=100...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/5; 3/9] END ....batch_size=10, epochs=100;, score=0.695 total time=   9.3s\n",
      "[CV 3/5; 3/9] START batch_size=10, epochs=100...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/5; 3/9] END ....batch_size=10, epochs=100;, score=0.773 total time=   9.4s\n",
      "[CV 4/5; 3/9] START batch_size=10, epochs=100...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 4/5; 3/9] END ....batch_size=10, epochs=100;, score=0.778 total time=   9.1s\n",
      "[CV 5/5; 3/9] START batch_size=10, epochs=100...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 5/5; 3/9] END ....batch_size=10, epochs=100;, score=0.778 total time=  10.4s\n",
      "[CV 1/5; 4/9] START batch_size=20, epochs=10....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/5; 4/9] END .....batch_size=20, epochs=10;, score=0.760 total time=   1.6s\n",
      "[CV 2/5; 4/9] START batch_size=20, epochs=10....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/5; 4/9] END .....batch_size=20, epochs=10;, score=0.695 total time=   1.1s\n",
      "[CV 3/5; 4/9] START batch_size=20, epochs=10....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/5; 4/9] END .....batch_size=20, epochs=10;, score=0.753 total time=   1.1s\n",
      "[CV 4/5; 4/9] START batch_size=20, epochs=10....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 4/5; 4/9] END .....batch_size=20, epochs=10;, score=0.843 total time=   1.2s\n",
      "[CV 5/5; 4/9] START batch_size=20, epochs=10....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 5/5; 4/9] END .....batch_size=20, epochs=10;, score=0.758 total time=   1.2s\n",
      "[CV 1/5; 5/9] START batch_size=20, epochs=50....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/5; 5/9] END .....batch_size=20, epochs=50;, score=0.701 total time=   2.7s\n",
      "[CV 2/5; 5/9] START batch_size=20, epochs=50....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/5; 5/9] END .....batch_size=20, epochs=50;, score=0.695 total time=   3.1s\n",
      "[CV 3/5; 5/9] START batch_size=20, epochs=50....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/5; 5/9] END .....batch_size=20, epochs=50;, score=0.773 total time=   2.6s\n",
      "[CV 4/5; 5/9] START batch_size=20, epochs=50....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 4/5; 5/9] END .....batch_size=20, epochs=50;, score=0.824 total time=   2.8s\n",
      "[CV 5/5; 5/9] START batch_size=20, epochs=50....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 5/5; 5/9] END .....batch_size=20, epochs=50;, score=0.784 total time=   2.7s\n",
      "[CV 1/5; 6/9] START batch_size=20, epochs=100...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/5; 6/9] END ....batch_size=20, epochs=100;, score=0.734 total time=   4.7s\n",
      "[CV 2/5; 6/9] START batch_size=20, epochs=100...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/5; 6/9] END ....batch_size=20, epochs=100;, score=0.708 total time=   4.7s\n",
      "[CV 3/5; 6/9] START batch_size=20, epochs=100...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/5; 6/9] END ....batch_size=20, epochs=100;, score=0.753 total time=   4.8s\n",
      "[CV 4/5; 6/9] START batch_size=20, epochs=100...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 4/5; 6/9] END ....batch_size=20, epochs=100;, score=0.771 total time=   5.0s\n",
      "[CV 5/5; 6/9] START batch_size=20, epochs=100...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 5/5; 6/9] END ....batch_size=20, epochs=100;, score=0.745 total time=   4.8s\n",
      "[CV 1/5; 7/9] START batch_size=40, epochs=10....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/5; 7/9] END .....batch_size=40, epochs=10;, score=0.727 total time=   0.9s\n",
      "[CV 2/5; 7/9] START batch_size=40, epochs=10....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/5; 7/9] END .....batch_size=40, epochs=10;, score=0.708 total time=   1.0s\n",
      "[CV 3/5; 7/9] START batch_size=40, epochs=10....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/5; 7/9] END .....batch_size=40, epochs=10;, score=0.773 total time=   0.9s\n",
      "[CV 4/5; 7/9] START batch_size=40, epochs=10....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 4/5; 7/9] END .....batch_size=40, epochs=10;, score=0.856 total time=   0.9s\n",
      "[CV 5/5; 7/9] START batch_size=40, epochs=10....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 5/5; 7/9] END .....batch_size=40, epochs=10;, score=0.778 total time=   0.9s\n",
      "[CV 1/5; 8/9] START batch_size=40, epochs=50....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/5; 8/9] END .....batch_size=40, epochs=50;, score=0.734 total time=   1.7s\n",
      "[CV 2/5; 8/9] START batch_size=40, epochs=50....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/5; 8/9] END .....batch_size=40, epochs=50;, score=0.714 total time=   1.8s\n",
      "[CV 3/5; 8/9] START batch_size=40, epochs=50....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/5; 8/9] END .....batch_size=40, epochs=50;, score=0.753 total time=   2.0s\n",
      "[CV 4/5; 8/9] START batch_size=40, epochs=50....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 4/5; 8/9] END .....batch_size=40, epochs=50;, score=0.791 total time=   1.6s\n",
      "[CV 5/5; 8/9] START batch_size=40, epochs=50....................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 5/5; 8/9] END .....batch_size=40, epochs=50;, score=0.765 total time=   1.8s\n",
      "[CV 1/5; 9/9] START batch_size=40, epochs=100...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/5; 9/9] END ....batch_size=40, epochs=100;, score=0.714 total time=   2.9s\n",
      "[CV 2/5; 9/9] START batch_size=40, epochs=100...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 2/5; 9/9] END ....batch_size=40, epochs=100;, score=0.701 total time=   3.0s\n",
      "[CV 3/5; 9/9] START batch_size=40, epochs=100...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/5; 9/9] END ....batch_size=40, epochs=100;, score=0.747 total time=   2.9s\n",
      "[CV 4/5; 9/9] START batch_size=40, epochs=100...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 4/5; 9/9] END ....batch_size=40, epochs=100;, score=0.837 total time=   2.9s\n",
      "[CV 5/5; 9/9] START batch_size=40, epochs=100...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Moin Dalvi\\anaconda3\\lib\\site-packages\\keras\\optimizers\\optimizer_v2\\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super(Adam, self).__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 5/5; 9/9] END ....batch_size=40, epochs=100;, score=0.752 total time=   2.8s\n"
     ]
    }
   ],
   "source": [
    "# Create the model\n",
    "model = KerasClassifier(build_fn = create_model,verbose = 0)\n",
    "# Define the grid search parameters\n",
    "batch_size = [10,20,40]\n",
    "epochs = [10,50,100]\n",
    "# Make a dictionary of the grid search parameters\n",
    "param_grid = dict(batch_size = batch_size,epochs = epochs)\n",
    "# Build and fit the GridSearchCV\n",
    "grid = GridSearchCV(estimator = model,param_grid = param_grid,cv = KFold(),verbose = 10)\n",
    "grid_result = grid.fit(X_standardized,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "641b8463",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best : 0.7722264766693115, using {'batch_size': 10, 'epochs': 10}\n",
      "0.7722264766693115,0.036975834159730075 with: {'batch_size': 10, 'epochs': 10}\n",
      "0.7722094893455506,0.034268601717236793 with: {'batch_size': 10, 'epochs': 50}\n",
      "0.7435786485671997,0.03986602898915845 with: {'batch_size': 10, 'epochs': 100}\n",
      "0.7618198752403259,0.04731728982889819 with: {'batch_size': 20, 'epochs': 10}\n",
      "0.7553348660469055,0.049752413490729626 with: {'batch_size': 20, 'epochs': 50}\n",
      "0.7422290205955505,0.021113391385727803 with: {'batch_size': 20, 'epochs': 100}\n",
      "0.7683558344841004,0.05134561358505284 with: {'batch_size': 40, 'epochs': 10}\n",
      "0.7513708591461181,0.026173072150101364 with: {'batch_size': 40, 'epochs': 50}\n",
      "0.7501146078109742,0.04724459308295821 with: {'batch_size': 40, 'epochs': 100}\n"
     ]
    }
   ],
   "source": [
    "# Summarize the results\n",
    "print('Best : {}, using {}'.format(grid_result.best_score_,grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "  print('{},{} with: {}'.format(mean, stdev, param))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc92b091",
   "metadata": {},
   "source": [
    "#### Tuning of Hyperparameters:- Learning rate and Drop out rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "510a55be",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 9 candidates, totalling 45 fits\n",
      "[CV 1/5; 1/9] START dropout_rate=0.0, learning_rate=0.001.......................\n",
      "[CV 1/5; 1/9] END dropout_rate=0.0, learning_rate=0.001;, score=0.766 total time=   0.9s\n",
      "[CV 2/5; 1/9] START dropout_rate=0.0, learning_rate=0.001.......................\n",
      "[CV 2/5; 1/9] END dropout_rate=0.0, learning_rate=0.001;, score=0.721 total time=   1.1s\n",
      "[CV 3/5; 1/9] START dropout_rate=0.0, learning_rate=0.001.......................\n",
      "[CV 3/5; 1/9] END dropout_rate=0.0, learning_rate=0.001;, score=0.760 total time=   1.2s\n",
      "[CV 4/5; 1/9] START dropout_rate=0.0, learning_rate=0.001.......................\n",
      "[CV 4/5; 1/9] END dropout_rate=0.0, learning_rate=0.001;, score=0.804 total time=   1.3s\n",
      "[CV 5/5; 1/9] START dropout_rate=0.0, learning_rate=0.001.......................\n",
      "[CV 5/5; 1/9] END dropout_rate=0.0, learning_rate=0.001;, score=0.765 total time=   1.3s\n",
      "[CV 1/5; 2/9] START dropout_rate=0.0, learning_rate=0.01........................\n",
      "[CV 1/5; 2/9] END dropout_rate=0.0, learning_rate=0.01;, score=0.714 total time=   1.5s\n",
      "[CV 2/5; 2/9] START dropout_rate=0.0, learning_rate=0.01........................\n",
      "[CV 2/5; 2/9] END dropout_rate=0.0, learning_rate=0.01;, score=0.721 total time=   1.5s\n",
      "[CV 3/5; 2/9] START dropout_rate=0.0, learning_rate=0.01........................\n",
      "[CV 3/5; 2/9] END dropout_rate=0.0, learning_rate=0.01;, score=0.766 total time=   1.1s\n",
      "[CV 4/5; 2/9] START dropout_rate=0.0, learning_rate=0.01........................\n",
      "[CV 4/5; 2/9] END dropout_rate=0.0, learning_rate=0.01;, score=0.784 total time=   1.2s\n",
      "[CV 5/5; 2/9] START dropout_rate=0.0, learning_rate=0.01........................\n",
      "[CV 5/5; 2/9] END dropout_rate=0.0, learning_rate=0.01;, score=0.752 total time=   1.1s\n",
      "[CV 1/5; 3/9] START dropout_rate=0.0, learning_rate=0.1.........................\n",
      "[CV 1/5; 3/9] END dropout_rate=0.0, learning_rate=0.1;, score=0.727 total time=   1.2s\n",
      "[CV 2/5; 3/9] START dropout_rate=0.0, learning_rate=0.1.........................\n",
      "[CV 2/5; 3/9] END dropout_rate=0.0, learning_rate=0.1;, score=0.643 total time=   1.0s\n",
      "[CV 3/5; 3/9] START dropout_rate=0.0, learning_rate=0.1.........................\n",
      "[CV 3/5; 3/9] END dropout_rate=0.0, learning_rate=0.1;, score=0.753 total time=   1.3s\n",
      "[CV 4/5; 3/9] START dropout_rate=0.0, learning_rate=0.1.........................\n",
      "[CV 4/5; 3/9] END dropout_rate=0.0, learning_rate=0.1;, score=0.784 total time=   1.1s\n",
      "[CV 5/5; 3/9] START dropout_rate=0.0, learning_rate=0.1.........................\n",
      "[CV 5/5; 3/9] END dropout_rate=0.0, learning_rate=0.1;, score=0.758 total time=   0.9s\n",
      "[CV 1/5; 4/9] START dropout_rate=0.1, learning_rate=0.001.......................\n",
      "[CV 1/5; 4/9] END dropout_rate=0.1, learning_rate=0.001;, score=0.753 total time=   1.0s\n",
      "[CV 2/5; 4/9] START dropout_rate=0.1, learning_rate=0.001.......................\n",
      "[CV 2/5; 4/9] END dropout_rate=0.1, learning_rate=0.001;, score=0.695 total time=   1.1s\n",
      "[CV 3/5; 4/9] START dropout_rate=0.1, learning_rate=0.001.......................\n",
      "[CV 3/5; 4/9] END dropout_rate=0.1, learning_rate=0.001;, score=0.760 total time=   1.0s\n",
      "[CV 4/5; 4/9] START dropout_rate=0.1, learning_rate=0.001.......................\n",
      "[CV 4/5; 4/9] END dropout_rate=0.1, learning_rate=0.001;, score=0.745 total time=   1.5s\n",
      "[CV 5/5; 4/9] START dropout_rate=0.1, learning_rate=0.001.......................\n",
      "[CV 5/5; 4/9] END dropout_rate=0.1, learning_rate=0.001;, score=0.641 total time=   1.4s\n",
      "[CV 1/5; 5/9] START dropout_rate=0.1, learning_rate=0.01........................\n",
      "[CV 1/5; 5/9] END dropout_rate=0.1, learning_rate=0.01;, score=0.753 total time=   1.2s\n",
      "[CV 2/5; 5/9] START dropout_rate=0.1, learning_rate=0.01........................\n",
      "[CV 2/5; 5/9] END dropout_rate=0.1, learning_rate=0.01;, score=0.721 total time=   1.2s\n",
      "[CV 3/5; 5/9] START dropout_rate=0.1, learning_rate=0.01........................\n",
      "[CV 3/5; 5/9] END dropout_rate=0.1, learning_rate=0.01;, score=0.747 total time=   1.4s\n",
      "[CV 4/5; 5/9] START dropout_rate=0.1, learning_rate=0.01........................\n",
      "[CV 4/5; 5/9] END dropout_rate=0.1, learning_rate=0.01;, score=0.837 total time=   1.3s\n",
      "[CV 5/5; 5/9] START dropout_rate=0.1, learning_rate=0.01........................\n",
      "[CV 5/5; 5/9] END dropout_rate=0.1, learning_rate=0.01;, score=0.752 total time=   1.4s\n",
      "[CV 1/5; 6/9] START dropout_rate=0.1, learning_rate=0.1.........................\n",
      "[CV 1/5; 6/9] END dropout_rate=0.1, learning_rate=0.1;, score=0.747 total time=   1.3s\n",
      "[CV 2/5; 6/9] START dropout_rate=0.1, learning_rate=0.1.........................\n",
      "[CV 2/5; 6/9] END dropout_rate=0.1, learning_rate=0.1;, score=0.610 total time=   1.4s\n",
      "[CV 3/5; 6/9] START dropout_rate=0.1, learning_rate=0.1.........................\n",
      "[CV 3/5; 6/9] END dropout_rate=0.1, learning_rate=0.1;, score=0.773 total time=   3.3s\n",
      "[CV 4/5; 6/9] START dropout_rate=0.1, learning_rate=0.1.........................\n",
      "[CV 4/5; 6/9] END dropout_rate=0.1, learning_rate=0.1;, score=0.745 total time=   1.4s\n",
      "[CV 5/5; 6/9] START dropout_rate=0.1, learning_rate=0.1.........................\n",
      "[CV 5/5; 6/9] END dropout_rate=0.1, learning_rate=0.1;, score=0.752 total time=   1.4s\n",
      "[CV 1/5; 7/9] START dropout_rate=0.2, learning_rate=0.001.......................\n",
      "[CV 1/5; 7/9] END dropout_rate=0.2, learning_rate=0.001;, score=0.779 total time=   1.2s\n",
      "[CV 2/5; 7/9] START dropout_rate=0.2, learning_rate=0.001.......................\n",
      "[CV 2/5; 7/9] END dropout_rate=0.2, learning_rate=0.001;, score=0.682 total time=   1.3s\n",
      "[CV 3/5; 7/9] START dropout_rate=0.2, learning_rate=0.001.......................\n",
      "[CV 3/5; 7/9] END dropout_rate=0.2, learning_rate=0.001;, score=0.773 total time=   1.1s\n",
      "[CV 4/5; 7/9] START dropout_rate=0.2, learning_rate=0.001.......................\n",
      "[CV 4/5; 7/9] END dropout_rate=0.2, learning_rate=0.001;, score=0.804 total time=   1.0s\n",
      "[CV 5/5; 7/9] START dropout_rate=0.2, learning_rate=0.001.......................\n",
      "[CV 5/5; 7/9] END dropout_rate=0.2, learning_rate=0.001;, score=0.647 total time=   1.0s\n",
      "[CV 1/5; 8/9] START dropout_rate=0.2, learning_rate=0.01........................\n",
      "[CV 1/5; 8/9] END dropout_rate=0.2, learning_rate=0.01;, score=0.734 total time=   1.3s\n",
      "[CV 2/5; 8/9] START dropout_rate=0.2, learning_rate=0.01........................\n",
      "[CV 2/5; 8/9] END dropout_rate=0.2, learning_rate=0.01;, score=0.701 total time=   1.0s\n",
      "[CV 3/5; 8/9] START dropout_rate=0.2, learning_rate=0.01........................\n",
      "[CV 3/5; 8/9] END dropout_rate=0.2, learning_rate=0.01;, score=0.760 total time=   1.3s\n",
      "[CV 4/5; 8/9] START dropout_rate=0.2, learning_rate=0.01........................\n",
      "[CV 4/5; 8/9] END dropout_rate=0.2, learning_rate=0.01;, score=0.843 total time=   1.6s\n",
      "[CV 5/5; 8/9] START dropout_rate=0.2, learning_rate=0.01........................\n",
      "[CV 5/5; 8/9] END dropout_rate=0.2, learning_rate=0.01;, score=0.758 total time=   1.8s\n",
      "[CV 1/5; 9/9] START dropout_rate=0.2, learning_rate=0.1.........................\n",
      "[CV 1/5; 9/9] END dropout_rate=0.2, learning_rate=0.1;, score=0.701 total time=   1.1s\n",
      "[CV 2/5; 9/9] START dropout_rate=0.2, learning_rate=0.1.........................\n",
      "[CV 2/5; 9/9] END dropout_rate=0.2, learning_rate=0.1;, score=0.721 total time=   1.1s\n",
      "[CV 3/5; 9/9] START dropout_rate=0.2, learning_rate=0.1.........................\n",
      "[CV 3/5; 9/9] END dropout_rate=0.2, learning_rate=0.1;, score=0.760 total time=   1.5s\n",
      "[CV 4/5; 9/9] START dropout_rate=0.2, learning_rate=0.1.........................\n",
      "[CV 4/5; 9/9] END dropout_rate=0.2, learning_rate=0.1;, score=0.797 total time=   1.4s\n",
      "[CV 5/5; 9/9] START dropout_rate=0.2, learning_rate=0.1.........................\n",
      "[CV 5/5; 9/9] END dropout_rate=0.2, learning_rate=0.1;, score=0.719 total time=   0.9s\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import Dropout\n",
    "\n",
    "# Defining the model\n",
    "\n",
    "def create_model(learning_rate,dropout_rate):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(8,input_dim = 8,kernel_initializer = 'normal',activation = 'relu'))\n",
    "    model.add(Dropout(dropout_rate))\n",
    "    model.add(Dense(4,input_dim = 8,kernel_initializer = 'normal',activation = 'relu'))\n",
    "    model.add(Dropout(dropout_rate))\n",
    "    model.add(Dense(1,activation = 'sigmoid'))\n",
    "    \n",
    "    adam = Adam(lr = learning_rate)\n",
    "    model.compile(loss = 'binary_crossentropy',optimizer = adam,metrics = ['accuracy'])\n",
    "    return model\n",
    "\n",
    "# Create the model\n",
    "\n",
    "model = KerasClassifier(build_fn = create_model,verbose = 0,batch_size = 40,epochs = 10)\n",
    "\n",
    "# Define the grid search parameters\n",
    "\n",
    "learning_rate = [0.001,0.01,0.1]\n",
    "dropout_rate = [0.0,0.1,0.2]\n",
    "\n",
    "# Make a dictionary of the grid search parameters\n",
    "\n",
    "param_grids = dict(learning_rate = learning_rate,dropout_rate = dropout_rate)\n",
    "\n",
    "# Build and fit the GridSearchCV\n",
    "\n",
    "grid = GridSearchCV(estimator = model,param_grid = param_grids,cv = KFold(),verbose = 10)\n",
    "grid_result = grid.fit(X_standardized,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0bf4cb31",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best : 0.7630761384963989, using {'dropout_rate': 0.0, 'learning_rate': 0.001}\n",
      "0.7630761384963989,0.02638610453792774 with: {'dropout_rate': 0.0, 'learning_rate': 0.001}\n",
      "0.7474492907524108,0.026610028916167146 with: {'dropout_rate': 0.0, 'learning_rate': 0.01}\n",
      "0.7331720590591431,0.04865171229822669 with: {'dropout_rate': 0.0, 'learning_rate': 0.1}\n",
      "0.7186826229095459,0.045292942798631296 with: {'dropout_rate': 0.1, 'learning_rate': 0.001}\n",
      "0.7618029117584229,0.039194003812358866 with: {'dropout_rate': 0.1, 'learning_rate': 0.01}\n",
      "0.7253204345703125,0.05830854131456221 with: {'dropout_rate': 0.1, 'learning_rate': 0.1}\n",
      "0.7369493246078491,0.06110966863612401 with: {'dropout_rate': 0.2, 'learning_rate': 0.001}\n",
      "0.7592224836349487,0.04700265707555715 with: {'dropout_rate': 0.2, 'learning_rate': 0.01}\n",
      "0.7396316170692444,0.03461866343073 with: {'dropout_rate': 0.2, 'learning_rate': 0.1}\n"
     ]
    }
   ],
   "source": [
    "# Summarize the results\n",
    "print('Best : {}, using {}'.format(grid_result.best_score_,grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "  print('{},{} with: {}'.format(mean, stdev, param))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1bd5848b",
   "metadata": {},
   "source": [
    "#### Tuning of Hyperparameters:- Activation Function and Kernel Initializer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "0c44dfe8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 12 candidates, totalling 60 fits\n",
      "[CV 1/5; 1/12] START activation_function=softmax, init=uniform..................\n",
      "[CV 1/5; 1/12] END activation_function=softmax, init=uniform;, score=0.649 total time=   1.4s\n",
      "[CV 2/5; 1/12] START activation_function=softmax, init=uniform..................\n",
      "[CV 2/5; 1/12] END activation_function=softmax, init=uniform;, score=0.584 total time=   1.2s\n",
      "[CV 3/5; 1/12] START activation_function=softmax, init=uniform..................\n",
      "[CV 3/5; 1/12] END activation_function=softmax, init=uniform;, score=0.630 total time=   1.5s\n",
      "[CV 4/5; 1/12] START activation_function=softmax, init=uniform..................\n",
      "[CV 4/5; 1/12] END activation_function=softmax, init=uniform;, score=0.745 total time=   1.6s\n",
      "[CV 5/5; 1/12] START activation_function=softmax, init=uniform..................\n",
      "[CV 5/5; 1/12] END activation_function=softmax, init=uniform;, score=0.647 total time=   1.2s\n",
      "[CV 1/5; 2/12] START activation_function=softmax, init=normal...................\n",
      "[CV 1/5; 2/12] END activation_function=softmax, init=normal;, score=0.649 total time=   1.2s\n",
      "[CV 2/5; 2/12] START activation_function=softmax, init=normal...................\n",
      "[CV 2/5; 2/12] END activation_function=softmax, init=normal;, score=0.584 total time=   1.7s\n",
      "[CV 3/5; 2/12] START activation_function=softmax, init=normal...................\n",
      "[CV 3/5; 2/12] END activation_function=softmax, init=normal;, score=0.630 total time=   1.5s\n",
      "[CV 4/5; 2/12] START activation_function=softmax, init=normal...................\n",
      "[CV 4/5; 2/12] END activation_function=softmax, init=normal;, score=0.745 total time=   0.9s\n",
      "[CV 5/5; 2/12] START activation_function=softmax, init=normal...................\n",
      "[CV 5/5; 2/12] END activation_function=softmax, init=normal;, score=0.353 total time=   1.1s\n",
      "[CV 1/5; 3/12] START activation_function=softmax, init=zero.....................\n",
      "[CV 1/5; 3/12] END activation_function=softmax, init=zero;, score=0.649 total time=   1.3s\n",
      "[CV 2/5; 3/12] START activation_function=softmax, init=zero.....................\n",
      "[CV 2/5; 3/12] END activation_function=softmax, init=zero;, score=0.416 total time=   1.5s\n",
      "[CV 3/5; 3/12] START activation_function=softmax, init=zero.....................\n",
      "[CV 3/5; 3/12] END activation_function=softmax, init=zero;, score=0.630 total time=   1.4s\n",
      "[CV 4/5; 3/12] START activation_function=softmax, init=zero.....................\n",
      "[CV 4/5; 3/12] END activation_function=softmax, init=zero;, score=0.745 total time=   1.0s\n",
      "[CV 5/5; 3/12] START activation_function=softmax, init=zero.....................\n",
      "[CV 5/5; 3/12] END activation_function=softmax, init=zero;, score=0.647 total time=   1.3s\n",
      "[CV 1/5; 4/12] START activation_function=relu, init=uniform.....................\n",
      "[CV 1/5; 4/12] END activation_function=relu, init=uniform;, score=0.747 total time=   1.6s\n",
      "[CV 2/5; 4/12] START activation_function=relu, init=uniform.....................\n",
      "[CV 2/5; 4/12] END activation_function=relu, init=uniform;, score=0.708 total time=   1.4s\n",
      "[CV 3/5; 4/12] START activation_function=relu, init=uniform.....................\n",
      "[CV 3/5; 4/12] END activation_function=relu, init=uniform;, score=0.766 total time=   1.1s\n",
      "[CV 4/5; 4/12] START activation_function=relu, init=uniform.....................\n",
      "[CV 4/5; 4/12] END activation_function=relu, init=uniform;, score=0.745 total time=   1.0s\n",
      "[CV 5/5; 4/12] START activation_function=relu, init=uniform.....................\n",
      "[CV 5/5; 4/12] END activation_function=relu, init=uniform;, score=0.778 total time=   1.2s\n",
      "[CV 1/5; 5/12] START activation_function=relu, init=normal......................\n",
      "[CV 1/5; 5/12] END activation_function=relu, init=normal;, score=0.740 total time=   0.9s\n",
      "[CV 2/5; 5/12] START activation_function=relu, init=normal......................\n",
      "[CV 2/5; 5/12] END activation_function=relu, init=normal;, score=0.727 total time=   1.5s\n",
      "[CV 3/5; 5/12] START activation_function=relu, init=normal......................\n",
      "[CV 3/5; 5/12] END activation_function=relu, init=normal;, score=0.786 total time=   1.8s\n",
      "[CV 4/5; 5/12] START activation_function=relu, init=normal......................\n",
      "[CV 4/5; 5/12] END activation_function=relu, init=normal;, score=0.804 total time=   1.6s\n",
      "[CV 5/5; 5/12] START activation_function=relu, init=normal......................\n",
      "[CV 5/5; 5/12] END activation_function=relu, init=normal;, score=0.758 total time=   1.0s\n",
      "[CV 1/5; 6/12] START activation_function=relu, init=zero........................\n",
      "[CV 1/5; 6/12] END activation_function=relu, init=zero;, score=0.649 total time=   1.0s\n",
      "[CV 2/5; 6/12] START activation_function=relu, init=zero........................\n",
      "[CV 2/5; 6/12] END activation_function=relu, init=zero;, score=0.584 total time=   1.1s\n",
      "[CV 3/5; 6/12] START activation_function=relu, init=zero........................\n",
      "[CV 3/5; 6/12] END activation_function=relu, init=zero;, score=0.630 total time=   1.2s\n",
      "[CV 4/5; 6/12] START activation_function=relu, init=zero........................\n",
      "[CV 4/5; 6/12] END activation_function=relu, init=zero;, score=0.745 total time=   1.2s\n",
      "[CV 5/5; 6/12] START activation_function=relu, init=zero........................\n",
      "[CV 5/5; 6/12] END activation_function=relu, init=zero;, score=0.647 total time=   0.9s\n",
      "[CV 1/5; 7/12] START activation_function=tanh, init=uniform.....................\n",
      "[CV 1/5; 7/12] END activation_function=tanh, init=uniform;, score=0.740 total time=   1.1s\n",
      "[CV 2/5; 7/12] START activation_function=tanh, init=uniform.....................\n",
      "[CV 2/5; 7/12] END activation_function=tanh, init=uniform;, score=0.708 total time=   1.2s\n",
      "[CV 3/5; 7/12] START activation_function=tanh, init=uniform.....................\n",
      "[CV 3/5; 7/12] END activation_function=tanh, init=uniform;, score=0.747 total time=   1.5s\n",
      "[CV 4/5; 7/12] START activation_function=tanh, init=uniform.....................\n",
      "[CV 4/5; 7/12] END activation_function=tanh, init=uniform;, score=0.824 total time=   1.3s\n",
      "[CV 5/5; 7/12] START activation_function=tanh, init=uniform.....................\n",
      "[CV 5/5; 7/12] END activation_function=tanh, init=uniform;, score=0.758 total time=   1.2s\n",
      "[CV 1/5; 8/12] START activation_function=tanh, init=normal......................\n",
      "[CV 1/5; 8/12] END activation_function=tanh, init=normal;, score=0.747 total time=   1.6s\n",
      "[CV 2/5; 8/12] START activation_function=tanh, init=normal......................\n",
      "[CV 2/5; 8/12] END activation_function=tanh, init=normal;, score=0.688 total time=   1.2s\n",
      "[CV 3/5; 8/12] START activation_function=tanh, init=normal......................\n",
      "[CV 3/5; 8/12] END activation_function=tanh, init=normal;, score=0.760 total time=   1.3s\n",
      "[CV 4/5; 8/12] START activation_function=tanh, init=normal......................\n",
      "[CV 4/5; 8/12] END activation_function=tanh, init=normal;, score=0.810 total time=   1.9s\n",
      "[CV 5/5; 8/12] START activation_function=tanh, init=normal......................\n",
      "[CV 5/5; 8/12] END activation_function=tanh, init=normal;, score=0.771 total time=   1.2s\n",
      "[CV 1/5; 9/12] START activation_function=tanh, init=zero........................\n",
      "[CV 1/5; 9/12] END activation_function=tanh, init=zero;, score=0.649 total time=   1.1s\n",
      "[CV 2/5; 9/12] START activation_function=tanh, init=zero........................\n",
      "[CV 2/5; 9/12] END activation_function=tanh, init=zero;, score=0.584 total time=   1.0s\n",
      "[CV 3/5; 9/12] START activation_function=tanh, init=zero........................\n",
      "[CV 3/5; 9/12] END activation_function=tanh, init=zero;, score=0.630 total time=   1.2s\n",
      "[CV 4/5; 9/12] START activation_function=tanh, init=zero........................\n",
      "[CV 4/5; 9/12] END activation_function=tanh, init=zero;, score=0.745 total time=   1.5s\n",
      "[CV 5/5; 9/12] START activation_function=tanh, init=zero........................\n",
      "[CV 5/5; 9/12] END activation_function=tanh, init=zero;, score=0.647 total time=   1.2s\n",
      "[CV 1/5; 10/12] START activation_function=linear, init=uniform..................\n",
      "[CV 1/5; 10/12] END activation_function=linear, init=uniform;, score=0.760 total time=   0.9s\n",
      "[CV 2/5; 10/12] START activation_function=linear, init=uniform..................\n",
      "[CV 2/5; 10/12] END activation_function=linear, init=uniform;, score=0.695 total time=   1.5s\n",
      "[CV 3/5; 10/12] START activation_function=linear, init=uniform..................\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 3/5; 10/12] END activation_function=linear, init=uniform;, score=0.766 total time=   1.0s\n",
      "[CV 4/5; 10/12] START activation_function=linear, init=uniform..................\n",
      "[CV 4/5; 10/12] END activation_function=linear, init=uniform;, score=0.837 total time=   1.0s\n",
      "[CV 5/5; 10/12] START activation_function=linear, init=uniform..................\n",
      "[CV 5/5; 10/12] END activation_function=linear, init=uniform;, score=0.765 total time=   1.1s\n",
      "[CV 1/5; 11/12] START activation_function=linear, init=normal...................\n",
      "[CV 1/5; 11/12] END activation_function=linear, init=normal;, score=0.740 total time=   1.4s\n",
      "[CV 2/5; 11/12] START activation_function=linear, init=normal...................\n",
      "[CV 2/5; 11/12] END activation_function=linear, init=normal;, score=0.688 total time=   1.0s\n",
      "[CV 3/5; 11/12] START activation_function=linear, init=normal...................\n",
      "[CV 3/5; 11/12] END activation_function=linear, init=normal;, score=0.753 total time=   1.1s\n",
      "[CV 4/5; 11/12] START activation_function=linear, init=normal...................\n",
      "[CV 4/5; 11/12] END activation_function=linear, init=normal;, score=0.824 total time=   0.9s\n",
      "[CV 5/5; 11/12] START activation_function=linear, init=normal...................\n",
      "[CV 5/5; 11/12] END activation_function=linear, init=normal;, score=0.758 total time=   1.3s\n",
      "[CV 1/5; 12/12] START activation_function=linear, init=zero.....................\n",
      "[CV 1/5; 12/12] END activation_function=linear, init=zero;, score=0.649 total time=   1.0s\n",
      "[CV 2/5; 12/12] START activation_function=linear, init=zero.....................\n",
      "[CV 2/5; 12/12] END activation_function=linear, init=zero;, score=0.584 total time=   0.9s\n",
      "[CV 3/5; 12/12] START activation_function=linear, init=zero.....................\n",
      "[CV 3/5; 12/12] END activation_function=linear, init=zero;, score=0.630 total time=   0.9s\n",
      "[CV 4/5; 12/12] START activation_function=linear, init=zero.....................\n",
      "[CV 4/5; 12/12] END activation_function=linear, init=zero;, score=0.745 total time=   1.0s\n",
      "[CV 5/5; 12/12] START activation_function=linear, init=zero.....................\n",
      "[CV 5/5; 12/12] END activation_function=linear, init=zero;, score=0.647 total time=   0.9s\n"
     ]
    }
   ],
   "source": [
    "# Defining the model\n",
    "\n",
    "def create_model(activation_function,init):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(8,input_dim = 8,kernel_initializer = init,activation = activation_function))\n",
    "    model.add(Dropout(0.1))\n",
    "    model.add(Dense(4,input_dim = 8,kernel_initializer = init,activation = activation_function))\n",
    "    model.add(Dropout(0.1))\n",
    "    model.add(Dense(1,activation = 'sigmoid'))\n",
    "    \n",
    "    adam = Adam(lr = 0.001)\n",
    "    model.compile(loss = 'binary_crossentropy',optimizer = adam,metrics = ['accuracy'])\n",
    "    return model\n",
    "\n",
    "# Create the model\n",
    "\n",
    "model = KerasClassifier(build_fn = create_model,verbose = 0,batch_size = 40,epochs = 10)\n",
    "\n",
    "# Define the grid search parameters\n",
    "activation_function = ['softmax','relu','tanh','linear']\n",
    "init = ['uniform','normal','zero']\n",
    "\n",
    "# Make a dictionary of the grid search parameters\n",
    "param_grids = dict(activation_function = activation_function,init = init)\n",
    "\n",
    "# Build and fit the GridSearchCV\n",
    "\n",
    "grid = GridSearchCV(estimator = model,param_grid = param_grids,cv = KFold(),verbose = 10)\n",
    "grid_result = grid.fit(X_standardized,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e7c4e155",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best : 0.7644172787666321, using {'activation_function': 'linear', 'init': 'uniform'}\n",
      "0.6511586427688598,0.05244526932680711 with: {'activation_function': 'softmax', 'init': 'uniform'}\n",
      "0.5923351109027862,0.1306662431539892 with: {'activation_function': 'softmax', 'init': 'normal'}\n",
      "0.6173924148082733,0.10871275688841411 with: {'activation_function': 'softmax', 'init': 'zero'}\n",
      "0.7487310171127319,0.023845986018844377 with: {'activation_function': 'relu', 'init': 'uniform'}\n",
      "0.7630676627159119,0.02831033352711466 with: {'activation_function': 'relu', 'init': 'normal'}\n",
      "0.6511586427688598,0.05244526932680711 with: {'activation_function': 'relu', 'init': 'zero'}\n",
      "0.7553009271621705,0.03799930187313782 with: {'activation_function': 'tanh', 'init': 'uniform'}\n",
      "0.7553009152412414,0.03969052852138509 with: {'activation_function': 'tanh', 'init': 'normal'}\n",
      "0.6511586427688598,0.05244526932680711 with: {'activation_function': 'tanh', 'init': 'zero'}\n",
      "0.7644172787666321,0.04490352723485743 with: {'activation_function': 'linear', 'init': 'uniform'}\n",
      "0.7527035117149353,0.04323803728794573 with: {'activation_function': 'linear', 'init': 'normal'}\n",
      "0.6511586427688598,0.05244526932680711 with: {'activation_function': 'linear', 'init': 'zero'}\n"
     ]
    }
   ],
   "source": [
    "# Summarize the results\n",
    "print('Best : {}, using {}'.format(grid_result.best_score_,grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "  print('{},{} with: {}'.format(mean, stdev, param))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1dfb35f5",
   "metadata": {},
   "source": [
    "#### Tuning of Hyperparameter :-Number of Neurons in activation layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "9c0eb00a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 9 candidates, totalling 45 fits\n",
      "[CV 1/5; 1/9] START neuron1=4, neuron2=2........................................\n",
      "[CV 1/5; 1/9] END .........neuron1=4, neuron2=2;, score=0.753 total time=   1.0s\n",
      "[CV 2/5; 1/9] START neuron1=4, neuron2=2........................................\n",
      "[CV 2/5; 1/9] END .........neuron1=4, neuron2=2;, score=0.695 total time=   1.0s\n",
      "[CV 3/5; 1/9] START neuron1=4, neuron2=2........................................\n",
      "[CV 3/5; 1/9] END .........neuron1=4, neuron2=2;, score=0.740 total time=   1.0s\n",
      "[CV 4/5; 1/9] START neuron1=4, neuron2=2........................................\n",
      "[CV 4/5; 1/9] END .........neuron1=4, neuron2=2;, score=0.810 total time=   1.0s\n",
      "[CV 5/5; 1/9] START neuron1=4, neuron2=2........................................\n",
      "[CV 5/5; 1/9] END .........neuron1=4, neuron2=2;, score=0.725 total time=   0.9s\n",
      "[CV 1/5; 2/9] START neuron1=4, neuron2=4........................................\n",
      "[CV 1/5; 2/9] END .........neuron1=4, neuron2=4;, score=0.760 total time=   1.0s\n",
      "[CV 2/5; 2/9] START neuron1=4, neuron2=4........................................\n",
      "[CV 2/5; 2/9] END .........neuron1=4, neuron2=4;, score=0.682 total time=   0.9s\n",
      "[CV 3/5; 2/9] START neuron1=4, neuron2=4........................................\n",
      "[CV 3/5; 2/9] END .........neuron1=4, neuron2=4;, score=0.753 total time=   0.9s\n",
      "[CV 4/5; 2/9] START neuron1=4, neuron2=4........................................\n",
      "[CV 4/5; 2/9] END .........neuron1=4, neuron2=4;, score=0.797 total time=   0.9s\n",
      "[CV 5/5; 2/9] START neuron1=4, neuron2=4........................................\n",
      "[CV 5/5; 2/9] END .........neuron1=4, neuron2=4;, score=0.771 total time=   1.2s\n",
      "[CV 1/5; 3/9] START neuron1=4, neuron2=8........................................\n",
      "[CV 1/5; 3/9] END .........neuron1=4, neuron2=8;, score=0.760 total time=   1.0s\n",
      "[CV 2/5; 3/9] START neuron1=4, neuron2=8........................................\n",
      "[CV 2/5; 3/9] END .........neuron1=4, neuron2=8;, score=0.682 total time=   0.9s\n",
      "[CV 3/5; 3/9] START neuron1=4, neuron2=8........................................\n",
      "[CV 3/5; 3/9] END .........neuron1=4, neuron2=8;, score=0.740 total time=   1.0s\n",
      "[CV 4/5; 3/9] START neuron1=4, neuron2=8........................................\n",
      "[CV 4/5; 3/9] END .........neuron1=4, neuron2=8;, score=0.824 total time=   1.0s\n",
      "[CV 5/5; 3/9] START neuron1=4, neuron2=8........................................\n",
      "[CV 5/5; 3/9] END .........neuron1=4, neuron2=8;, score=0.752 total time=   0.9s\n",
      "[CV 1/5; 4/9] START neuron1=8, neuron2=2........................................\n",
      "[CV 1/5; 4/9] END .........neuron1=8, neuron2=2;, score=0.747 total time=   1.1s\n",
      "[CV 2/5; 4/9] START neuron1=8, neuron2=2........................................\n",
      "[CV 2/5; 4/9] END .........neuron1=8, neuron2=2;, score=0.688 total time=   1.0s\n",
      "[CV 3/5; 4/9] START neuron1=8, neuron2=2........................................\n",
      "[CV 3/5; 4/9] END .........neuron1=8, neuron2=2;, score=0.766 total time=   0.9s\n",
      "[CV 4/5; 4/9] START neuron1=8, neuron2=2........................................\n",
      "[CV 4/5; 4/9] END .........neuron1=8, neuron2=2;, score=0.804 total time=   1.0s\n",
      "[CV 5/5; 4/9] START neuron1=8, neuron2=2........................................\n",
      "[CV 5/5; 4/9] END .........neuron1=8, neuron2=2;, score=0.758 total time=   0.9s\n",
      "[CV 1/5; 5/9] START neuron1=8, neuron2=4........................................\n",
      "[CV 1/5; 5/9] END .........neuron1=8, neuron2=4;, score=0.760 total time=   0.9s\n",
      "[CV 2/5; 5/9] START neuron1=8, neuron2=4........................................\n",
      "[CV 2/5; 5/9] END .........neuron1=8, neuron2=4;, score=0.688 total time=   0.9s\n",
      "[CV 3/5; 5/9] START neuron1=8, neuron2=4........................................\n",
      "[CV 3/5; 5/9] END .........neuron1=8, neuron2=4;, score=0.747 total time=   0.9s\n",
      "[CV 4/5; 5/9] START neuron1=8, neuron2=4........................................\n",
      "[CV 4/5; 5/9] END .........neuron1=8, neuron2=4;, score=0.837 total time=   0.9s\n",
      "[CV 5/5; 5/9] START neuron1=8, neuron2=4........................................\n",
      "[CV 5/5; 5/9] END .........neuron1=8, neuron2=4;, score=0.758 total time=   0.9s\n",
      "[CV 1/5; 6/9] START neuron1=8, neuron2=8........................................\n",
      "[CV 1/5; 6/9] END .........neuron1=8, neuron2=8;, score=0.760 total time=   1.3s\n",
      "[CV 2/5; 6/9] START neuron1=8, neuron2=8........................................\n",
      "[CV 2/5; 6/9] END .........neuron1=8, neuron2=8;, score=0.695 total time=   0.9s\n",
      "[CV 3/5; 6/9] START neuron1=8, neuron2=8........................................\n",
      "[CV 3/5; 6/9] END .........neuron1=8, neuron2=8;, score=0.760 total time=   1.0s\n",
      "[CV 4/5; 6/9] START neuron1=8, neuron2=8........................................\n",
      "[CV 4/5; 6/9] END .........neuron1=8, neuron2=8;, score=0.817 total time=   1.0s\n",
      "[CV 5/5; 6/9] START neuron1=8, neuron2=8........................................\n",
      "[CV 5/5; 6/9] END .........neuron1=8, neuron2=8;, score=0.758 total time=   1.0s\n",
      "[CV 1/5; 7/9] START neuron1=16, neuron2=2.......................................\n",
      "[CV 1/5; 7/9] END ........neuron1=16, neuron2=2;, score=0.760 total time=   1.0s\n",
      "[CV 2/5; 7/9] START neuron1=16, neuron2=2.......................................\n",
      "[CV 2/5; 7/9] END ........neuron1=16, neuron2=2;, score=0.721 total time=   1.0s\n",
      "[CV 3/5; 7/9] START neuron1=16, neuron2=2.......................................\n",
      "[CV 3/5; 7/9] END ........neuron1=16, neuron2=2;, score=0.753 total time=   0.9s\n",
      "[CV 4/5; 7/9] START neuron1=16, neuron2=2.......................................\n",
      "[CV 4/5; 7/9] END ........neuron1=16, neuron2=2;, score=0.824 total time=   0.9s\n",
      "[CV 5/5; 7/9] START neuron1=16, neuron2=2.......................................\n",
      "[CV 5/5; 7/9] END ........neuron1=16, neuron2=2;, score=0.771 total time=   1.0s\n",
      "[CV 1/5; 8/9] START neuron1=16, neuron2=4.......................................\n",
      "[CV 1/5; 8/9] END ........neuron1=16, neuron2=4;, score=0.773 total time=   0.9s\n",
      "[CV 2/5; 8/9] START neuron1=16, neuron2=4.......................................\n",
      "[CV 2/5; 8/9] END ........neuron1=16, neuron2=4;, score=0.740 total time=   0.9s\n",
      "[CV 3/5; 8/9] START neuron1=16, neuron2=4.......................................\n",
      "[CV 3/5; 8/9] END ........neuron1=16, neuron2=4;, score=0.760 total time=   0.9s\n",
      "[CV 4/5; 8/9] START neuron1=16, neuron2=4.......................................\n",
      "[CV 4/5; 8/9] END ........neuron1=16, neuron2=4;, score=0.830 total time=   0.9s\n",
      "[CV 5/5; 8/9] START neuron1=16, neuron2=4.......................................\n",
      "[CV 5/5; 8/9] END ........neuron1=16, neuron2=4;, score=0.771 total time=   0.9s\n",
      "[CV 1/5; 9/9] START neuron1=16, neuron2=8.......................................\n",
      "[CV 1/5; 9/9] END ........neuron1=16, neuron2=8;, score=0.773 total time=   0.9s\n",
      "[CV 2/5; 9/9] START neuron1=16, neuron2=8.......................................\n",
      "[CV 2/5; 9/9] END ........neuron1=16, neuron2=8;, score=0.721 total time=   1.2s\n",
      "[CV 3/5; 9/9] START neuron1=16, neuron2=8.......................................\n",
      "[CV 3/5; 9/9] END ........neuron1=16, neuron2=8;, score=0.753 total time=   0.9s\n",
      "[CV 4/5; 9/9] START neuron1=16, neuron2=8.......................................\n",
      "[CV 4/5; 9/9] END ........neuron1=16, neuron2=8;, score=0.810 total time=   1.0s\n",
      "[CV 5/5; 9/9] START neuron1=16, neuron2=8.......................................\n",
      "[CV 5/5; 9/9] END ........neuron1=16, neuron2=8;, score=0.758 total time=   0.9s\n"
     ]
    }
   ],
   "source": [
    "# Defining the model\n",
    "\n",
    "def create_model(neuron1,neuron2):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(neuron1,input_dim = 8,kernel_initializer = 'uniform',activation = 'tanh'))\n",
    "    model.add(Dropout(0.1))\n",
    "    model.add(Dense(neuron2,input_dim = neuron1,kernel_initializer = 'uniform',activation = 'tanh'))\n",
    "    model.add(Dropout(0.1))\n",
    "    model.add(Dense(1,activation = 'sigmoid'))\n",
    "    \n",
    "    adam = Adam(lr = 0.001)\n",
    "    model.compile(loss = 'binary_crossentropy',optimizer = adam,metrics = ['accuracy'])\n",
    "    return model\n",
    "\n",
    "# Create the model\n",
    "\n",
    "model = KerasClassifier(build_fn = create_model,verbose = 0,batch_size = 40,epochs = 10)\n",
    "\n",
    "# Define the grid search parameters\n",
    "\n",
    "neuron1 = [4,8,16]\n",
    "neuron2 = [2,4,8]\n",
    "\n",
    "# Make a dictionary of the grid search parameters\n",
    "\n",
    "param_grids = dict(neuron1 = neuron1,neuron2 = neuron2)\n",
    "\n",
    "# Build and fit the GridSearchCV\n",
    "\n",
    "grid = GridSearchCV(estimator = model,param_grid = param_grids,cv = KFold(),verbose = 10)\n",
    "grid_result = grid.fit(X_standardized,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3f747197",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best : 0.7748068928718567, using {'neuron1': 16, 'neuron2': 4}\n",
      "0.7448518872261047,0.03814500173280944 with: {'neuron1': 4, 'neuron2': 2}\n",
      "0.7526865243911743,0.038508960076189196 with: {'neuron1': 4, 'neuron2': 4}\n",
      "0.7513963222503662,0.04525041703609891 with: {'neuron1': 4, 'neuron2': 8}\n",
      "0.7526780486106872,0.03746451019558621 with: {'neuron1': 8, 'neuron2': 2}\n",
      "0.7579152941703796,0.04725280301533656 with: {'neuron1': 8, 'neuron2': 4}\n",
      "0.7578898191452026,0.03867773837216689 with: {'neuron1': 8, 'neuron2': 8}\n",
      "0.7657074928283691,0.03341685618184123 with: {'neuron1': 16, 'neuron2': 2}\n",
      "0.7748068928718567,0.029970124787145606 with: {'neuron1': 16, 'neuron2': 4}\n",
      "0.7630761384963989,0.02914740363768991 with: {'neuron1': 16, 'neuron2': 8}\n"
     ]
    }
   ],
   "source": [
    "# Summarize the results\n",
    "print('Best : {}, using {}'.format(grid_result.best_score_,grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "  print('{},{} with: {}'.format(mean, stdev, param))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be8a7c06",
   "metadata": {},
   "source": [
    "#### Training model with optimum values of Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "8dc50a69",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 0s 4ms/step\n",
      "0.77734375\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report, accuracy_score\n",
    "\n",
    "# Defining the model\n",
    "\n",
    "def create_model():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(16,input_dim = 8,kernel_initializer = 'uniform',activation = 'tanh'))\n",
    "    model.add(Dropout(0.1))\n",
    "    model.add(Dense(4,input_dim = 16,kernel_initializer = 'uniform',activation = 'tanh'))\n",
    "    model.add(Dropout(0.1))\n",
    "    model.add(Dense(1,activation = 'sigmoid'))\n",
    "    \n",
    "    adam = Adam(lr = 0.001) #sgd = SGD(lr=learning_rate, momentum=momentum, decay=decay_rate, nesterov=False)\n",
    "    model.compile(loss = 'binary_crossentropy',optimizer = adam,metrics = ['accuracy'])\n",
    "    return model\n",
    "\n",
    "# Create the model\n",
    "\n",
    "model = KerasClassifier(build_fn = create_model,verbose = 0,batch_size = 40,epochs = 10)\n",
    "\n",
    "# Fitting the model\n",
    "\n",
    "model.fit(X_standardized,y)\n",
    "\n",
    "# Predicting using trained model\n",
    "\n",
    "y_predict = model.predict(X_standardized)\n",
    "\n",
    "# Printing the metrics\n",
    "print(accuracy_score(y,y_predict))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "861a1acb",
   "metadata": {},
   "source": [
    "# Hyperparameters all at once"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf54bb92",
   "metadata": {},
   "source": [
    "The hyperparameter optimization was carried out by taking 2 hyperparameters at once. We may have missed the best values. The performance can be further improved by finding the optimum values of hyperparameters all at once given by the code snippet below.\n",
    "#### This process is computationally expensive."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc8396ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model(learning_rate,dropout_rate,activation_function,init,neuron1,neuron2):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(neuron1,input_dim = 8,kernel_initializer = init,activation = activation_function))\n",
    "    model.add(Dropout(dropout_rate))\n",
    "    model.add(Dense(neuron2,input_dim = neuron1,kernel_initializer = init,activation = activation_function))\n",
    "    model.add(Dropout(dropout_rate))\n",
    "    model.add(Dense(1,activation = 'sigmoid'))\n",
    "    \n",
    "    adam = Adam(lr = learning_rate)\n",
    "    model.compile(loss = 'binary_crossentropy',optimizer = adam,metrics = ['accuracy'])\n",
    "    return model\n",
    "\n",
    "# Create the model\n",
    "\n",
    "model = KerasClassifier(build_fn = create_model,verbose = 0)\n",
    "\n",
    "# Define the grid search parameters\n",
    "\n",
    "batch_size = [10,20,40]\n",
    "epochs = [10,50,100]\n",
    "learning_rate = [0.001,0.01,0.1]\n",
    "dropout_rate = [0.0,0.1,0.2]\n",
    "activation_function = ['softmax','relu','tanh','linear']\n",
    "init = ['uniform','normal','zero']\n",
    "neuron1 = [4,8,16]\n",
    "neuron2 = [2,4,8]\n",
    "\n",
    "# Make a dictionary of the grid search parameters\n",
    "\n",
    "param_grids = dict(batch_size = batch_size,epochs = epochs,learning_rate = learning_rate,dropout_rate = dropout_rate,\n",
    "                   activation_function = activation_function,init = init,neuron1 = neuron1,neuron2 = neuron2)\n",
    "\n",
    "# Build and fit the GridSearchCV\n",
    "\n",
    "grid = GridSearchCV(estimator = model,param_grid = param_grids,cv = KFold(),verbose = 10)\n",
    "grid_result = grid.fit(X_standardized,y)\n",
    "\n",
    "# Summarize the results\n",
    "print('Best : {}, using {}'.format(grid_result.best_score_,grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "  print('{},{} with: {}'.format(mean, stdev, param))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}