Model_2nd_Dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# Load the Parkinson's Disease dataset\n",
    "# Load the Parkinson's Disease dataset\n",
    "df = pd.read_csv('pd_speech_features.csv')\n",
    "\n",
    "# Data cleaning\n",
    "df = df.drop('id', axis=1)  # Remove the 'name' column\n",
    "\n",
    "# Data preprocessing\n",
    "X = df.drop('class', axis=1)  # Features\n",
    "Y = df['class']  # Target variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['gender', 'PPE', 'DFA', 'RPDE', 'numPulses', 'numPeriodsPulses',\n",
      "       'meanPeriodPulses', 'stdDevPeriodPulses', 'locPctJitter',\n",
      "       'locAbsJitter',\n",
      "       ...\n",
      "       'tqwt_kurtosisValue_dec_28', 'tqwt_kurtosisValue_dec_29',\n",
      "       'tqwt_kurtosisValue_dec_30', 'tqwt_kurtosisValue_dec_31',\n",
      "       'tqwt_kurtosisValue_dec_32', 'tqwt_kurtosisValue_dec_33',\n",
      "       'tqwt_kurtosisValue_dec_34', 'tqwt_kurtosisValue_dec_35',\n",
      "       'tqwt_kurtosisValue_dec_36', 'class'],\n",
      "      dtype='object', length=754)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 528/528 [01:15<00:00,  7.00it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['std_6th_delta_delta', 'tqwt_stdValue_dec_26', 'mean_2nd_delta', 'tqwt_TKEO_mean_dec_33', 'tqwt_maxValue_dec_11', 'std_9th_delta_delta', 'std_delta_delta_log_energy', 'mean_MFCC_2nd_coef', 'std_7th_delta_delta', 'tqwt_entropy_log_dec_12', 'tqwt_kurtosisValue_dec_33', 'std_8th_delta_delta', 'tqwt_entropy_log_dec_26', 'std_6th_delta', 'tqwt_minValue_dec_12', 'std_11th_delta_delta', 'DFA', 'std_9th_delta', 'tqwt_maxValue_dec_12', 'locPctJitter', 'std_10th_delta_delta', 'tqwt_energy_dec_6', 'std_8th_delta', 'tqwt_energy_dec_25', 'std_7th_delta', 'tqwt_entropy_log_dec_13', 'app_det_TKEO_mean_7_coef', 'std_10th_delta', 'tqwt_maxValue_dec_13', 'std_11th_delta', 'tqwt_stdValue_dec_12', 'std_4th_delta_delta', 'tqwt_stdValue_dec_11', 'tqwt_meanValue_dec_25', 'tqwt_entropy_log_dec_16', 'std_4th_delta', 'tqwt_entropy_log_dec_27', 'app_LT_TKEO_std_7_coef', 'std_12th_delta_delta', 'tqwt_minValue_dec_13', 'std_delta_log_energy', 'tqwt_minValue_dec_11', 'std_5th_delta_delta', 'app_TKEO_std_8_coef', 'tqwt_kurtosisValue_dec_20', 'tqwt_kurtosisValue_dec_36', 'ppq5Jitter', 'tqwt_energy_dec_15', 'std_5th_delta', 'tqwt_kurtosisValue_dec_26', 'tqwt_entropy_shannon_dec_17', 'std_12th_delta', 'app_det_TKEO_mean_8_coef', 'tqwt_entropy_log_dec_15', 'tqwt_energy_dec_26', 'RPDE', 'tqwt_entropy_log_dec_11', 'f1', 'std_MFCC_6th_coef', 'app_det_TKEO_mean_9_coef', 'tqwt_stdValue_dec_15', 'tqwt_TKEO_std_dec_11', 'tqwt_kurtosisValue_dec_27', 'tqwt_entropy_shannon_dec_16', 'locAbsJitter', 'tqwt_energy_dec_21', 'tqwt_entropy_log_dec_14', 'app_det_TKEO_mean_10_coef', 'tqwt_kurtosisValue_dec_34', 'tqwt_entropy_log_dec_17', 'tqwt_stdValue_dec_13', 'app_LT_TKEO_std_8_coef', 'tqwt_maxValue_dec_10', 'std_3rd_delta_delta', 'minIntensity', 'app_LT_TKEO_mean_7_coef', 'tqwt_entropy_shannon_dec_11', 'std_MFCC_8th_coef', 'tqwt_entropy_shannon_dec_15', 'tqwt_stdValue_dec_7', 'std_MFCC_7th_coef', 'app_LT_TKEO_mean_8_coef', 'tqwt_TKEO_std_dec_12', 'tqwt_maxValue_dec_14', 'app_LT_TKEO_mean_9_coef', 'apq11Shimmer', 'tqwt_kurtosisValue_dec_18', 'IMF_SNR_SEO', 'std_3rd_delta', 'tqwt_energy_dec_14', 'app_det_TKEO_mean_6_coef', 'tqwt_stdValue_dec_6', 'gender', 'mean_delta_log_energy', 'det_LT_entropy_log_3_coef', 'tqwt_stdValue_dec_14', 'app_LT_TKEO_mean_6_coef', 'tqwt_energy_dec_12', 'std_MFCC_11th_coef', 'tqwt_entropy_log_dec_34', 'app_LT_TKEO_mean_10_coef', 'meanHarmToNoiseHarmonicity', 'tqwt_stdValue_dec_16', 'app_TKEO_std_10_coef', 'tqwt_minValue_dec_10', 'tqwt_minValue_dec_14', 'app_LT_TKEO_std_10_coef', 'ddpJitter', 'tqwt_maxValue_dec_7', 'app_LT_TKEO_mean_5_coef', 'tqwt_TKEO_mean_dec_11', 'tqwt_kurtosisValue_dec_17', 'mean_MFCC_3rd_coef', 'meanIntensity', 'app_LT_TKEO_std_9_coef', 'std_MFCC_10th_coef', 'tqwt_entropy_log_dec_8', 'tqwt_entropy_shannon_dec_14', 'app_TKEO_std_9_coef', 'rapJitter', 'app_LT_TKEO_std_5_coef', 'tqwt_maxValue_dec_6', 'tqwt_stdValue_dec_10', 'tqwt_entropy_log_dec_18', 'app_det_TKEO_mean_5_coef', 'tqwt_energy_dec_16', 'app_LT_TKEO_std_6_coef', 'std_MFCC_4th_coef', 'tqwt_entropy_log_dec_7', 'IMF_SNR_entropy', 'app_entropy_shannon_10_coef', 'tqwt_entropy_shannon_dec_13', 'std_Log_energy', 'tqwt_entropy_log_dec_10', 'app_entropy_shannon_9_coef', 'tqwt_entropy_log_dec_35', 'app_LT_entropy_shannon_5_coef', 'tqwt_minValue_dec_7', 'app_LT_entropy_shannon_8_coef', 'tqwt_stdValue_dec_8', 'app_LT_TKEO_std_4_coef', 'tqwt_kurtosisValue_dec_28', 'tqwt_energy_dec_13', 'std_MFCC_9th_coef', 'app_LT_entropy_shannon_4_coef', 'tqwt_entropy_shannon_dec_12', 'tqwt_entropy_log_dec_33', 'tqwt_energy_dec_27', 'det_LT_entropy_log_4_coef', 'app_LT_entropy_shannon_6_coef', 'tqwt_entropy_log_dec_9', 'app_LT_entropy_shannon_9_coef', 'tqwt_minValue_dec_15', 'app_LT_entropy_shannon_10_coef', 'tqwt_maxValue_dec_9', 'app_entropy_shannon_8_coef', 'tqwt_maxValue_dec_15', 'tqwt_entropy_shannon_dec_18', 'app_entropy_shannon_7_coef', 'tqwt_TKEO_mean_dec_15', 'tqwt_entropy_log_dec_20', 'app_LT_entropy_shannon_7_coef', 'tqwt_minValue_dec_8', 'app_TKEO_std_6_coef', 'tqwt_entropy_shannon_dec_10', 'VFER_entropy', 'app_LT_entropy_shannon_3_coef', 'std_2nd_delta', 'det_LT_entropy_log_2_coef', 'app_LT_TKEO_mean_4_coef', 'maxIntensity', 'tqwt_stdValue_dec_9', 'VFER_mean', 'app_LT_TKEO_std_3_coef', 'std_MFCC_5th_coef', 'tqwt_TKEO_std_dec_15', 'app_entropy_log_9_coef', 'tqwt_maxValue_dec_8', 'app_entropy_shannon_6_coef', 'tqwt_entropy_shannon_dec_34', 'std_MFCC_12th_coef', 'app_entropy_log_8_coef', 'tqwt_entropy_log_dec_6', 'app_LT_entropy_shannon_2_coef', 'det_LT_entropy_log_5_coef', 'app_entropy_log_10_coef', 'tqwt_kurtosisValue_dec_19', 'tqwt_TKEO_std_dec_10', 'app_LT_entropy_shannon_1_coef', 'locDbShimmer', 'app_TKEO_std_5_coef', 'tqwt_kurtosisValue_dec_35', 'tqwt_entropy_log_dec_19', 'app_TKEO_std_7_coef', 'mean_0th_delta', 'std_2nd_delta_delta', 'tqwt_TKEO_std_dec_14', 'app_entropy_shannon_5_coef', 'numPeriodsPulses', 'VFER_NSR_SEO', 'tqwt_minValue_dec_6', 'tqwt_TKEO_mean_dec_16', 'mean_Log_energy', 'tqwt_meanValue_dec_11', 'numPulses', 'tqwt_entropy_log_dec_23', 'app_entropy_shannon_4_coef', 'tqwt_entropy_shannon_dec_9', 'app_det_TKEO_mean_4_coef', 'tqwt_minValue_dec_9', 'tqwt_entropy_log_dec_28', 'app_entropy_log_6_coef', 'app_TKEO_std_4_coef', 'std_0th_delta', 'app_LT_TKEO_std_2_coef', 'tqwt_TKEO_mean_dec_10', 'app_LT_TKEO_mean_3_coef', 'app_entropy_log_7_coef', 'tqwt_entropy_shannon_dec_7', 'tqwt_maxValue_dec_16', 'app_entropy_shannon_3_coef', 'tqwt_kurtosisValue_dec_16', 'app_LT_entropy_log_9_coef', 'tqwt_TKEO_std_dec_13', 'mean_MFCC_5th_coef', 'app_LT_entropy_log_8_coef', 'tqwt_minValue_dec_16', 'app_LT_entropy_log_10_coef', 'std_delta_delta_0th', 'locShimmer', 'app_entropy_shannon_2_coef', 'tqwt_entropy_shannon_dec_8', 'GQ_std_cycle_open', 'f2', 'tqwt_stdValue_dec_17', 'app_entropy_shannon_1_coef', 'tqwt_skewnessValue_dec_11', 'app_entropy_log_5_coef', 'tqwt_TKEO_std_dec_16', 'tqwt_entropy_shannon_dec_35', 'tqwt_TKEO_mean_dec_9', 'det_LT_entropy_log_6_coef', 'tqwt_entropy_log_dec_5', 'tqwt_TKEO_mean_dec_12', 'tqwt_meanValue_dec_5', 'app_det_TKEO_mean_3_coef', 'tqwt_TKEO_std_dec_7', 'tqwt_TKEO_std_dec_9', 'tqwt_entropy_shannon_dec_19', 'app_entropy_log_4_coef', 'tqwt_energy_dec_11', 'GNE_SNR_SEO', 'apq5Shimmer', 'tqwt_minValue_dec_26', 'det_LT_entropy_log_1_coef', 'tqwt_TKEO_mean_dec_14', 'app_LT_TKEO_std_1_coef', 'tqwt_maxValue_dec_1', 'app_LT_TKEO_mean_2_coef', 'tqwt_energy_dec_8', 'std_1st_delta', 'mean_MFCC_7th_coef', 'tqwt_TKEO_mean_dec_7', 'app_LT_entropy_log_6_coef', 'tqwt_stdValue_dec_34', 'stdDevPeriodPulses', 'tqwt_stdValue_dec_27', 'tqwt_TKEO_std_dec_8', 'tqwt_meanValue_dec_2', 'app_entropy_log_3_coef', 'tqwt_entropy_log_dec_36', 'meanAutoCorrHarmonicity', 'app_LT_entropy_log_7_coef', 'std_MFCC_0th_coef', 'tqwt_TKEO_mean_dec_27', 'app_TKEO_std_3_coef', 'GQ_std_cycle_closed', 'tqwt_maxValue_dec_26', 'tqwt_maxValue_dec_5', 'det_entropy_log_3_coef', 'app_TKEO_std_2_coef', 'tqwt_minValue_dec_27', 'app_entropy_log_2_coef', 'tqwt_TKEO_mean_dec_8', 'app_entropy_log_1_coef', 'tqwt_stdValue_dec_35', 'tqwt_meanValue_dec_7', 'tqwt_entropy_shannon_dec_6', 'tqwt_stdValue_dec_23', 'app_det_TKEO_mean_2_coef', 'tqwt_maxValue_dec_27', 'tqwt_entropy_shannon_dec_20', 'app_LT_TKEO_mean_1_coef', 'meanNoiseToHarmHarmonicity', 'tqwt_minValue_dec_5', 'mean_MFCC_0th_coef', 'app_LT_entropy_log_5_coef', 'tqwt_minValue_dec_1', 'tqwt_energy_dec_9', 'GQ_prc5_95', 'tqwt_energy_dec_7', 'tqwt_entropy_log_dec_4', 'b4', 'tqwt_TKEO_std_dec_27', 'tqwt_skewnessValue_dec_17', 'std_MFCC_3rd_coef', 'tqwt_skewnessValue_dec_10', 'tqwt_energy_dec_17', 'tqwt_stdValue_dec_5', 'tqwt_TKEO_mean_dec_23', 'apq3Shimmer', 'tqwt_energy_dec_28', 'std_1st_delta_delta', 'ddaShimmer', 'det_LT_entropy_shannon_3_coef', 'tqwt_maxValue_dec_2', 'tqwt_meanValue_dec_8', 'tqwt_stdValue_dec_33', 'tqwt_TKEO_mean_dec_13', 'tqwt_medianValue_dec_10', 'tqwt_stdValue_dec_18', 'tqwt_kurtosisValue_dec_15', 'tqwt_medianValue_dec_6', 'tqwt_medianValue_dec_25', 'meanPeriodPulses', 'app_LT_entropy_log_4_coef', 'tqwt_minValue_dec_2', 'det_entropy_log_2_coef', 'tqwt_entropy_log_dec_25', 'std_MFCC_1st_coef', 'app_TKEO_std_1_coef', 'tqwt_skewnessValue_dec_9', 'IMF_NSR_TKEO', 'tqwt_TKEO_mean_dec_34', 'det_LT_entropy_log_7_coef', 'tqwt_entropy_shannon_dec_27', 'tqwt_kurtosisValue_dec_25', 'tqwt_maxValue_dec_17', 'tqwt_entropy_log_dec_3', 'tqwt_kurtosisValue_dec_22', 'tqwt_minValue_dec_17', 'app_det_TKEO_mean_1_coef', 'tqwt_kurtosisValue_dec_21', 'tqwt_medianValue_dec_13', 'tqwt_TKEO_std_dec_21', 'GNE_mean', 'tqwt_maxValue_dec_28', 'tqwt_energy_dec_10', 'tqwt_energy_dec_24', 'tqwt_kurtosisValue_dec_29', 'tqwt_stdValue_dec_28', 'tqwt_meanValue_dec_23', 'app_LT_entropy_log_1_coef', 'det_entropy_log_1_coef', 'tqwt_energy_dec_33', 'mean_MFCC_11th_coef', 'tqwt_TKEO_std_dec_26', 'GNE_NSR_SEO', 'app_LT_entropy_log_3_coef', 'det_LT_entropy_shannon_2_coef', 'tqwt_TKEO_std_dec_1', 'app_LT_entropy_log_2_coef', 'mean_MFCC_6th_coef', 'tqwt_skewnessValue_dec_6', 'tqwt_minValue_dec_28', 'tqwt_kurtosisValue_dec_8', 'tqwt_minValue_dec_23', 'tqwt_medianValue_dec_24', 'tqwt_maxValue_dec_23', 'IMF_NSR_SEO', 'det_LT_entropy_log_8_coef', 'tqwt_medianValue_dec_9', 'tqwt_skewnessValue_dec_7', 'tqwt_entropy_log_dec_2', 'tqwt_TKEO_mean_dec_17', 'tqwt_TKEO_mean_dec_21', 'tqwt_kurtosisValue_dec_13', 'tqwt_TKEO_mean_dec_35', 'tqwt_energy_dec_20', 'tqwt_TKEO_mean_dec_6', 'tqwt_energy_dec_22', 'tqwt_skewnessValue_dec_28', 'tqwt_entropy_shannon_dec_36', 'tqwt_medianValue_dec_26', 'mean_3rd_delta_delta', 'det_LT_entropy_log_10_coef', 'tqwt_kurtosisValue_dec_7', 'tqwt_TKEO_std_dec_25', 'tqwt_stdValue_dec_19', 'det_entropy_log_4_coef', 'tqwt_kurtosisValue_dec_12', 'tqwt_entropy_shannon_dec_5', 'mean_MFCC_4th_coef', 'tqwt_TKEO_mean_dec_26', 'PPE', 'Ed2_10_coef', 'tqwt_energy_dec_29', 'tqwt_TKEO_std_dec_33', 'tqwt_energy_dec_34', 'mean_1st_delta', 'tqwt_TKEO_std_dec_28', 'tqwt_minValue_dec_4', 'det_LT_TKEO_mean_10_coef', 'tqwt_TKEO_std_dec_17', 'GNE_SNR_TKEO', 'tqwt_medianValue_dec_5', 'tqwt_maxValue_dec_4', 'det_LT_entropy_shannon_4_coef', 'tqwt_maxValue_dec_34', 'tqwt_energy_dec_3', 'tqwt_TKEO_std_dec_6', 'tqwt_entropy_shannon_dec_23', 'tqwt_skewnessValue_dec_31', 'det_LT_TKEO_std_10_coef', 'tqwt_kurtosisValue_dec_14', 'tqwt_stdValue_dec_25', 'tqwt_entropy_log_dec_29', 'tqwt_medianValue_dec_36', 'tqwt_minValue_dec_34', 'tqwt_medianValue_dec_12', 'det_entropy_log_5_coef', 'VFER_SNR_SEO', 'tqwt_entropy_log_dec_22', 'tqwt_kurtosisValue_dec_32', 'tqwt_kurtosisValue_dec_6', 'det_LT_entropy_shannon_10_coef', 'tqwt_maxValue_dec_29', 'det_LT_entropy_log_9_coef', 'tqwt_minValue_dec_25', 'tqwt_medianValue_dec_34', 'IMF_NSR_entropy', 'tqwt_medianValue_dec_2', 'tqwt_minValue_dec_35', 'tqwt_minValue_dec_29', 'tqwt_entropy_log_dec_24', 'tqwt_meanValue_dec_36', 'tqwt_stdValue_dec_20', 'tqwt_meanValue_dec_22', 'Ed_10_coef', 'tqwt_TKEO_std_dec_34', 'tqwt_maxValue_dec_25', 'tqwt_minValue_dec_3', 'tqwt_TKEO_mean_dec_28', 'tqwt_minValue_dec_33', 'tqwt_TKEO_mean_dec_5', 'tqwt_TKEO_std_dec_23', 'mean_10th_delta_delta', 'det_LT_TKEO_std_5_coef', 'tqwt_medianValue_dec_1', 'tqwt_kurtosisValue_dec_9', 'tqwt_kurtosisValue_dec_5', 'det_LT_entropy_shannon_1_coef', 'tqwt_maxValue_dec_3', 'tqwt_TKEO_mean_dec_32', 'mean_MFCC_9th_coef', 'Ed2_7_coef', 'tqwt_meanValue_dec_4', 'Ea2', 'tqwt_medianValue_dec_28', 'tqwt_energy_dec_35', 'tqwt_skewnessValue_dec_32', 'tqwt_maxValue_dec_33', 'tqwt_skewnessValue_dec_8', 'tqwt_TKEO_std_dec_5', 'tqwt_TKEO_mean_dec_18', 'tqwt_kurtosisValue_dec_3', 'det_entropy_log_6_coef', 'tqwt_stdValue_dec_36', 'b3', 'tqwt_medianValue_dec_27', 'Ed_7_coef', 'VFER_std', 'det_LT_TKEO_mean_7_coef', 'mean_4th_delta_delta', 'tqwt_TKEO_std_dec_29', 'tqwt_maxValue_dec_35', 'tqwt_energy_dec_2', 'tqwt_energy_dec_32', 'tqwt_meanValue_dec_16', 'Ed2_5_coef', 'tqwt_entropy_log_dec_21', 'tqwt_medianValue_dec_32', 'mean_MFCC_8th_coef', 'tqwt_TKEO_mean_dec_25', 'GNE_NSR_TKEO', 'det_LT_TKEO_std_7_coef', 'tqwt_stdValue_dec_4', 'tqwt_stdValue_dec_29', 'tqwt_kurtosisValue_dec_4', 'tqwt_skewnessValue_dec_30', 'tqwt_skewnessValue_dec_33', 'tqwt_kurtosisValue_dec_24', 'tqwt_meanValue_dec_34', 'tqwt_TKEO_std_dec_32', 'tqwt_meanValue_dec_12', 'Ea', 'VFER_NSR_TKEO', 'tqwt_kurtosisValue_dec_10', 'tqwt_TKEO_std_dec_20', 'tqwt_entropy_log_dec_32', 'tqwt_skewnessValue_dec_29', 'tqwt_TKEO_mean_dec_31', 'GNE_std', 'det_LT_TKEO_std_9_coef', 'tqwt_maxValue_dec_18', 'det_LT_TKEO_mean_9_coef', 'tqwt_skewnessValue_dec_4', 'tqwt_medianValue_dec_33', 'tqwt_stdValue_dec_24', 'tqwt_minValue_dec_18', 'tqwt_energy_dec_5', 'tqwt_kurtosisValue_dec_2', 'tqwt_energy_dec_1', 'tqwt_TKEO_mean_dec_29', 'tqwt_meanValue_dec_33', 'det_TKEO_std_5_coef', 'tqwt_medianValue_dec_7', 'mean_2nd_delta_delta', 'Ed2_9_coef', 'tqwt_stdValue_dec_32', 'det_LT_TKEO_mean_5_coef', 'VFER_SNR_TKEO', 'tqwt_entropy_shannon_dec_24', 'tqwt_skewnessValue_dec_16', 'tqwt_TKEO_std_dec_35', 'tqwt_medianValue_dec_18', 'tqwt_entropy_log_dec_1', 'tqwt_skewnessValue_dec_24']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from imblearn.over_sampling import SVMSMOTE\n",
    "from mrmr import mrmr_classif\n",
    "svmsmote = SVMSMOTE(random_state=42)\n",
    "\n",
    "# Oversampling the data\n",
    "X, Y = svmsmote.fit_resample(X, Y)\n",
    "#Shuffling the data\n",
    "X['class'] = Y\n",
    "X=X.sample(frac=1,random_state=0).reset_index(drop=True)\n",
    "Y=X['class']\n",
    "X=X.drop('class', axis=1)\n",
    "\n",
    "print(df.columns)\n",
    "# Select the features\n",
    "features = mrmr_classif(X,Y, K=528)\n",
    "\n",
    "# Print the selected features\n",
    "print(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['gender', 'PPE', 'DFA', 'RPDE', 'numPulses', 'numPeriodsPulses',\n",
      "       'meanPeriodPulses', 'stdDevPeriodPulses', 'locPctJitter',\n",
      "       'locAbsJitter',\n",
      "       ...\n",
      "       'tqwt_kurtosisValue_dec_27', 'tqwt_kurtosisValue_dec_28',\n",
      "       'tqwt_kurtosisValue_dec_29', 'tqwt_kurtosisValue_dec_30',\n",
      "       'tqwt_kurtosisValue_dec_31', 'tqwt_kurtosisValue_dec_32',\n",
      "       'tqwt_kurtosisValue_dec_33', 'tqwt_kurtosisValue_dec_34',\n",
      "       'tqwt_kurtosisValue_dec_35', 'tqwt_kurtosisValue_dec_36'],\n",
      "      dtype='object', length=753)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "X_temp, X_test, Y_temp, Y_test = train_test_split(X, Y, test_size=0.4, random_state=42)\n",
    "\n",
    "X_train, X_val, Y_train, Y_val = train_test_split(X_temp, Y_temp, test_size=0.25, random_state=42)\n",
    "\n",
    "scaler=StandardScaler()\n",
    "columns=X_train.columns\n",
    "\n",
    "X_train=pd.DataFrame(scaler.fit_transform(X_train), columns=columns)\n",
    "X_val=pd.DataFrame(scaler.fit_transform(X_val), columns=columns)\n",
    "X_test=pd.DataFrame(scaler.fit_transform(X_test), columns=columns)\n",
    "print(X_train.columns)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['tqwt_TKEO_std_dec_23', 'tqwt_TKEO_std_dec_28',\n",
      "       'app_LT_entropy_log_9_coef', 'app_LT_entropy_shannon_5_coef',\n",
      "       'app_entropy_log_3_coef', 'std_0th_delta', 'det_LT_TKEO_std_10_coef',\n",
      "       'tqwt_entropy_log_dec_20', 'GNE_SNR_SEO', 'tqwt_stdValue_dec_5',\n",
      "       ...\n",
      "       'GQ_std_cycle_closed', 'tqwt_entropy_log_dec_34',\n",
      "       'app_LT_TKEO_std_10_coef', 'mean_MFCC_11th_coef',\n",
      "       'tqwt_stdValue_dec_19', 'tqwt_entropy_shannon_dec_10',\n",
      "       'mean_MFCC_9th_coef', 'app_entropy_shannon_9_coef',\n",
      "       'app_entropy_log_8_coef', 'tqwt_kurtosisValue_dec_13'],\n",
      "      dtype='object', length=753)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Index(['tqwt_energy_dec_25', 'tqwt_TKEO_std_dec_32', 'std_12th_delta_delta',\n",
      "       'tqwt_TKEO_mean_dec_12', 'tqwt_entropy_shannon_dec_36',\n",
      "       'tqwt_entropy_log_dec_13', 'det_LT_entropy_log_9_coef',\n",
      "       'tqwt_medianValue_dec_7', 'tqwt_TKEO_mean_dec_12',\n",
      "       'tqwt_kurtosisValue_dec_27',\n",
      "       ...\n",
      "       'app_entropy_shannon_5_coef', 'tqwt_TKEO_mean_dec_25',\n",
      "       'std_delta_delta_0th', 'locDbShimmer', 'app_entropy_log_6_coef',\n",
      "       'GNE_SNR_TKEO', 'tqwt_TKEO_std_dec_35', 'Ea2', 'tqwt_minValue_dec_26',\n",
      "       'app_LT_entropy_shannon_10_coef'],\n",
      "      dtype='object', length=753)\n",
      "Index(['app_entropy_shannon_1_coef', 'app_entropy_log_8_coef',\n",
      "       'tqwt_entropy_log_dec_7', 'std_MFCC_11th_coef', 'tqwt_maxValue_dec_18',\n",
      "       'app_LT_entropy_log_6_coef', 'std_MFCC_6th_coef', 'mean_Log_energy',\n",
      "       'tqwt_entropy_log_dec_23', 'meanAutoCorrHarmonicity',\n",
      "       ...\n",
      "       'tqwt_TKEO_std_dec_32', 'tqwt_maxValue_dec_33', 'tqwt_meanValue_dec_11',\n",
      "       'Ed2_7_coef', 'tqwt_stdValue_dec_15', 'std_4th_delta_delta',\n",
      "       'tqwt_stdValue_dec_32', 'det_LT_TKEO_mean_10_coef', 'meanPeriodPulses',\n",
      "       'mean_MFCC_0th_coef'],\n",
      "      dtype='object', length=753)\n",
      "Index(['app_TKEO_std_6_coef', 'tqwt_TKEO_std_dec_29',\n",
      "       'tqwt_kurtosisValue_dec_5', 'det_LT_entropy_log_1_coef',\n",
      "       'tqwt_entropy_log_dec_8', 'GNE_NSR_SEO', 'tqwt_medianValue_dec_26',\n",
      "       'tqwt_medianValue_dec_18', 'tqwt_medianValue_dec_36', 'std_5th_delta',\n",
      "       ...\n",
      "       'tqwt_entropy_shannon_dec_35', 'app_det_TKEO_mean_10_coef',\n",
      "       'tqwt_TKEO_std_dec_32', 'app_LT_entropy_shannon_3_coef',\n",
      "       'GQ_std_cycle_open', 'tqwt_minValue_dec_23', 'app_LT_TKEO_mean_8_coef',\n",
      "       'det_LT_TKEO_std_10_coef', 'mean_10th_delta_delta',\n",
      "       'tqwt_energy_dec_8'],\n",
      "      dtype='object', length=753)\n",
      "Index(['tqwt_TKEO_mean_dec_17', 'tqwt_skewnessValue_dec_28',\n",
      "       'tqwt_skewnessValue_dec_16', 'tqwt_energy_dec_3', 'app_TKEO_std_5_coef',\n",
      "       'app_entropy_log_3_coef', 'tqwt_entropy_shannon_dec_9',\n",
      "       'app_entropy_log_2_coef', 'det_LT_TKEO_mean_7_coef',\n",
      "       'tqwt_entropy_shannon_dec_17',\n",
      "       ...\n",
      "       'mean_1st_delta', 'det_LT_entropy_log_6_coef', 'GQ_prc5_95',\n",
      "       'tqwt_kurtosisValue_dec_4', 'tqwt_stdValue_dec_8',\n",
      "       'tqwt_kurtosisValue_dec_29', 'det_entropy_log_4_coef',\n",
      "       'tqwt_minValue_dec_29', 'b4', 'tqwt_meanValue_dec_33'],\n",
      "      dtype='object', length=753)\n",
      "Index(['tqwt_stdValue_dec_24', 'meanAutoCorrHarmonicity',\n",
      "       'tqwt_TKEO_std_dec_16', 'tqwt_energy_dec_26', 'apq3Shimmer',\n",
      "       'tqwt_entropy_shannon_dec_10', 'tqwt_skewnessValue_dec_6',\n",
      "       'tqwt_energy_dec_20', 'Ea2', 'mean_MFCC_4th_coef',\n",
      "       ...\n",
      "       'GQ_prc5_95', 'tqwt_entropy_shannon_dec_18', 'tqwt_TKEO_mean_dec_16',\n",
      "       'tqwt_medianValue_dec_25', 'tqwt_kurtosisValue_dec_32',\n",
      "       'det_LT_entropy_log_5_coef', 'tqwt_meanValue_dec_23',\n",
      "       'tqwt_entropy_log_dec_19', 'tqwt_minValue_dec_2', 'tqwt_energy_dec_25'],\n",
      "      dtype='object', length=753)\n",
      "Index(['tqwt_entropy_log_dec_15', 'tqwt_energy_dec_10', 'tqwt_maxValue_dec_3',\n",
      "       'app_TKEO_std_6_coef', 'tqwt_entropy_shannon_dec_5',\n",
      "       'det_LT_entropy_shannon_4_coef', 'std_4th_delta', 'tqwt_energy_dec_33',\n",
      "       'locShimmer', 'tqwt_minValue_dec_14',\n",
      "       ...\n",
      "       'Ea2', 'IMF_NSR_entropy', 'tqwt_entropy_shannon_dec_6',\n",
      "       'app_LT_entropy_log_6_coef', 'tqwt_minValue_dec_35',\n",
      "       'tqwt_entropy_log_dec_2', 'tqwt_kurtosisValue_dec_20',\n",
      "       'tqwt_TKEO_std_dec_28', 'tqwt_medianValue_dec_2',\n",
      "       'tqwt_stdValue_dec_27'],\n",
      "      dtype='object', length=753)\n",
      "Index(['tqwt_entropy_shannon_dec_14', 'tqwt_entropy_shannon_dec_35',\n",
      "       'tqwt_TKEO_mean_dec_7', 'tqwt_meanValue_dec_36', 'mean_MFCC_7th_coef',\n",
      "       'tqwt_meanValue_dec_4', 'mean_MFCC_5th_coef', 'mean_delta_log_energy',\n",
      "       'app_LT_TKEO_mean_3_coef', 'tqwt_entropy_log_dec_22',\n",
      "       ...\n",
      "       'tqwt_maxValue_dec_3', 'VFER_std', 'tqwt_energy_dec_33',\n",
      "       'mean_10th_delta_delta', 'app_entropy_log_4_coef',\n",
      "       'app_TKEO_std_8_coef', 'app_det_TKEO_mean_3_coef',\n",
      "       'tqwt_TKEO_mean_dec_6', 'app_TKEO_std_9_coef',\n",
      "       'app_det_TKEO_mean_10_coef'],\n",
      "      dtype='object', length=753)\n",
      "Index(['tqwt_stdValue_dec_32', 'app_entropy_shannon_1_coef',\n",
      "       'tqwt_kurtosisValue_dec_21', 'tqwt_skewnessValue_dec_16',\n",
      "       'tqwt_maxValue_dec_23', 'tqwt_entropy_shannon_dec_7',\n",
      "       'tqwt_stdValue_dec_35', 'tqwt_TKEO_mean_dec_7', 'tqwt_TKEO_std_dec_16',\n",
      "       'GQ_prc5_95',\n",
      "       ...\n",
      "       'tqwt_medianValue_dec_36', 'tqwt_medianValue_dec_18',\n",
      "       'app_LT_entropy_shannon_10_coef', 'app_LT_entropy_log_4_coef',\n",
      "       'tqwt_TKEO_std_dec_21', 'VFER_entropy', 'tqwt_entropy_shannon_dec_18',\n",
      "       'tqwt_entropy_log_dec_15', 'tqwt_entropy_log_dec_32', 'std_1st_delta'],\n",
      "      dtype='object', length=753)\n",
      "Index(['det_LT_entropy_log_8_coef', 'tqwt_stdValue_dec_25',\n",
      "       'tqwt_kurtosisValue_dec_28', 'tqwt_stdValue_dec_16',\n",
      "       'tqwt_TKEO_std_dec_14', 'tqwt_medianValue_dec_6',\n",
      "       'det_entropy_log_3_coef', 'tqwt_stdValue_dec_15', 'Ed2_9_coef',\n",
      "       'tqwt_entropy_shannon_dec_24',\n",
      "       ...\n",
      "       'tqwt_medianValue_dec_7', 'tqwt_kurtosisValue_dec_34',\n",
      "       'tqwt_medianValue_dec_1', 'app_LT_TKEO_mean_2_coef',\n",
      "       'tqwt_kurtosisValue_dec_28', 'tqwt_kurtosisValue_dec_4',\n",
      "       'app_LT_entropy_log_5_coef', 'tqwt_kurtosisValue_dec_14',\n",
      "       'app_entropy_shannon_3_coef', 'tqwt_stdValue_dec_9'],\n",
      "      dtype='object', length=753)\n",
      "Index(['tqwt_TKEO_std_dec_5', 'tqwt_entropy_shannon_dec_7',\n",
      "       'mean_delta_log_energy', 'GNE_mean', 'mean_MFCC_4th_coef',\n",
      "       'tqwt_medianValue_dec_32', 'tqwt_TKEO_std_dec_26', 'std_2nd_delta',\n",
      "       'app_TKEO_std_10_coef', 'apq3Shimmer',\n",
      "       ...\n",
      "       'tqwt_meanValue_dec_25', 'tqwt_medianValue_dec_24', 'rapJitter',\n",
      "       'tqwt_skewnessValue_dec_16', 'tqwt_entropy_log_dec_26',\n",
      "       'tqwt_minValue_dec_35', 'app_entropy_log_3_coef',\n",
      "       'app_LT_TKEO_std_4_coef', 'app_LT_TKEO_mean_8_coef', 'ddpJitter'],\n",
      "      dtype='object', length=753)\n",
      "Index(['app_LT_entropy_shannon_9_coef', 'tqwt_meanValue_dec_25', 'Ea2',\n",
      "       'tqwt_entropy_shannon_dec_14', 'std_4th_delta', 'tqwt_energy_dec_8',\n",
      "       'tqwt_kurtosisValue_dec_34', 'tqwt_kurtosisValue_dec_19',\n",
      "       'tqwt_maxValue_dec_33', 'Ea2',\n",
      "       ...\n",
      "       'tqwt_kurtosisValue_dec_22', 'GQ_prc5_95', 'std_MFCC_0th_coef',\n",
      "       'tqwt_maxValue_dec_9', 'tqwt_entropy_shannon_dec_11',\n",
      "       'det_entropy_log_1_coef', 'tqwt_skewnessValue_dec_8',\n",
      "       'tqwt_TKEO_std_dec_11', 'mean_delta_log_energy', 'tqwt_maxValue_dec_4'],\n",
      "      dtype='object', length=753)\n",
      "Index(['tqwt_maxValue_dec_14', 'tqwt_TKEO_std_dec_23', 'mean_0th_delta',\n",
      "       'GQ_std_cycle_open', 'app_LT_TKEO_std_6_coef', 'app_entropy_log_5_coef',\n",
      "       'tqwt_medianValue_dec_7', 'tqwt_medianValue_dec_2',\n",
      "       'tqwt_maxValue_dec_11', 'tqwt_entropy_log_dec_34',\n",
      "       ...\n",
      "       'mean_MFCC_6th_coef', 'tqwt_kurtosisValue_dec_22',\n",
      "       'tqwt_TKEO_mean_dec_9', 'tqwt_stdValue_dec_9', 'tqwt_energy_dec_20',\n",
      "       'tqwt_skewnessValue_dec_6', 'tqwt_meanValue_dec_2',\n",
      "       'std_delta_delta_log_energy', 'tqwt_maxValue_dec_35',\n",
      "       'tqwt_TKEO_mean_dec_23'],\n",
      "      dtype='object', length=753)\n",
      "Index(['app_LT_TKEO_mean_10_coef', 'mean_Log_energy',\n",
      "       'tqwt_kurtosisValue_dec_34', 'locPctJitter', 'tqwt_maxValue_dec_4',\n",
      "       'app_LT_TKEO_std_2_coef', 'tqwt_meanValue_dec_5',\n",
      "       'det_LT_entropy_shannon_2_coef', 'app_LT_TKEO_mean_3_coef',\n",
      "       'tqwt_stdValue_dec_17',\n",
      "       ...\n",
      "       'tqwt_maxValue_dec_3', 'tqwt_TKEO_mean_dec_29', 'tqwt_minValue_dec_26',\n",
      "       'tqwt_maxValue_dec_27', 'tqwt_minValue_dec_8', 'tqwt_minValue_dec_2',\n",
      "       'std_Log_energy', 'tqwt_meanValue_dec_23', 'tqwt_entropy_log_dec_21',\n",
      "       'app_entropy_log_2_coef'],\n",
      "      dtype='object', length=753)\n",
      "Index(['tqwt_TKEO_std_dec_29', 'tqwt_stdValue_dec_25', 'tqwt_stdValue_dec_11',\n",
      "       'tqwt_TKEO_std_dec_5', 'tqwt_entropy_shannon_dec_5',\n",
      "       'app_det_TKEO_mean_3_coef', 'tqwt_minValue_dec_26',\n",
      "       'tqwt_skewnessValue_dec_31', 'app_entropy_log_8_coef',\n",
      "       'tqwt_minValue_dec_15',\n",
      "       ...\n",
      "       'mean_MFCC_0th_coef', 'tqwt_TKEO_std_dec_1', 'tqwt_stdValue_dec_36',\n",
      "       'tqwt_medianValue_dec_27', 'std_MFCC_0th_coef',\n",
      "       'tqwt_kurtosisValue_dec_8', 'IMF_NSR_entropy',\n",
      "       'app_LT_entropy_shannon_6_coef', 'app_det_TKEO_mean_5_coef',\n",
      "       'tqwt_entropy_shannon_dec_17'],\n",
      "      dtype='object', length=753)\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "k=15 #Number of bags\n",
    "\n",
    "# Randomly select 753 elements from the list, allowing duplicates\n",
    "X_train_sets=[]\n",
    "X_val_sets=[]\n",
    "X_test_sets=[]\n",
    "random.seed(0)\n",
    "for i in range(k):\n",
    "    resampled_features = random.choices(features, k=753)\n",
    "    X_train_sets.append(X_train[resampled_features])\n",
    "    X_val_sets.append(X_val[resampled_features])\n",
    "    X_test_sets.append(X_test[resampled_features])\n",
    "    print(X_train_sets[i].columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.linear_model import LogisticRegression,RidgeClassifier\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "models=[]\n",
    "for i in range(len(X_train_sets)):\n",
    "    models.append([SVC(), GaussianNB(), KNeighborsClassifier(),\\\n",
    "                    DecisionTreeClassifier(random_state=42), LogisticRegression(random_state=42), RidgeClassifier(random_state=42)])\n",
    "\n",
    "k_fold=KFold(n_splits=5, shuffle=True, random_state=42)\n",
    "\n",
    "for i in range(len(X_train_sets)):\n",
    "    X_train_fromsets=X_train_sets[i]\n",
    "    \n",
    "    for train_index, val_index in k_fold.split(X_train_fromsets):\n",
    "        X_train_kfold, X_val_kfold = X_train_fromsets.iloc[train_index].values, X_train_fromsets.iloc[val_index].values\n",
    "        Y_train_kfold, Y_val_kfold = Y_train.iloc[train_index].values, Y_train.iloc[val_index].values\n",
    "\n",
    "        for model in models[i]:\n",
    "            model.fit(X_train_kfold, Y_train_kfold)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bag 1:\n",
      "SVC :  0.8757396449704142\n",
      "GaussianNB :  0.7337278106508875\n",
      "KNeighborsClassifier :  0.8106508875739645\n",
      "DecisionTreeClassifier :  0.757396449704142\n",
      "LogisticRegression :  0.8816568047337278\n",
      "RidgeClassifier :  0.7988165680473372\n",
      "Best model: LogisticRegression\n",
      "\n",
      "\n",
      "Bag 2:\n",
      "SVC :  0.8994082840236687\n",
      "GaussianNB :  0.7514792899408284\n",
      "KNeighborsClassifier :  0.7869822485207101\n",
      "DecisionTreeClassifier :  0.7869822485207101\n",
      "LogisticRegression :  0.8698224852071006\n",
      "RidgeClassifier :  0.7633136094674556\n",
      "Best model: SVC\n",
      "\n",
      "\n",
      "Bag 3:\n",
      "SVC :  0.8994082840236687\n",
      "GaussianNB :  0.7633136094674556\n",
      "KNeighborsClassifier :  0.834319526627219\n",
      "DecisionTreeClassifier :  0.7337278106508875\n",
      "LogisticRegression :  0.863905325443787\n",
      "RidgeClassifier :  0.7396449704142012\n",
      "Best model: SVC\n",
      "\n",
      "\n",
      "Bag 4:\n",
      "SVC :  0.893491124260355\n",
      "GaussianNB :  0.7455621301775148\n",
      "KNeighborsClassifier :  0.8224852071005917\n",
      "DecisionTreeClassifier :  0.7751479289940828\n",
      "LogisticRegression :  0.863905325443787\n",
      "RidgeClassifier :  0.7041420118343196\n",
      "Best model: SVC\n",
      "\n",
      "\n",
      "Bag 5:\n",
      "SVC :  0.893491124260355\n",
      "GaussianNB :  0.727810650887574\n",
      "KNeighborsClassifier :  0.8224852071005917\n",
      "DecisionTreeClassifier :  0.7692307692307693\n",
      "LogisticRegression :  0.8461538461538461\n",
      "RidgeClassifier :  0.7337278106508875\n",
      "Best model: SVC\n",
      "\n",
      "\n",
      "Bag 6:\n",
      "SVC :  0.8816568047337278\n",
      "GaussianNB :  0.7396449704142012\n",
      "KNeighborsClassifier :  0.7928994082840237\n",
      "DecisionTreeClassifier :  0.727810650887574\n",
      "LogisticRegression :  0.8520710059171598\n",
      "RidgeClassifier :  0.7633136094674556\n",
      "Best model: SVC\n",
      "\n",
      "\n",
      "Bag 7:\n",
      "SVC :  0.8816568047337278\n",
      "GaussianNB :  0.7218934911242604\n",
      "KNeighborsClassifier :  0.8284023668639053\n",
      "DecisionTreeClassifier :  0.8106508875739645\n",
      "LogisticRegression :  0.8461538461538461\n",
      "RidgeClassifier :  0.7455621301775148\n",
      "Best model: SVC\n",
      "\n",
      "\n",
      "Bag 8:\n",
      "SVC :  0.9171597633136095\n",
      "GaussianNB :  0.7337278106508875\n",
      "KNeighborsClassifier :  0.8402366863905325\n",
      "DecisionTreeClassifier :  0.7455621301775148\n",
      "LogisticRegression :  0.863905325443787\n",
      "RidgeClassifier :  0.7810650887573964\n",
      "Best model: SVC\n",
      "\n",
      "\n",
      "Bag 9:\n",
      "SVC :  0.8757396449704142\n",
      "GaussianNB :  0.7455621301775148\n",
      "KNeighborsClassifier :  0.8106508875739645\n",
      "DecisionTreeClassifier :  0.7159763313609467\n",
      "LogisticRegression :  0.8402366863905325\n",
      "RidgeClassifier :  0.7928994082840237\n",
      "Best model: SVC\n",
      "\n",
      "\n",
      "Bag 10:\n",
      "SVC :  0.893491124260355\n",
      "GaussianNB :  0.7514792899408284\n",
      "KNeighborsClassifier :  0.8047337278106509\n",
      "DecisionTreeClassifier :  0.7988165680473372\n",
      "LogisticRegression :  0.8816568047337278\n",
      "RidgeClassifier :  0.7455621301775148\n",
      "Best model: SVC\n",
      "\n",
      "\n",
      "Bag 11:\n",
      "SVC :  0.863905325443787\n",
      "GaussianNB :  0.7633136094674556\n",
      "KNeighborsClassifier :  0.834319526627219\n",
      "DecisionTreeClassifier :  0.5857988165680473\n",
      "LogisticRegression :  0.8579881656804734\n",
      "RidgeClassifier :  0.7928994082840237\n",
      "Best model: SVC\n",
      "\n",
      "\n",
      "Bag 12:\n",
      "SVC :  0.8816568047337278\n",
      "GaussianNB :  0.727810650887574\n",
      "KNeighborsClassifier :  0.8047337278106509\n",
      "DecisionTreeClassifier :  0.7810650887573964\n",
      "LogisticRegression :  0.8402366863905325\n",
      "RidgeClassifier :  0.6863905325443787\n",
      "Best model: SVC\n",
      "\n",
      "\n",
      "Bag 13:\n",
      "SVC :  0.8994082840236687\n",
      "GaussianNB :  0.7514792899408284\n",
      "KNeighborsClassifier :  0.8284023668639053\n",
      "DecisionTreeClassifier :  0.6686390532544378\n",
      "LogisticRegression :  0.8698224852071006\n",
      "RidgeClassifier :  0.7633136094674556\n",
      "Best model: SVC\n",
      "\n",
      "\n",
      "Bag 14:\n",
      "SVC :  0.8875739644970414\n",
      "GaussianNB :  0.727810650887574\n",
      "KNeighborsClassifier :  0.834319526627219\n",
      "DecisionTreeClassifier :  0.834319526627219\n",
      "LogisticRegression :  0.8402366863905325\n",
      "RidgeClassifier :  0.7514792899408284\n",
      "Best model: SVC\n",
      "\n",
      "\n",
      "Bag 15:\n",
      "SVC :  0.9112426035502958\n",
      "GaussianNB :  0.7396449704142012\n",
      "KNeighborsClassifier :  0.834319526627219\n",
      "DecisionTreeClassifier :  0.7928994082840237\n",
      "LogisticRegression :  0.8284023668639053\n",
      "RidgeClassifier :  0.7633136094674556\n",
      "Best model: SVC\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "best_models=[]\n",
    "for i in range(len(X_val_sets)):\n",
    "    X_val_fromsets=X_val_sets[i]\n",
    "    accuracies=[]\n",
    "    print(f\"Bag {i+1}:\")\n",
    "    for model in models[i]:\n",
    "        Y_pred = model.predict(X_val_fromsets.values)\n",
    "        acc=accuracy_score(Y_val, Y_pred)\n",
    "        accuracies.append(acc)\n",
    "        print(f\"{type(model).__name__} : \",acc)\n",
    "    best_model=models[i][accuracies.index(max(accuracies))]\n",
    "    best_models.append(best_model)\n",
    "    print(f\"Best model: {type(best_model).__name__}\")\n",
    "        \n",
    "    print(\"\\n\")\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LogisticRegression :  0.8805309734513275\n",
      "\n",
      "\n",
      "SVC :  0.8849557522123894\n",
      "\n",
      "\n",
      "SVC :  0.8761061946902655\n",
      "\n",
      "\n",
      "SVC :  0.8716814159292036\n",
      "\n",
      "\n",
      "SVC :  0.8783185840707964\n",
      "\n",
      "\n",
      "SVC :  0.8761061946902655\n",
      "\n",
      "\n",
      "SVC :  0.8761061946902655\n",
      "\n",
      "\n",
      "SVC :  0.8915929203539823\n",
      "\n",
      "\n",
      "SVC :  0.8650442477876106\n",
      "\n",
      "\n",
      "SVC :  0.8871681415929203\n",
      "\n",
      "\n",
      "SVC :  0.8716814159292036\n",
      "\n",
      "\n",
      "SVC :  0.8871681415929203\n",
      "\n",
      "\n",
      "SVC :  0.8849557522123894\n",
      "\n",
      "\n",
      "SVC :  0.8805309734513275\n",
      "\n",
      "\n",
      "SVC :  0.8783185840707964\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "Y_pred_sets=pd.DataFrame()\n",
    "for i in range(len(X_test_sets)):\n",
    "    X_test_fromsets=X_test_sets[i]\n",
    "    best_model=best_models[i]\n",
    "    Y_pred = best_model.predict(X_test_fromsets.values)\n",
    "    Y_pred_sets[f\"Y_pred_{i}\"]=Y_pred\n",
    "    acc=accuracy_score(Y_test, Y_pred)\n",
    "    print(f\"{type(best_model).__name__} : \",acc)\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sensitivity: 0.8716814159292036\n",
      "Specificity: 0.9026548672566371\n",
      "False Positive Rate: 0.09734513274336283\n",
      "Positive Predictive Value: 0.8995433789954338\n",
      "Negative Predictive Value: 0.8755364806866953\n",
      "\n",
      "Majority Voting Accuracy :  0.8871681415929203\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "Y_pred_sets['mode']=Y_pred_sets.mode(axis=1)\n",
    "# print(Y_pred_sets)K\n",
    "\n",
    "tn, fp, fn, tp = confusion_matrix(Y_test, Y_pred_sets['mode']).ravel()\n",
    "\n",
    "sensitivity = tp / (tp + fn)\n",
    "specificity = tn / (tn + fp)\n",
    "fpr = fp / (fp + tn)\n",
    "ppv = tp / (tp + fp)\n",
    "npv = tn / (tn + fn)\n",
    "accuracy = (tp + tn) / (tp + fp + tn + fn)\n",
    "\n",
    "print(f\"Sensitivity: {sensitivity}\")\n",
    "print(f\"Specificity: {specificity}\")\n",
    "print(f\"False Positive Rate: {fpr}\")\n",
    "print(f\"Positive Predictive Value: {ppv}\")\n",
    "print(f\"Negative Predictive Value: {npv}\")\n",
    "print(f\"\\nMajority Voting Accuracy : \",accuracy)\n",
    "Y_pred_sets.drop('mode', axis=1, inplace=True)\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy :  0.9644970414201184\n"
     ]
    }
   ],
   "source": [
    "Y_pred_val_sets=pd.DataFrame()\n",
    "for i in range(len(X_val_sets)):\n",
    "    X_val_fromsets=X_val_sets[i]\n",
    "    best_model=best_models[i]\n",
    "    Y_pred_val = best_model.predict(X_val_fromsets.values)\n",
    "    Y_pred_val_sets[f\"Y_pred_{i}\"]=Y_pred_val\n",
    "    acc=accuracy_score(Y_val, Y_pred_val)\n",
    "\n",
    "model=DecisionTreeClassifier(random_state=42)\n",
    "model.fit(Y_pred_val_sets, Y_val)\n",
    "Y_pred_val = model.predict(Y_pred_val_sets)\n",
    "acc=accuracy_score(Y_val, Y_pred_val)\n",
    "print(\"Validation Accuracy : \",acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final Accuracy :  0.9070796460176991\n"
     ]
    }
   ],
   "source": [
    "final_preds=model.predict(Y_pred_sets)\n",
    "acc=accuracy_score(Y_test, final_preds)\n",
    "print(\"Final Accuracy : \",acc)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}