30042024-lactose_prediction_load_data_save_3dnpy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5c26e514-03b8-4c05-9488-1223b84a7962",
   "metadata": {},
   "source": [
    "Data description\n",
    "A total of 72 milk samples were analysed using hyperspectral imaging. The number of pixels considered for each sample ranges from 112 to 300. For each pixel, a spectrum containing 3,424 wavelengths in the range from 400 cm-1 to 7,000 cm-1 was generated. For each sample, the spectra were extracted from the pixels and included into a single dataset, therefore the spatial information were successively lost.  For each sample the lactose concentration was analysed and expressed as mg/mL. \n",
    "Data provided\n",
    "Training set covariates: an excel file with 64 different sheets, each one corresponding to a different sample. Each sheet corresponds to a data matrix with ni rows and p = 3424 columns, corresponding to the spectra for the ni considered pixels for the i-th sample. \n",
    "Training set response: an excel file containing the information on the lactose content for the samples included in the training set. \n",
    "Test set covariates: an excel file with 8 different sheets, with the same structure as the training set covariates but without the information on the lactose content. \n",
    "Task\n",
    "Participants should develop prediction models to quantify the lactose content employing the spectral information. Each participant should send a csv file with the predicted lactose contents for the samples in the test set. \n",
    "Tips\n",
    "For each sample there are both outlier spectra and noise regions to be deleted before the development of the prediction model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56eb41fa",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-03T12:19:21.891534Z",
     "start_time": "2023-11-03T12:19:21.847712Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "#import seaborn as sns\n",
    "#sns.set()\n",
    "import os, sys\n",
    "import time\n",
    "\n",
    "#linear models\n",
    "from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet, RidgeCV, LassoCV, ElasticNetCV\n",
    "from sklearn.cross_decomposition import PLSRegression\n",
    "\n",
    "#ensembles\n",
    "from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor\n",
    "#from lightgbm import LGBMRegressor\n",
    "#from xgboost.sklearn import XGBRegressor\n",
    "\n",
    "#knn\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "\n",
    "#neural networks\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "\n",
    "#svm: try both linear kernel and rbf kernel\n",
    "from sklearn.svm import SVR\n",
    "\n",
    "#deep neural networks aka deep learning\n",
    "#tbd: initial results by Ashish did not look good\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.model_selection import cross_val_score \n",
    "#from patsy import dmatrices\n",
    "from sklearn.utils import shuffle\n",
    "#from pandas_profiling import ProfileReport\n",
    "\n",
    "from sklearn import preprocessing\n",
    "from scipy.interpolate import interp1d\n",
    "\n",
    "import math\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error,r2_score\n",
    "from sklearn.feature_selection import SelectKBest, f_regression\n",
    "\n",
    "\n",
    "# Import package matplotlib for visualisation/plotting\n",
    "import matplotlib.pyplot as plt\n",
    "#For showing plots directly in the notebook run the command below\n",
    "%matplotlib inline\n",
    "# For saving multiple plots into a single pdf file\n",
    "from matplotlib.backends.backend_pdf import PdfPages\n",
    "\n",
    "import plotly.express as px\n",
    "\n",
    "SMALL_SIZE = 8\n",
    "MEDIUM_SIZE = 10\n",
    "BIG_SIZE = 12\n",
    "\n",
    "from sklearn.base import BaseEstimator, RegressorMixin, TransformerMixin\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "\n",
    "plt.rc('font', size=BIG_SIZE)          # controls default text sizes\n",
    "plt.rc('axes', titlesize=BIG_SIZE)     # fontsize of the axes title\n",
    "plt.rc('axes', labelsize=BIG_SIZE)    # fontsize of the x and y labels\n",
    "plt.rc('xtick', labelsize=BIG_SIZE)    # fontsize of the tick labels\n",
    "plt.rc('ytick', labelsize=BIG_SIZE)    # fontsize of the tick labels\n",
    "plt.rc('legend', fontsize=BIG_SIZE)    # legend fontsize\n",
    "plt.rc('figure', titlesize=BIG_SIZE)  # fontsize of the figure title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f68ac01-cb9b-4ebb-9967-e5761255c169",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data_from_excel_multisheets(path_to_input_file, output_3d_npy):\n",
    "\n",
    "    print(path_to_input_file)\n",
    "    xls = pd.ExcelFile(path_to_input_file, engine='openpyxl')\n",
    "    print(xls.sheet_names)\n",
    "\n",
    "    idx = 0\n",
    "    for sheet_name in xls.sheet_names:\n",
    "        df_2d = pd.read_excel(xls, sheet_name)\n",
    "        print(sheet_name)\n",
    "        print(df_2d.shape)\n",
    "        print(df_2d.head(1))\n",
    "       \n",
    "        # if sample has less rows than expected, add zero rows\n",
    "        if(df_2d.shape[0] < output_3d_npy.shape[1]):\n",
    "            nrows = output_3d_npy.shape[1] - df_2d.shape[0]\n",
    "            print(nrows)\n",
    "            a = np.zeros((nrows, df_2d.shape[1]))\n",
    "            print(a.shape)\n",
    "            output_3d_npy[idx] = np.concatenate((df_2d, a), axis = 0)\n",
    "        else:\n",
    "            output_3d_npy[idx] = df_2d\n",
    "        idx += 1\n",
    "\n",
    "    return output_3d_npy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee667e74-dc12-4bc5-b182-097fc8a5c531",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = load_data_from_excel_multisheets('train_dataset_samples.xlsx', np.zeros((64,300,3424)))\n",
    "test_X = load_data_from_excel_multisheets('test_dataset_samples.xlsx', np.zeros((8,300,3424)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80575913-a7c1-429c-bd76-35cf7aac1e6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(train_X.shape)\n",
    "print(train_X[0].shape)\n",
    "print(train_X[0])\n",
    "\n",
    "print(test_X.shape)\n",
    "print(test_X[0].shape)\n",
    "print(test_X[0])\n",
    "\n",
    "#save 3d numpy array to disk\n",
    "np.save(\"train_X.npy\", train_X)\n",
    "np.save(\"test_X.npy\", test_X)\n",
    "\n",
    "#load 3d numpy array from disk\n",
    "#test_X_tryload = np.load(\"test_X.npy\")\n",
    "\n",
    "#print(test_X_tryload.shape)\n",
    "#print(test_X_tryload[7])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15d7c83b-78ac-4a83-8683-72d8f5902272",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y = pd.read_excel('train_targets.xlsx', 0, engine='openpyxl')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}