1_dataset2hdf5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Import modules"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "51071d199f462d8c"
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-06-05T21:59:22.925091Z",
     "start_time": "2024-06-05T21:59:22.784105Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import cv2\n",
    "import h5py\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Get dataset images\n",
    "* Dataset downloaded from Kaggle: [Brain Tumor Classification (MRI)](https://www.kaggle.com/datasets/sartajbhuvaji/brain-tumor-classification-mri/discussion?sort=hotness)\n",
    "* Unzip file"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "efcc912b177f292c"
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Store images in hdf5 file structure"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "bd353e8072e754bf"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Define the size of hdf5 structure\n",
    "With this function, we get the total number of images, and with that we define the size of the structure"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "dd54f7f67408cc3e"
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "def count_images_in_directory(directory):\n",
    "    image_count = 0\n",
    "    for root, _, files in os.walk(directory):\n",
    "        for file in files:\n",
    "            if file.lower().endswith('.jpg'):\n",
    "                image_count += 1\n",
    "    return image_count"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-06-05T21:59:22.941123Z",
     "start_time": "2024-06-05T21:59:22.926088Z"
    }
   },
   "id": "5c73e1856515c6ac",
   "execution_count": 2
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Store images in hdf5 structure\n",
    "* This function receives the path to the folder with the data, also receives the name in which the hdf5 file is going to be saved.\n",
    "* The hdf5 file store the images in a random way in a dataset of images.\n",
    "* Also saves its associated class in a dataset of labels."
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "b5a8520a55a3740b"
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "def image_hdf5_from_directory(pathfile, name_hdf5_file = 'dataset.h5'):\n",
    "    #output_hdf5 = 'Dataset/'+name_hdf5_file\n",
    "    output_hdf5 = name_hdf5_file\n",
    "\n",
    "    path_to_each_folder = [os.path.join(pathfile, name) for name in os.listdir(pathfile)]\n",
    "\n",
    "    # Initialize the HDF5 file\n",
    "    with h5py.File(output_hdf5, 'w') as hdf5_file:\n",
    "\n",
    "      ammount_images = count_images_in_directory(pathfile)\n",
    "\n",
    "      # Create datasets for images and labels\n",
    "      image_dataset = hdf5_file.create_dataset(\"images\", (ammount_images, 256, 256, 3), dtype='uint8')\n",
    "      label_dataset = hdf5_file.create_dataset(\"labels\", (ammount_images,), dtype='int')\n",
    "\n",
    "      # Create lists to keep track of the order of images\n",
    "      #image_indices = list(range(4000))\n",
    "      image_indices = np.arange(ammount_images)\n",
    "\n",
    "      # Shuffle the order in which images will be added\n",
    "      #random.shuffle(image_indices)\n",
    "      np.random.shuffle(image_indices)\n",
    "\n",
    "      # Function to add image and label to the dataset\n",
    "      def add_image(image, label, index):\n",
    "          image_dataset[index] = image\n",
    "          label_dataset[index] = label\n",
    "\n",
    "      i = 0\n",
    "      for class_label, folder_path in enumerate(path_to_each_folder):\n",
    "        for filename in tqdm(os.listdir(folder_path), desc = f'storing images class {class_label}'):\n",
    "            if filename.endswith(('.jpg')):\n",
    "                file_path = os.path.join(folder_path, filename)\n",
    "                image = cv2.imread(file_path)\n",
    "                image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
    "                image = cv2.resize(image, (256, 256))\n",
    "\n",
    "                if image is not None:\n",
    "                  index = image_indices[i]\n",
    "                  add_image(image, class_label, index)\n",
    "                  #print('Saved image', i, 'to h5 file in', index, 'position')\n",
    "                  i+=1"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-06-05T21:59:22.956786Z",
     "start_time": "2024-06-05T21:59:22.942691Z"
    }
   },
   "id": "a8038b6c1cd7a8e9",
   "execution_count": 3
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Store images\n",
    "*Note: the testing folder in Kaggle was mislabeled, that's why we'll only work with the training folder."
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "6350102ac25ec248"
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "storing images class 0: 100%|██████████| 826/826 [00:11<00:00, 72.16it/s] \n",
      "storing images class 1: 100%|██████████| 822/822 [00:01<00:00, 565.64it/s]\n",
      "storing images class 2: 100%|██████████| 395/395 [00:00<00:00, 636.06it/s]\n",
      "storing images class 3: 100%|██████████| 827/827 [00:01<00:00, 538.50it/s]\n"
     ]
    }
   ],
   "source": [
    "image_hdf5_from_directory(r'C:\\Users\\edfli\\Downloads\\brain_tumor_dataset\\Training',\n",
    "                          name_hdf5_file = r'C:\\Users\\edfli\\Downloads\\brain_tumor_dataset\\brain_tumor_dataset.h5')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-06-05T21:59:38.092052Z",
     "start_time": "2024-06-05T21:59:22.957783Z"
    }
   },
   "id": "b6d420afc9262615",
   "execution_count": 4
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Split data into train and test splits"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "f06a9b6f8747fd16"
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Open the file we've just created"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "ff9a1778d3c13c96"
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2870, 256, 256, 3) (2870,)\n"
     ]
    }
   ],
   "source": [
    "# Open dataset\n",
    "with h5py.File(r'C:\\Users\\edfli\\Downloads\\brain_tumor_dataset\\brain_tumor_dataset.h5', 'r') as hdf5_file:\n",
    "    images = hdf5_file[\"images\"][:]\n",
    "    labels = hdf5_file[\"labels\"][:]\n",
    "print(images.shape, labels.shape)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-06-05T21:59:38.261690Z",
     "start_time": "2024-06-05T21:59:38.093050Z"
    }
   },
   "id": "d9986e0219886f8d",
   "execution_count": 5
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Split data into train and test splits"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "1eea6785796ee039"
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2296, 256, 256, 3) (2296,)\n",
      "(574, 256, 256, 3) (574,)\n"
     ]
    }
   ],
   "source": [
    "images_training, images_testing, labels_training, labels_testing = train_test_split(images, labels, test_size=0.2)\n",
    "print(images_training.shape, labels_training.shape)\n",
    "print(images_testing.shape, labels_testing.shape)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-06-05T21:59:38.356228Z",
     "start_time": "2024-06-05T21:59:38.262648Z"
    }
   },
   "id": "2965b9578c7385a8",
   "execution_count": 6
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Store train split into train.h5"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "571b6e8f6b676dd2"
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "with h5py.File(r'C:\\Users\\edfli\\Downloads\\brain_tumor_dataset\\brain_tumor_dataset_train.h5', 'w') as f:\n",
    "    f.create_dataset('images', data=images_training)\n",
    "    f.create_dataset('labels', data=labels_training)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-06-05T22:00:02.911555Z",
     "start_time": "2024-06-05T22:00:02.606725Z"
    }
   },
   "id": "ffc44cedf41855e9",
   "execution_count": 9
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Store test split into train.h5"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "16e4a44e9e0d7020"
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "with h5py.File(r'C:\\Users\\edfli\\Downloads\\brain_tumor_dataset\\brain_tumor_dataset_test.h5', 'w') as f:\n",
    "    f.create_dataset('images', data=images_testing)\n",
    "    f.create_dataset('labels', data=labels_testing)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-06-05T22:00:04.106051Z",
     "start_time": "2024-06-05T22:00:03.304097Z"
    }
   },
   "id": "d6af2475b359fc5a",
   "execution_count": 10
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}