papers_vs_datasets.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from pathlib import Path\n",
    "import math\n",
    "import itertools\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib as mpl\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from tueplots import fonts\n",
    "import openml as oml\n",
    "from tqdm import tqdm\n",
    "\n",
    "from uq.analysis.dataframes import (\n",
    "\tload_config, load_df, make_test_df_for_tuning, make_test_df, get_datasets_df,\n",
    "\tset_hparams_columns, set_test_metrics_columns, make_df_abb\n",
    ")\n",
    "from uq.utils.general import filter_dict, set_notebook_options, savefig\n",
    "\n",
    "set_notebook_options()\n",
    "plt.rcParams.update(fonts.icml2022_tex())\n",
    "\n",
    "path = Path('results')\n",
    "ext = 'pdf'"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determine which datasets to select per benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dataset_names(suite_id):\n",
    "    names = [oml.datasets.get_dataset(ds_id).name for ds_id in oml.study.get_suite(suite_id).data]\n",
    "    if suite_id == 297:\n",
    "        names.remove('houses')\n",
    "    return names\n",
    "\n",
    "suites = {\n",
    "    suite_id: set(dataset_names(suite_id))\n",
    "    for suite_id in [269, 297, 299]\n",
    "}\n",
    "s = suites[297].copy()\n",
    "suites[299] -= s\n",
    "s |= suites[299]\n",
    "suites[269] -= s\n",
    "s |= suites[269]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataframe linking papers to datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Our datasets come from this config\n",
    "config = load_config('logs/full/')\n",
    "\n",
    "papers_to_datasets = {\n",
    "    'Gal 2016': {\n",
    "        'uci': [\n",
    "            'boston',\n",
    "            'Concrete',\n",
    "            'Energy',\n",
    "            'Kin8nm',\n",
    "            'Naval',\n",
    "            'Power',\n",
    "            'Protein',\n",
    "            'wine_quality',\n",
    "            'Yacht',\n",
    "            'year',\n",
    "        ],\n",
    "    },\n",
    "    'Utpala 2020': {\n",
    "        'uci': [\n",
    "            'Airfoil',\n",
    "            'boston',\n",
    "            'Concrete',\n",
    "            'Fish',\n",
    "            'Kin8nm',\n",
    "            'Protein',\n",
    "            'wine_quality',\n",
    "            'Yacht',\n",
    "            'year',\n",
    "        ],\n",
    "    },\n",
    "    'Zhou 2021': {\n",
    "        'uci': [\n",
    "            'CPU',\n",
    "            'Crime',\n",
    "            'Energy',\n",
    "            'MPG',\n",
    "        ]\n",
    "    },\n",
    "    'Chung 2021': {\n",
    "        'uci': [\n",
    "            'boston',\n",
    "            'Concrete',\n",
    "            'Energy',\n",
    "            'Kin8nm',\n",
    "            'Naval',\n",
    "            'Power',\n",
    "            'wine_quality',\n",
    "            'Yacht',\n",
    "        ]\n",
    "    },\n",
    "    'Fakoor 2021': {\n",
    "        'uci': [\n",
    "            'boston',\n",
    "            'Concrete',\n",
    "            'Energy',\n",
    "            'Kin8nm',\n",
    "            'Naval',\n",
    "            'Power',\n",
    "            'Protein',\n",
    "            'Yacht',\n",
    "        ],\n",
    "        'oml_269': [name for name in dataset_names(269) if name != 'boston'],\n",
    "    },\n",
    "    'Grinsztajn 2022': {\n",
    "        f'oml_{suite_id}': list(suites[suite_id])\n",
    "        for suite_id in [297, 299]\n",
    "    },\n",
    "    r'\\textbf{Ours}': {\n",
    "        source: dataset_group.names\n",
    "        for source, dataset_group in config.dataset_groups.items()\n",
    "    },\n",
    "}\n",
    "\n",
    "data = []\n",
    "for paper, sources in papers_to_datasets.items():\n",
    "    for source, datasets in sources.items():\n",
    "        for dataset in datasets:\n",
    "            data.append((paper, source, dataset))\n",
    "\n",
    "df_papers = pd.DataFrame(data, columns=['paper', 'source', 'dataset'])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pivot table with the selected datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The `order.pickle` file must first be generated in `main_figures.ipynb`.\n",
    "order = pd.read_pickle(Path(config.log_dir) / 'order.pickle').to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_abb = make_df_abb(order['dataset'].unique())\n",
    "df_abb2 = make_df_abb(df_papers.query('dataset not in @order.dataset').dataset.unique())\n",
    "df_abb2['abb'] = '(' + df_abb2['abb'] + ')'\n",
    "df_abb = pd.concat((df_abb, df_abb2))\n",
    "\n",
    "df = df_papers.merge(df_abb, how='outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "source_names = {\n",
    "    f'oml_{suite_id}': f'OpenML {suite_id}'\n",
    "    for suite_id in [297, 299, 269]\n",
    "}\n",
    "source_names['uci'] = 'UCI'\n",
    "df['source'] = df['source'].map(source_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "order2 = np.concatenate((order.merge(df_abb)['abb'].to_numpy(), df_abb2['abb'].unique()))\n",
    "df['abb'] = pd.Categorical(df['abb'], order2)\n",
    "df['paper'] = pd.Categorical(df['paper'], list(papers_to_datasets))\n",
    "df['source'] = pd.Categorical(df['source'], list(source_names.values()))\n",
    "df = df.sort_values('dataset')\n",
    "df_pivot = df.pivot_table(values='source', \n",
    "        index='paper', columns='abb', dropna=False, fill_value=None, aggfunc=lambda x: x, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pivot"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Figure with the selected datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sources = df['source'].sort_values().unique()\n",
    "source_to_int = {source: i for i, source in enumerate(sources)}\n",
    "source_to_int[np.nan] = np.nan\n",
    "df_pivot = df_pivot.applymap(lambda x: source_to_int[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib.lines import Line2D\n",
    "from matplotlib.patches import Rectangle\n",
    "import matplotlib.patches as patches\n",
    "from matplotlib.legend_handler import HandlerPatch\n",
    "\n",
    "fig, axis = plt.subplots(figsize=(8, 1.3), dpi=300)\n",
    "cmap = sns.color_palette('deep', 4)\n",
    "g = sns.heatmap(df_pivot, square=True, cmap=cmap, cbar=False, xticklabels=1, yticklabels=1, mask=df_pivot.isna(), ax=axis)\n",
    "axis.tick_params(axis='x', which='major', labelsize=7, labelrotation=90)\n",
    "axis.tick_params(axis='y', which='major', labelsize=7)\n",
    "axis.set(xlabel=None, ylabel=None)\n",
    "g.set_facecolor('white')\n",
    "\n",
    "custom_lines = [\n",
    "    Rectangle((0, 0), 1, 1, color=cmap[i], lw=4)\n",
    "    for i in range(len(sources))\n",
    "]\n",
    "\n",
    "class HandlerRect(HandlerPatch):\n",
    "    def create_artists(self, legend, orig_handle,\n",
    "                       xdescent, ydescent, width, height,\n",
    "                       fontsize, trans):\n",
    "        x = width//2\n",
    "        y = 0\n",
    "        w = h = 3\n",
    "        # create\n",
    "        p = patches.Rectangle(xy=(x, y), width=w, height=h)\n",
    "        # update with data from original object\n",
    "        self.update_prop(p, orig_handle, legend)\n",
    "        # move xy to legend\n",
    "        p.set_transform(trans)\n",
    "        return [p]\n",
    "\n",
    "fig.legend(custom_lines, sources,\n",
    "    loc='lower center',\n",
    "    bbox_to_anchor=(0.5, 1-0.15),\n",
    "    frameon=True,\n",
    "    ncol=4,\n",
    "    fontsize=7,\n",
    "    handlelength=1, handleheight=1,\n",
    ")\n",
    "\n",
    "savefig(path / f'papers_vs_datasets.{ext}', fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.pivot_table(values='dataset', index='paper', columns='source', aggfunc='count', fill_value=0)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### List of the selected datasets sorted by size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_datasets_to_papers(papers_to_datasets):\n",
    "    datasets = itertools.chain(*papers_to_datasets.values())\n",
    "    datasets_to_papers = {dataset: [] for dataset in datasets}\n",
    "    for paper, datasets in papers_to_datasets.items():\n",
    "        for dataset in datasets:\n",
    "            datasets_to_papers[dataset].append(paper)\n",
    "    return datasets_to_papers\n",
    "\n",
    "datasets_to_papers = make_datasets_to_papers(papers_to_datasets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pprint import pprint\n",
    "\n",
    "def get_size_df(suite_id):\n",
    "    data = []\n",
    "    for ds_id in oml.study.get_suite(suite_id).data:\n",
    "        ds = oml.datasets.get_dataset(ds_id)\n",
    "        if ds.name not in suites[suite_id]:\n",
    "            continue\n",
    "        x, y, categorical_indicator, attribute_names = ds.get_data(\n",
    "            dataset_format='dataframe', target=ds.default_target_attribute)\n",
    "        data.append((ds.name, x.shape[0], suite_id))\n",
    "    return pd.DataFrame(data, columns=['dataset', 'Size', 'Suite ID'])\n",
    "\n",
    "dfs = []\n",
    "for suite_id in [269, 297, 299]:\n",
    "    print(suite_id)\n",
    "    pprint(get_size_df(suite_id).sort_values('Size')['dataset'].to_list())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('regul')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "8a8dedf9293b37ff9b138d946f5378258f0cf3b04aeac4c8b4c7c642523b0a45"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}