ros3vfd-log-info.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c1d4cad4-c84c-4104-981c-9eb0a20f75fd",
   "metadata": {},
   "source": [
    "# ROS3 VFD Log Analysis Dashboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f53c6f1c-e624-4952-a3e8-69302152da81",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dataclasses import dataclass\n",
    "import io\n",
    "import re\n",
    "import numpy as np\n",
    "from bokeh.models import HoverTool\n",
    "import holoviews as hv\n",
    "import panel as pn\n",
    "hv.extension('bokeh')\n",
    "pn.extension()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc1a4585-45e7-48c7-a806-d7096f0f82bc",
   "metadata": {},
   "source": [
    "## Log Parser\n",
    "\n",
    "The class that represents information of one HTTP range GET request:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2cef30c0-71b2-405f-83d4-a440df748a52",
   "metadata": {},
   "outputs": [],
   "source": [
    "@dataclass(frozen=True)\n",
    "class ByteRange:\n",
    "    start: int\n",
    "    end: int\n",
    "    filesize: int\n",
    "\n",
    "    def __post_init__(self):\n",
    "        if self.start < 0 or self.end <= 0 or self.filesize <= 0:\n",
    "            raise ValueError('Start, end, and file size values must be positive integers')\n",
    "        elif self.end > self.filesize:\n",
    "            raise ValueError('End value must be smaller or equal to file size')\n",
    "        elif self.start > self.end:\n",
    "            raise ValueError('Start value must be smaller or equal to end value')\n",
    "\n",
    "    @property\n",
    "    def size(self):\n",
    "        return self.end - self.start + 1\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.size"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5954181-f1d7-40f3-8594-3afe62bcd2aa",
   "metadata": {},
   "source": [
    "Log file parsers:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4e4e020-c058-4a6c-b2dd-ed12962ae785",
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_ros3vfd_log(content: bytes) -> list[ByteRange]:\n",
    "    head_line = re.compile('HEAD: Bytes 0 - ')\n",
    "    fsize_line = re.compile('(FILESIZE| -- size): (?P<size>[0-9]+)')\n",
    "    range_line = re.compile('GET: Bytes (?P<start>[0-9]+) - (?P<end>[0-9]+), Request Size: (?P<size>[0-9]+)')\n",
    "\n",
    "    ranges = list()\n",
    "    with io.TextIOWrapper(io.BytesIO(content)) as logtxt:\n",
    "        for line in logtxt:\n",
    "            if head_line.match(line):\n",
    "                break\n",
    "        else:\n",
    "            raise RuntimeError('HEAD line not found in the log file')\n",
    "\n",
    "        for line in logtxt:\n",
    "            match = fsize_line.match(line)\n",
    "            if match:\n",
    "                fsize = int(match.group('size'))\n",
    "                break\n",
    "        else:\n",
    "            raise RuntimeError('FILESIZE line not found in the log file')\n",
    "\n",
    "        for line in logtxt:\n",
    "            match = range_line.search(line)\n",
    "            if match:\n",
    "                range = ByteRange(start=int(match.group('start')), \n",
    "                                  end=int(match.group('end')),\n",
    "                                  filesize=fsize)\n",
    "                if range.size != int(match.group('size')):\n",
    "                    raise ValueError(f'Reported size different for {match.group()}')\n",
    "                ranges.append(range)\n",
    "                \n",
    "    return ranges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fcc67d0-1a85-43d8-88c7-45ac97d62ea1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_fsspec_log(content: bytes) -> list[ByteRange]:\n",
    "    \"\"\"\n",
    "    This method only parses fsspec logs that have a FileSize: attached to them.\n",
    "    \"\"\"\n",
    "    head_line  = re.compile('<File-like object S3FileSystem, .*?>\\s*(read: 0 - \\d+)')\n",
    "    fsize_line = re.compile('FileSize: (\\d+)')\n",
    "    range_line = re.compile('<File-like object S3FileSystem, .*?>\\s* read: (?P<start>[0-9]+) - (?P<end>[0-9]+)')\n",
    "\n",
    "    ranges = list()\n",
    "    with io.TextIOWrapper(io.BytesIO(content)) as logtxt:\n",
    "        for line in logtxt:\n",
    "            if head_line.match(line):\n",
    "                break\n",
    "        else:\n",
    "            raise RuntimeError('HEAD line not found in the log file')\n",
    "\n",
    "        for line in logtxt:\n",
    "            match = fsize_line.match(line)\n",
    "            if match:\n",
    "                fsize = int(match.group(1))\n",
    "                break\n",
    "        else:\n",
    "            raise RuntimeError('FILESIZE line not found in the log file')\n",
    "            \n",
    "        logtxt.seek(0)\n",
    "        for line in logtxt:\n",
    "            match = range_line.match(line)\n",
    "            if match:\n",
    "                start=int(match.group(1))\n",
    "                end=int(match.group(2))\n",
    "                rsize=end-start+1\n",
    "                range = ByteRange(start=int(match.group(1)), \n",
    "                                  end=int(match.group(2)),\n",
    "                                  filesize=fsize)\n",
    "                \n",
    "                ranges.append(range)\n",
    "\n",
    "    return ranges"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "502ba537-da68-4bc2-95bc-ccf73b1f3322",
   "metadata": {},
   "source": [
    "## Dashboard\n",
    "\n",
    "Function for generating log stats and plots:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8738d59c-3ff6-4d83-9ef2-4eeeb93ee851",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_ros3vfd_log(from_file):\n",
    "    if from_file is None:\n",
    "        return\n",
    "    elif len(from_file) == 0:\n",
    "        return [pn.pane.Alert('ros3vfd log file empty.', alert_type='danger')]\n",
    "    logging_schema = \"fsspec\"\n",
    "    try:\n",
    "        ranges = parse_fsspec_log(from_file)\n",
    "    except Exception as fsspec_e:\n",
    "        try: \n",
    "            ranges = parse_ros3vfd_log(from_file)\n",
    "            logging_schema = \"ros3vfd\"\n",
    "        except Exception as ros3_e:  \n",
    "            return [pn.pane.Alert(f'Error: ROS3 {str(ros3_e)}', alert_type='danger')]\n",
    "    start = np.fromiter([r.start for r in ranges], dtype=np.uint64)\n",
    "    end = np.fromiter([r.end for r in ranges], dtype=np.uint64)\n",
    "    req_no = np.arange(len(ranges)) + 1\n",
    "    sizes = np.fromiter([r.size for r in ranges], np.uint64)\n",
    "    info = pn.pane.Markdown(f\"\"\"\n",
    "# {logging_schema} Log Information\n",
    "\n",
    "Log size: {len(from_file):,} bytes\n",
    "\n",
    "HDF5 file size: {ranges[0].filesize:,} bytes\n",
    "\n",
    "Number of range _GET_ requests: {len(ranges):,}\n",
    "\n",
    "Overall range _GET_ requests stats:\n",
    "\n",
    "* Smallest: {np.min(sizes):,} bytes<br/>\n",
    "* Median: {int(np.median(sizes)):,} bytes<br/>\n",
    "* Largest: {np.max(sizes):,} bytes\n",
    "\n",
    "Maximum file byte read: {end.max():,}\n",
    "\n",
    "Total of file content read: {sizes.sum():,} bytes\n",
    "\n",
    "Percentage of content read to file size: {100 * (sizes.sum() / ranges[0].filesize) :.2f} %\n",
    "\"\"\")\n",
    "    data = dict(start=start, end=end, start_event=req_no, end_event=req_no)\n",
    "    max_offset_range = min(16_000_000, np.max(end))\n",
    "    req_range = np.where(end <= max_offset_range)[0]\n",
    "    if req_range.size == 0:\n",
    "        max_req_range = req_no[-1]\n",
    "    else:\n",
    "        max_req_range = req_no[np.where(end <= max_offset_range)[0][-1]] + 1\n",
    "    ros3plt = hv.Segments(\n",
    "        data, \n",
    "        [\n",
    "            hv.Dimension('start', label='File offset', range=(0, max_offset_range)),\n",
    "            hv.Dimension('start_event', label='Req. No.', range=(0, max_req_range)), \n",
    "            'end', \n",
    "            'end_event'\n",
    "        ]\n",
    "    )\n",
    "    hvrtip = HoverTool(\n",
    "        tooltips = [\n",
    "            ('req no', '@start_event'),\n",
    "            ('start byte', '@start'),\n",
    "            ('end byte', '@end')\n",
    "        ]\n",
    "    )\n",
    "    ros3plt.opts(width=700, height=600, invert_axes=True, color='blue', \n",
    "                 line_width=3, tools=[hvrtip])\n",
    "    size_hist = hv.Histogram(np.histogram(sizes, bins=512))\n",
    "    size_hist.opts(color='blue', line_color=None, tools=['hover'],\n",
    "                   xlabel='Size (bytes)', ylabel='Number of requests')\n",
    "    \n",
    "    return [pn.Row(info, size_hist), ros3plt]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1578b9a-1a8f-43cb-9902-96e21c83cf3a",
   "metadata": {},
   "source": [
    "### Dashboard Components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "553c70f5-6f78-4f28-9d6b-7998cbbc7ec2",
   "metadata": {},
   "outputs": [],
   "source": [
    "log_file = pn.widgets.FileInput()\n",
    "upld_form = pn.Row(\n",
    "    pn.pane.Markdown('Please select a ros3vfd or fsspec log file (limit 10MB):'),\n",
    "    log_file\n",
    ")\n",
    "res = pn.Column()\n",
    "app = pn.WidgetBox(upld_form, res)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15126686-d315-4ae4-8666-054dd6127ba5",
   "metadata": {},
   "source": [
    "Callback function for interactive log processing invocation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "322e3372-be21-4d65-b97e-9db5a552aaf0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def callback(value):\n",
    "    res.objects = plot_ros3vfd_log(value)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd2b2c83-1d2e-4db4-a23f-db9ad5d6f84d",
   "metadata": {},
   "source": [
    "Register callback with the appropriate dashboard object:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "580279df-8b79-426f-bb22-002f117f200a",
   "metadata": {},
   "outputs": [],
   "source": [
    "log_file.param.watch_values(callback, ['value']);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3f35ce7-0698-4b02-a95e-75107138a29a",
   "metadata": {},
   "source": [
    "Run the dashboard:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56c63583-922a-4b67-8cb0-8f3751ed7c28",
   "metadata": {},
   "outputs": [],
   "source": [
    "app.servable()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}