Notebooks/JuliaHTMRunTests.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "075e525c-ff95-4396-8a2f-3ec4d5395352",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m\u001b[1m  Activating\u001b[22m\u001b[39m project at `~/Projects/HTM/Notebooks`\n"
     ]
    }
   ],
   "source": [
    "] activate ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "35d8e7e7-190c-43d6-878a-9d28b5478364",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m\u001b[1m    Updating\u001b[22m\u001b[39m registry at `~/.julia/registries/General.toml`\n",
      "\u001b[32m\u001b[1m   Resolving\u001b[22m\u001b[39m package versions...\n",
      "\u001b[32m\u001b[1m  No Changes\u001b[22m\u001b[39m to `~/Projects/HTM/Notebooks/Project.toml`\n",
      "\u001b[32m\u001b[1m  No Changes\u001b[22m\u001b[39m to `~/Projects/HTM/Notebooks/Manifest.toml`\n"
     ]
    }
   ],
   "source": [
    "] add DataFrames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "362ddb4c-3471-4858-b24f-d68408347cf8",
   "metadata": {},
   "outputs": [],
   "source": [
    "using HierarchicalTemporalMemory\n",
    "using DataFrames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "27185e16-7516-435a-9fcf-352ab74bdeb0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "37"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_cols = 1024\n",
    "cells_per_col = 32\n",
    "small_incr = \n",
    "\n",
    "ps = TMParams(\n",
    "    Nc=num_cols,\n",
    "    k=cells_per_col,\n",
    ")\n",
    "\n",
    "tm = TemporalMemory(ps)\n",
    "\n",
    "input = BitArray(rand(num_cols, cells_per_col) .< cells_per_col / (num_cols * cells_per_col))\n",
    "sum(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "df7954c0-b992-4734-a939-9ef975ca3def",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"/Users/djpassey/.julia/packages/HierarchicalTemporalMemory/ETJjd/test/\""
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dir = \"/Users/djpassey/.julia/packages/HierarchicalTemporalMemory/ETJjd/test/\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f2bc999c-de27-41af-a8b7-06133af5181d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: creating Spatial Pooler\n",
      "└ @ Main In[6]:47\n",
      "┌ Info: creating Temporal Memory\n",
      "└ @ Main In[6]:60\n",
      "┌ Info: t=439\n",
      "└ @ Main In[6]:29\n",
      "┌ Info: t=878\n",
      "└ @ Main In[6]:29\n",
      "┌ Info: t=1317\n",
      "└ @ Main In[6]:29\n",
      "┌ Info: t=1756\n",
      "└ @ Main In[6]:29\n",
      "┌ Info: t=2195\n",
      "└ @ Main In[6]:29\n",
      "┌ Info: t=2634\n",
      "└ @ Main In[6]:29\n",
      "┌ Info: t=3073\n",
      "└ @ Main In[6]:29\n",
      "┌ Info: t=3512\n",
      "└ @ Main In[6]:29\n",
      "┌ Info: t=3951\n",
      "└ @ Main In[6]:29\n",
      "┌ Info: t=4390\n",
      "└ @ Main In[6]:29\n",
      "┌ Info: Prediction MASE: 1.105\n",
      "└ @ Main In[6]:91\n",
      "┌ Info: avg_TMout_sparsity: 0.898%\n",
      "└ @ Main In[6]:95\n",
      "┌ Info: avg_burst: 0.783%\n",
      "└ @ Main In[6]:96\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m\u001b[1mTest Passed\u001b[22m\u001b[39m\n",
       "  Expression: errormetric < 1.15\n",
       "   Evaluated: 1.1054595098880178 < 1.15"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Parameters for the test script (set in runtests.jl)\n",
    "#ENV[\"JULIA_DEBUG\"] = \"HierarchicalTemporalMemory\"\n",
    "plot_enabled= false\n",
    "\n",
    "using HierarchicalTemporalMemory, BenchmarkTools, CSV, Printf, Lazy, Test\n",
    "using Plots; gr()\n",
    "import Random: seed!, bitrand\n",
    "\n",
    "using StatsBase: median\n",
    "seed!(0)\n",
    "\n",
    "include(test_dir * \"utils/utils.jl\")\n",
    "\n",
    "plot_mase(data,pred,pred_timesteps)= begin\n",
    "  windowLength= 240\n",
    "  errormetric= zeros(length(data)-windowLength)\n",
    "  for w=1:length(data)-windowLength\n",
    "    errormetric[w]= mase(data[w:w+windowLength-1], pred[w:w+windowLength-1], pred_timesteps)\n",
    "  end\n",
    "  crit_t1= 1:500; crit_t2= 3000:3200\n",
    "  plot(plot(errormetric, label=\"10-day MASE\"),\n",
    "       plot([data pred], label=[\"timeseries\",\"prediction\"], legend=:none),\n",
    "       plot(plot(crit_t1, [data[crit_t1] pred[crit_t1]], legend= :none),\n",
    "            plot(crit_t2, [data[crit_t2] pred[crit_t2]], legend= :none),\n",
    "            layout= (1,2)),\n",
    "       layout= (3,1))|> display\n",
    "  @info @sprintf(\"Min 10-day MASE: %.2f\\n\",minimum(errormetric))\n",
    "end\n",
    "display_evaluation(t,sp,sp_activity,spDims)= @info(\"t=$t\")\n",
    "process_data!(tN,data,encParams,sp,tm,decoder)=\n",
    "  for t in 1:tN\n",
    "    z,a,power_bucket= _process_sp(t,tN,data,encParams,sp,display_evaluation)\n",
    "    A,Π,B= _process_tm(t,tN, tm,a)\n",
    "    prediction= predict!(decoder,Π,power_bucket)\n",
    "    likelyPred= reverse_simpleArithmetic(prediction,\"highmean\",encParams.power_p)\n",
    "    history_enc[:,t]= z;    history_SP[:,t]= a\n",
    "    history_TMout[:,t]= A;  history_TMpred[:,t]= Π\n",
    "    history_decodedPred[:,t]= prediction; history_likelyPred[t]= likelyPred\n",
    "\n",
    "    global avg_burst= ((t-1)*avg_burst+count(B)/length(B))/t\n",
    "  end\n",
    "\n",
    "prediction_timesteps=1\n",
    "inputDims= ((15,6,3).*25,)\n",
    "spDims= (1600,)\n",
    "k= 8\n",
    "@info \"creating Spatial Pooler\"\n",
    "sp= SpatialPooler(SPParams(\n",
    "      szᵢₙ= map(sum,inputDims), szₛₚ=spDims,\n",
    "      γ=600,\n",
    "      s=0.03,\n",
    "      prob_synapse=0.85,\n",
    "      θ_stimulus_activate=5,\n",
    "      p⁺_01= 0.20,\n",
    "      p⁻_01= 0.12,\n",
    "      β=3,\n",
    "      Tboost=350,\n",
    "      enable_local_inhibit=false,\n",
    "      enable_boosting=true))\n",
    "@info \"creating Temporal Memory\"\n",
    "tm= TemporalMemory(TMParams(\n",
    "      Nc=prod(spDims),\n",
    "      k=k,\n",
    "      θ_stimulus_activate=14,\n",
    "      θ_stimulus_learn=12,\n",
    "      synapseSampleSize=35,\n",
    "      p⁺_01=0.24,\n",
    "      p⁻_01=0.08,\n",
    "      LTD_p⁻_01= 0.012\n",
    "     ))\n",
    "\n",
    "Ncol= prod(spDims); Ncell= Ncol*k\n",
    "# Define input data\n",
    "data,tN= read_gympower(test_dir * \"test_data/gym_power_benchmark-extended.csv\")\n",
    "encParams= initenc_powerDay(data.power_hourly_kw, data.hour, data.is_weekend,\n",
    "                 encoder_size=inputDims[1], w=(34,35,35))\n",
    "decoder= SDRClassifier(Ncell,encParams.power_p.buckets,\n",
    "                  α=0.09, buffer_length=prediction_timesteps)\n",
    "# Histories\n",
    "history_enc= falses(map(sum,inputDims)|>prod,tN)\n",
    "history_SP= falses(Ncol,tN)\n",
    "history_TMout=  falses(Ncell,tN)\n",
    "history_TMpred= falses(Ncell,tN)\n",
    "history_decodedPred= zeros(encParams.power_p.buckets,tN)\n",
    "history_likelyPred= zeros(tN)\n",
    "avg_burst= 0\n",
    "\n",
    "process_data!(tN,data,encParams,sp,tm,decoder)\n",
    "\n",
    "errormetric= mase(data.power_hourly_kw[400:end], history_likelyPred[400:end],prediction_timesteps)\n",
    "@info @sprintf(\"Prediction MASE: %.3f\\n\", errormetric)\n",
    "\n",
    "avg_TMout_sparsity= mapslices(x->count(x)./length(x),history_TMout,dims=1)'|>median\n",
    "plot_enabled && plot_mase(data.power_hourly_kw, history_likelyPred, prediction_timesteps)\n",
    "@info @sprintf(\"avg_TMout_sparsity: %.3f%%\\n\", 100*avg_TMout_sparsity)\n",
    "@info @sprintf(\"avg_burst: %.3f%%\\n\", 100*avg_burst)\n",
    "\n",
    "# Sanity check: If this is not true, there's something very wrong with the temporal memory\n",
    "@test errormetric < 1.15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ad3ddca7-7b2e-4d9a-adef-2d21e558a452",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num cols 1600\n",
      "Cells per col 8\n",
      "Total cells 12800\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SparseArrays.SparseMatrixCSC{Bool, Int64}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "println(\"Num cols $(tm.params.Nc)\")\n",
    "println(\"Cells per col $(tm.params.k)\")\n",
    "println(\"Total cells $(tm.params.Nₙ)\")\n",
    "typeof(tm.distalSynapses.connected)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3544515-eeb1-4369-b23a-46d037efbf2e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "76718f9d-752f-4ee6-9182-d8a38e49a0dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "print_field_types (generic function with 1 method)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function print_field_types(obj)\n",
    "    println(typeof(obj))\n",
    "    for fn in fieldnames(typeof(obj))\n",
    "        println(fn, \": \\t\", typeof(getfield(obj, fn)))\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1111f770-efde-4f1c-ac3e-3e79c6c390d7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TemporalMemory\n",
      "params: \tTMParams\n",
      "distalSynapses: \tHierarchicalTemporalMemory.DistalSynapses\n",
      "previous: \tHierarchicalTemporalMemory.TMState\n",
      "recurrent: \tBool\n"
     ]
    }
   ],
   "source": [
    "print_field_types(tm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "126d0429-bef1-4e62-8fd2-48fb9294de0a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HierarchicalTemporalMemory.DistalSynapses\n",
      "Dd: \tSparseArrays.SparseMatrixCSC{UInt8, Int64}\n",
      "neurSeg: \tSparseArrays.SparseMatrixCSC{Bool, Int64}\n",
      "connected: \tSparseArrays.SparseMatrixCSC{Bool, Int64}\n",
      "segCol: \tSparseArrays.SparseMatrixCSC{Bool, Int64}\n",
      "k: \tInt64\n",
      "params: \tHierarchicalTemporalMemory.DistalSynapseParams\n",
      "init_Nseg: \tInt64\n"
     ]
    }
   ],
   "source": [
    "print_field_types(tm.distalSynapses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "59eae00c-f2d5-447d-9250-f9c93aa73fef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HierarchicalTemporalMemory.TMState\n",
      "state: \tNamedTuple{(:α, :Π, :WN, :Πₛ, :Mₛ, :ovp_Mₛ), Tuple{BitArray, BitArray, BitArray, BitVector, BitVector, Vector{Int64}}}\n",
      "init_Nseg: \tInt64\n"
     ]
    }
   ],
   "source": [
    "print_field_types(tm.previous)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9455e171-3676-4aa6-8cc1-5b2a1db26d90",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"600\" height=\"400\" viewBox=\"0 0 2400 1600\">\n",
       "<defs>\n",
       "  <clipPath id=\"clip040\">\n",
       "    <rect x=\"0\" y=\"0\" width=\"2400\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip040)\" d=\"\n",
       "M0 1600 L2400 1600 L2400 0 L0 0  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip041\">\n",
       "    <rect x=\"480\" y=\"0\" width=\"1681\" height=\"1600\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<path clip-path=\"url(#clip040)\" d=\"\n",
       "M195.778 1486.45 L2352.76 1486.45 L2352.76 47.2441 L195.778 47.2441  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip042\">\n",
       "    <rect x=\"195\" y=\"47\" width=\"2158\" height=\"1440\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  314.415,1486.45 314.415,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  683.589,1486.45 683.589,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1052.76,1486.45 1052.76,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1421.94,1486.45 1421.94,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  1791.11,1486.45 1791.11,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  2160.28,1486.45 2160.28,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip040)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  195.778,1486.45 2352.76,1486.45 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip040)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  314.415,1486.45 314.415,1467.55 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip040)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  683.589,1486.45 683.589,1467.55 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip040)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1052.76,1486.45 1052.76,1467.55 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip040)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1421.94,1486.45 1421.94,1467.55 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip040)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1791.11,1486.45 1791.11,1467.55 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip040)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  2160.28,1486.45 2160.28,1467.55 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip040)\" d=\"M314.415 1517.37 Q310.804 1517.37 308.976 1520.93 Q307.17 1524.47 307.17 1531.6 Q307.17 1538.71 308.976 1542.27 Q310.804 1545.82 314.415 1545.82 Q318.05 1545.82 319.855 1542.27 Q321.684 1538.71 321.684 1531.6 Q321.684 1524.47 319.855 1520.93 Q318.05 1517.37 314.415 1517.37 M314.415 1513.66 Q320.226 1513.66 323.281 1518.27 Q326.36 1522.85 326.36 1531.6 Q326.36 1540.33 323.281 1544.94 Q320.226 1549.52 314.415 1549.52 Q308.605 1549.52 305.527 1544.94 Q302.471 1540.33 302.471 1531.6 Q302.471 1522.85 305.527 1518.27 Q308.605 1513.66 314.415 1513.66 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M658.288 1514.29 L676.645 1514.29 L676.645 1518.22 L662.571 1518.22 L662.571 1526.7 Q663.589 1526.35 664.608 1526.19 Q665.626 1526 666.645 1526 Q672.432 1526 675.811 1529.17 Q679.191 1532.34 679.191 1537.76 Q679.191 1543.34 675.719 1546.44 Q672.246 1549.52 665.927 1549.52 Q663.751 1549.52 661.483 1549.15 Q659.237 1548.78 656.83 1548.04 L656.83 1543.34 Q658.913 1544.47 661.135 1545.03 Q663.358 1545.58 665.834 1545.58 Q669.839 1545.58 672.177 1543.48 Q674.515 1541.37 674.515 1537.76 Q674.515 1534.15 672.177 1532.04 Q669.839 1529.94 665.834 1529.94 Q663.959 1529.94 662.084 1530.35 Q660.233 1530.77 658.288 1531.65 L658.288 1514.29 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M698.404 1517.37 Q694.793 1517.37 692.964 1520.93 Q691.158 1524.47 691.158 1531.6 Q691.158 1538.71 692.964 1542.27 Q694.793 1545.82 698.404 1545.82 Q702.038 1545.82 703.844 1542.27 Q705.672 1538.71 705.672 1531.6 Q705.672 1524.47 703.844 1520.93 Q702.038 1517.37 698.404 1517.37 M698.404 1513.66 Q704.214 1513.66 707.269 1518.27 Q710.348 1522.85 710.348 1531.6 Q710.348 1540.33 707.269 1544.94 Q704.214 1549.52 698.404 1549.52 Q692.594 1549.52 689.515 1544.94 Q686.459 1540.33 686.459 1531.6 Q686.459 1522.85 689.515 1518.27 Q692.594 1513.66 698.404 1513.66 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M1012.37 1544.91 L1020.01 1544.91 L1020.01 1518.55 L1011.7 1520.21 L1011.7 1515.95 L1019.96 1514.29 L1024.64 1514.29 L1024.64 1544.91 L1032.28 1544.91 L1032.28 1548.85 L1012.37 1548.85 L1012.37 1544.91 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M1051.72 1517.37 Q1048.11 1517.37 1046.28 1520.93 Q1044.48 1524.47 1044.48 1531.6 Q1044.48 1538.71 1046.28 1542.27 Q1048.11 1545.82 1051.72 1545.82 Q1055.36 1545.82 1057.16 1542.27 Q1058.99 1538.71 1058.99 1531.6 Q1058.99 1524.47 1057.16 1520.93 Q1055.36 1517.37 1051.72 1517.37 M1051.72 1513.66 Q1057.53 1513.66 1060.59 1518.27 Q1063.67 1522.85 1063.67 1531.6 Q1063.67 1540.33 1060.59 1544.94 Q1057.53 1549.52 1051.72 1549.52 Q1045.91 1549.52 1042.83 1544.94 Q1039.78 1540.33 1039.78 1531.6 Q1039.78 1522.85 1042.83 1518.27 Q1045.91 1513.66 1051.72 1513.66 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M1081.88 1517.37 Q1078.27 1517.37 1076.44 1520.93 Q1074.64 1524.47 1074.64 1531.6 Q1074.64 1538.71 1076.44 1542.27 Q1078.27 1545.82 1081.88 1545.82 Q1085.52 1545.82 1087.32 1542.27 Q1089.15 1538.71 1089.15 1531.6 Q1089.15 1524.47 1087.32 1520.93 Q1085.52 1517.37 1081.88 1517.37 M1081.88 1513.66 Q1087.69 1513.66 1090.75 1518.27 Q1093.83 1522.85 1093.83 1531.6 Q1093.83 1540.33 1090.75 1544.94 Q1087.69 1549.52 1081.88 1549.52 Q1076.07 1549.52 1072.99 1544.94 Q1069.94 1540.33 1069.94 1531.6 Q1069.94 1522.85 1072.99 1518.27 Q1076.07 1513.66 1081.88 1513.66 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M1381.54 1544.91 L1389.18 1544.91 L1389.18 1518.55 L1380.87 1520.21 L1380.87 1515.95 L1389.14 1514.29 L1393.81 1514.29 L1393.81 1544.91 L1401.45 1544.91 L1401.45 1548.85 L1381.54 1548.85 L1381.54 1544.91 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M1410.94 1514.29 L1429.3 1514.29 L1429.3 1518.22 L1415.22 1518.22 L1415.22 1526.7 Q1416.24 1526.35 1417.26 1526.19 Q1418.28 1526 1419.3 1526 Q1425.08 1526 1428.46 1529.17 Q1431.84 1532.34 1431.84 1537.76 Q1431.84 1543.34 1428.37 1546.44 Q1424.9 1549.52 1418.58 1549.52 Q1416.4 1549.52 1414.14 1549.15 Q1411.89 1548.78 1409.48 1548.04 L1409.48 1543.34 Q1411.57 1544.47 1413.79 1545.03 Q1416.01 1545.58 1418.49 1545.58 Q1422.49 1545.58 1424.83 1543.48 Q1427.17 1541.37 1427.17 1537.76 Q1427.17 1534.15 1424.83 1532.04 Q1422.49 1529.94 1418.49 1529.94 Q1416.61 1529.94 1414.74 1530.35 Q1412.89 1530.77 1410.94 1531.65 L1410.94 1514.29 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M1451.06 1517.37 Q1447.45 1517.37 1445.62 1520.93 Q1443.81 1524.47 1443.81 1531.6 Q1443.81 1538.71 1445.62 1542.27 Q1447.45 1545.82 1451.06 1545.82 Q1454.69 1545.82 1456.5 1542.27 Q1458.32 1538.71 1458.32 1531.6 Q1458.32 1524.47 1456.5 1520.93 Q1454.69 1517.37 1451.06 1517.37 M1451.06 1513.66 Q1456.87 1513.66 1459.92 1518.27 Q1463 1522.85 1463 1531.6 Q1463 1540.33 1459.92 1544.94 Q1456.87 1549.52 1451.06 1549.52 Q1445.25 1549.52 1442.17 1544.94 Q1439.11 1540.33 1439.11 1531.6 Q1439.11 1522.85 1442.17 1518.27 Q1445.25 1513.66 1451.06 1513.66 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M1754.8 1544.91 L1771.12 1544.91 L1771.12 1548.85 L1749.18 1548.85 L1749.18 1544.91 Q1751.84 1542.16 1756.42 1537.53 Q1761.03 1532.88 1762.21 1531.53 Q1764.45 1529.01 1765.33 1527.27 Q1766.24 1525.51 1766.24 1523.82 Q1766.24 1521.07 1764.29 1519.33 Q1762.37 1517.6 1759.27 1517.6 Q1757.07 1517.6 1754.62 1518.36 Q1752.19 1519.13 1749.41 1520.68 L1749.41 1515.95 Q1752.23 1514.82 1754.69 1514.24 Q1757.14 1513.66 1759.18 1513.66 Q1764.55 1513.66 1767.74 1516.35 Q1770.94 1519.03 1770.94 1523.52 Q1770.94 1525.65 1770.13 1527.57 Q1769.34 1529.47 1767.23 1532.07 Q1766.65 1532.74 1763.55 1535.95 Q1760.45 1539.15 1754.8 1544.91 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M1790.94 1517.37 Q1787.33 1517.37 1785.5 1520.93 Q1783.69 1524.47 1783.69 1531.6 Q1783.69 1538.71 1785.5 1542.27 Q1787.33 1545.82 1790.94 1545.82 Q1794.57 1545.82 1796.38 1542.27 Q1798.2 1538.71 1798.2 1531.6 Q1798.2 1524.47 1796.38 1520.93 Q1794.57 1517.37 1790.94 1517.37 M1790.94 1513.66 Q1796.75 1513.66 1799.8 1518.27 Q1802.88 1522.85 1802.88 1531.6 Q1802.88 1540.33 1799.8 1544.94 Q1796.75 1549.52 1790.94 1549.52 Q1785.13 1549.52 1782.05 1544.94 Q1778.99 1540.33 1778.99 1531.6 Q1778.99 1522.85 1782.05 1518.27 Q1785.13 1513.66 1790.94 1513.66 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M1821.1 1517.37 Q1817.49 1517.37 1815.66 1520.93 Q1813.85 1524.47 1813.85 1531.6 Q1813.85 1538.71 1815.66 1542.27 Q1817.49 1545.82 1821.1 1545.82 Q1824.73 1545.82 1826.54 1542.27 Q1828.37 1538.71 1828.37 1531.6 Q1828.37 1524.47 1826.54 1520.93 Q1824.73 1517.37 1821.1 1517.37 M1821.1 1513.66 Q1826.91 1513.66 1829.96 1518.27 Q1833.04 1522.85 1833.04 1531.6 Q1833.04 1540.33 1829.96 1544.94 Q1826.91 1549.52 1821.1 1549.52 Q1815.29 1549.52 1812.21 1544.94 Q1809.15 1540.33 1809.15 1531.6 Q1809.15 1522.85 1812.21 1518.27 Q1815.29 1513.66 1821.1 1513.66 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M2123.98 1544.91 L2140.3 1544.91 L2140.3 1548.85 L2118.35 1548.85 L2118.35 1544.91 Q2121.01 1542.16 2125.6 1537.53 Q2130.2 1532.88 2131.38 1531.53 Q2133.63 1529.01 2134.51 1527.27 Q2135.41 1525.51 2135.41 1523.82 Q2135.41 1521.07 2133.47 1519.33 Q2131.55 1517.6 2128.44 1517.6 Q2126.24 1517.6 2123.79 1518.36 Q2121.36 1519.13 2118.58 1520.68 L2118.58 1515.95 Q2121.41 1514.82 2123.86 1514.24 Q2126.31 1513.66 2128.35 1513.66 Q2133.72 1513.66 2136.92 1516.35 Q2140.11 1519.03 2140.11 1523.52 Q2140.11 1525.65 2139.3 1527.57 Q2138.51 1529.47 2136.41 1532.07 Q2135.83 1532.74 2132.73 1535.95 Q2129.62 1539.15 2123.98 1544.91 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M2150.16 1514.29 L2168.51 1514.29 L2168.51 1518.22 L2154.44 1518.22 L2154.44 1526.7 Q2155.46 1526.35 2156.48 1526.19 Q2157.49 1526 2158.51 1526 Q2164.3 1526 2167.68 1529.17 Q2171.06 1532.34 2171.06 1537.76 Q2171.06 1543.34 2167.59 1546.44 Q2164.11 1549.52 2157.8 1549.52 Q2155.62 1549.52 2153.35 1549.15 Q2151.11 1548.78 2148.7 1548.04 L2148.7 1543.34 Q2150.78 1544.47 2153 1545.03 Q2155.23 1545.58 2157.7 1545.58 Q2161.71 1545.58 2164.05 1543.48 Q2166.38 1541.37 2166.38 1537.76 Q2166.38 1534.15 2164.05 1532.04 Q2161.71 1529.94 2157.7 1529.94 Q2155.83 1529.94 2153.95 1530.35 Q2152.1 1530.77 2150.16 1531.65 L2150.16 1514.29 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M2190.27 1517.37 Q2186.66 1517.37 2184.83 1520.93 Q2183.03 1524.47 2183.03 1531.6 Q2183.03 1538.71 2184.83 1542.27 Q2186.66 1545.82 2190.27 1545.82 Q2193.91 1545.82 2195.71 1542.27 Q2197.54 1538.71 2197.54 1531.6 Q2197.54 1524.47 2195.71 1520.93 Q2193.91 1517.37 2190.27 1517.37 M2190.27 1513.66 Q2196.08 1513.66 2199.14 1518.27 Q2202.22 1522.85 2202.22 1531.6 Q2202.22 1540.33 2199.14 1544.94 Q2196.08 1549.52 2190.27 1549.52 Q2184.46 1549.52 2181.38 1544.94 Q2178.33 1540.33 2178.33 1531.6 Q2178.33 1522.85 2181.38 1518.27 Q2184.46 1513.66 2190.27 1513.66 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  195.778,1066.65 2352.76,1066.65 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  195.778,571.009 2352.76,571.009 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:2; stroke-opacity:0.1; fill:none\" points=\"\n",
       "  195.778,75.367 2352.76,75.367 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip040)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  195.778,1486.45 195.778,47.2441 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip040)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  195.778,1066.65 214.675,1066.65 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip040)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  195.778,571.009 214.675,571.009 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip040)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  195.778,75.367 214.675,75.367 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip040)\" d=\"M53.5254 1086.44 L61.1642 1086.44 L61.1642 1060.08 L52.8541 1061.74 L52.8541 1057.49 L61.1179 1055.82 L65.7938 1055.82 L65.7938 1086.44 L73.4327 1086.44 L73.4327 1090.38 L53.5254 1090.38 L53.5254 1086.44 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M92.877 1058.9 Q89.2659 1058.9 87.4372 1062.46 Q85.6317 1066 85.6317 1073.13 Q85.6317 1080.24 87.4372 1083.8 Q89.2659 1087.35 92.877 1087.35 Q96.5113 1087.35 98.3168 1083.8 Q100.146 1080.24 100.146 1073.13 Q100.146 1066 98.3168 1062.46 Q96.5113 1058.9 92.877 1058.9 M92.877 1055.19 Q98.6872 1055.19 101.743 1059.8 Q104.821 1064.38 104.821 1073.13 Q104.821 1081.86 101.743 1086.47 Q98.6872 1091.05 92.877 1091.05 Q87.0669 1091.05 83.9882 1086.47 Q80.9326 1081.86 80.9326 1073.13 Q80.9326 1064.38 83.9882 1059.8 Q87.0669 1055.19 92.877 1055.19 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M117.517 1047.83 Q120.244 1048.41 121.767 1050.25 Q123.309 1052.1 123.309 1054.81 Q123.309 1058.96 120.451 1061.24 Q117.592 1063.51 112.326 1063.51 Q110.558 1063.51 108.677 1063.16 Q106.815 1062.82 104.821 1062.12 L104.821 1058.45 Q106.401 1059.38 108.282 1059.85 Q110.163 1060.32 112.213 1060.32 Q115.786 1060.32 117.648 1058.91 Q119.529 1057.5 119.529 1054.81 Q119.529 1052.32 117.78 1050.93 Q116.05 1049.52 112.946 1049.52 L109.674 1049.52 L109.674 1046.4 L113.097 1046.4 Q115.899 1046.4 117.385 1045.29 Q118.871 1044.16 118.871 1042.05 Q118.871 1039.89 117.329 1038.74 Q115.805 1037.58 112.946 1037.58 Q111.385 1037.58 109.599 1037.92 Q107.812 1038.26 105.668 1038.97 L105.668 1035.58 Q107.831 1034.98 109.711 1034.68 Q111.611 1034.38 113.285 1034.38 Q117.611 1034.38 120.131 1036.36 Q122.651 1038.31 122.651 1041.66 Q122.651 1043.99 121.316 1045.61 Q119.98 1047.21 117.517 1047.83 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M130.513 1058.19 L134.481 1058.19 L134.481 1062.97 L130.513 1062.97 L130.513 1058.19 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M142.794 1034.89 L157.709 1034.89 L157.709 1038.09 L146.274 1038.09 L146.274 1044.97 Q147.101 1044.69 147.929 1044.56 Q148.756 1044.41 149.584 1044.41 Q154.286 1044.41 157.032 1046.98 Q159.778 1049.56 159.778 1053.96 Q159.778 1058.49 156.957 1061.01 Q154.135 1063.51 149.001 1063.51 Q147.233 1063.51 145.39 1063.21 Q143.565 1062.91 141.609 1062.31 L141.609 1058.49 Q143.302 1059.41 145.108 1059.87 Q146.913 1060.32 148.926 1060.32 Q152.179 1060.32 154.079 1058.61 Q155.979 1056.89 155.979 1053.96 Q155.979 1051.03 154.079 1049.31 Q152.179 1047.6 148.926 1047.6 Q147.402 1047.6 145.879 1047.94 Q144.374 1048.28 142.794 1048.99 L142.794 1034.89 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M51.6634 590.802 L59.3023 590.802 L59.3023 564.436 L50.9921 566.103 L50.9921 561.843 L59.256 560.177 L63.9319 560.177 L63.9319 590.802 L71.5707 590.802 L71.5707 594.737 L51.6634 594.737 L51.6634 590.802 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M91.0151 563.255 Q87.404 563.255 85.5753 566.82 Q83.7697 570.362 83.7697 577.491 Q83.7697 584.598 85.5753 588.163 Q87.404 591.704 91.0151 591.704 Q94.6493 591.704 96.4548 588.163 Q98.2835 584.598 98.2835 577.491 Q98.2835 570.362 96.4548 566.82 Q94.6493 563.255 91.0151 563.255 M91.0151 559.552 Q96.8252 559.552 99.8808 564.158 Q102.959 568.741 102.959 577.491 Q102.959 586.218 99.8808 590.825 Q96.8252 595.408 91.0151 595.408 Q85.2049 595.408 82.1262 590.825 Q79.0707 586.218 79.0707 577.491 Q79.0707 568.741 82.1262 564.158 Q85.2049 559.552 91.0151 559.552 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M115.636 542.556 L106.044 557.546 L115.636 557.546 L115.636 542.556 M114.639 539.246 L119.416 539.246 L119.416 557.546 L123.422 557.546 L123.422 560.706 L119.416 560.706 L119.416 567.326 L115.636 567.326 L115.636 560.706 L102.959 560.706 L102.959 557.038 L114.639 539.246 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M129.704 562.549 L133.673 562.549 L133.673 567.326 L129.704 567.326 L129.704 562.549 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M150.073 541.748 Q147.139 541.748 145.653 544.644 Q144.186 547.522 144.186 553.315 Q144.186 559.089 145.653 561.985 Q147.139 564.862 150.073 564.862 Q153.026 564.862 154.493 561.985 Q155.979 559.089 155.979 553.315 Q155.979 547.522 154.493 544.644 Q153.026 541.748 150.073 541.748 M150.073 538.738 Q154.794 538.738 157.276 542.481 Q159.778 546.205 159.778 553.315 Q159.778 560.405 157.276 564.148 Q154.794 567.872 150.073 567.872 Q145.352 567.872 142.851 564.148 Q140.368 560.405 140.368 553.315 Q140.368 546.205 142.851 542.481 Q145.352 538.738 150.073 538.738 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M52.4722 95.1594 L60.111 95.1594 L60.111 68.7938 L51.8009 70.4605 L51.8009 66.2012 L60.0647 64.5346 L64.7406 64.5346 L64.7406 95.1594 L72.3795 95.1594 L72.3795 99.0946 L52.4722 99.0946 L52.4722 95.1594 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M91.8238 67.6132 Q88.2127 67.6132 86.384 71.178 Q84.5785 74.7197 84.5785 81.8493 Q84.5785 88.9557 86.384 92.5205 Q88.2127 96.0622 91.8238 96.0622 Q95.458 96.0622 97.2636 92.5205 Q99.0923 88.9557 99.0923 81.8493 Q99.0923 74.7197 97.2636 71.178 Q95.458 67.6132 91.8238 67.6132 M91.8238 63.9096 Q97.6339 63.9096 100.689 68.516 Q103.768 73.0993 103.768 81.8493 Q103.768 90.5761 100.689 95.1825 Q97.6339 99.7659 91.8238 99.7659 Q86.0136 99.7659 82.935 95.1825 Q79.8794 90.5761 79.8794 81.8493 Q79.8794 73.0993 82.935 68.516 Q86.0136 63.9096 91.8238 63.9096 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M116.445 46.9143 L106.853 61.9041 L116.445 61.9041 L116.445 46.9143 M115.448 43.6042 L120.225 43.6042 L120.225 61.9041 L124.231 61.9041 L124.231 65.0638 L120.225 65.0638 L120.225 71.6842 L116.445 71.6842 L116.445 65.0638 L103.768 65.0638 L103.768 61.3963 L115.448 43.6042 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M130.513 66.907 L134.481 66.907 L134.481 71.6842 L130.513 71.6842 L130.513 66.907 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M142.794 43.6042 L157.709 43.6042 L157.709 46.8015 L146.274 46.8015 L146.274 53.6851 Q147.101 53.403 147.929 53.2714 Q148.756 53.1209 149.584 53.1209 Q154.286 53.1209 157.032 55.6976 Q159.778 58.2742 159.778 62.6752 Q159.778 67.2079 156.957 69.7282 Q154.135 72.2296 149.001 72.2296 Q147.233 72.2296 145.39 71.9287 Q143.565 71.6277 141.609 71.0259 L141.609 67.2079 Q143.302 68.1295 145.108 68.5809 Q146.913 69.0323 148.926 69.0323 Q152.179 69.0323 154.079 67.3208 Q155.979 65.6092 155.979 62.6752 Q155.979 59.7412 154.079 58.0297 Q152.179 56.3182 148.926 56.3182 Q147.402 56.3182 145.879 56.6568 Q144.374 56.9953 142.794 57.71 L142.794 43.6042 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip042)\" d=\"\n",
       "M314.415 807.304 L314.415 1445.72 L388.25 1445.72 L388.25 807.304 L314.415 807.304 L314.415 807.304  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  314.415,807.304 314.415,1445.72 388.25,1445.72 388.25,807.304 314.415,807.304 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M388.25 1068.6 L388.25 1445.72 L462.085 1445.72 L462.085 1068.6 L388.25 1068.6 L388.25 1068.6  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  388.25,1068.6 388.25,1445.72 462.085,1445.72 462.085,1068.6 388.25,1068.6 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M462.085 934.346 L462.085 1445.72 L535.92 1445.72 L535.92 934.346 L462.085 934.346 L462.085 934.346  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  462.085,934.346 462.085,1445.72 535.92,1445.72 535.92,934.346 462.085,934.346 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M535.92 974.068 L535.92 1445.72 L609.754 1445.72 L609.754 974.068 L535.92 974.068 L535.92 974.068  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  535.92,974.068 535.92,1445.72 609.754,1445.72 609.754,974.068 535.92,974.068 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M609.754 870.795 L609.754 1445.72 L683.589 1445.72 L683.589 870.795 L609.754 870.795 L609.754 870.795  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  609.754,870.795 609.754,1445.72 683.589,1445.72 683.589,870.795 609.754,870.795 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M683.589 931.552 L683.589 1445.72 L757.424 1445.72 L757.424 931.552 L683.589 931.552 L683.589 931.552  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  683.589,931.552 683.589,1445.72 757.424,1445.72 757.424,931.552 683.589,931.552 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M757.424 767.399 L757.424 1445.72 L831.258 1445.72 L831.258 767.399 L757.424 767.399 L757.424 767.399  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  757.424,767.399 757.424,1445.72 831.258,1445.72 831.258,767.399 757.424,767.399 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M831.258 1023.58 L831.258 1445.72 L905.093 1445.72 L905.093 1023.58 L831.258 1023.58 L831.258 1023.58  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  831.258,1023.58 831.258,1445.72 905.093,1445.72 905.093,1023.58 831.258,1023.58 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M905.093 971.114 L905.093 1445.72 L978.928 1445.72 L978.928 971.114 L905.093 971.114 L905.093 971.114  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  905.093,971.114 905.093,1445.72 978.928,1445.72 978.928,971.114 905.093,971.114 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M978.928 923.569 L978.928 1445.72 L1052.76 1445.72 L1052.76 923.569 L978.928 923.569 L978.928 923.569  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  978.928,923.569 978.928,1445.72 1052.76,1445.72 1052.76,923.569 978.928,923.569 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M1052.76 904.004 L1052.76 1445.72 L1126.6 1445.72 L1126.6 904.004 L1052.76 904.004 L1052.76 904.004  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1052.76,904.004 1052.76,1445.72 1126.6,1445.72 1126.6,904.004 1052.76,904.004 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M1126.6 1101.7 L1126.6 1445.72 L1200.43 1445.72 L1200.43 1101.7 L1126.6 1101.7 L1126.6 1101.7  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1126.6,1101.7 1126.6,1445.72 1200.43,1445.72 1200.43,1101.7 1126.6,1101.7 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M1200.43 1143.55 L1200.43 1445.72 L1274.27 1445.72 L1274.27 1143.55 L1200.43 1143.55 L1200.43 1143.55  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1200.43,1143.55 1200.43,1445.72 1274.27,1445.72 1274.27,1143.55 1200.43,1143.55 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M1274.27 1133.42 L1274.27 1445.72 L1348.1 1445.72 L1348.1 1133.42 L1274.27 1133.42 L1274.27 1133.42  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1274.27,1133.42 1274.27,1445.72 1348.1,1445.72 1348.1,1133.42 1274.27,1133.42 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M1348.1 1147.31 L1348.1 1445.72 L1421.94 1445.72 L1421.94 1147.31 L1348.1 1147.31 L1348.1 1147.31  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1348.1,1147.31 1348.1,1445.72 1421.94,1445.72 1421.94,1147.31 1348.1,1147.31 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M1421.94 1070.93 L1421.94 1445.72 L1495.77 1445.72 L1495.77 1070.93 L1421.94 1070.93 L1421.94 1070.93  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1421.94,1070.93 1421.94,1445.72 1495.77,1445.72 1495.77,1070.93 1421.94,1070.93 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M1495.77 1078.98 L1495.77 1445.72 L1569.61 1445.72 L1569.61 1078.98 L1495.77 1078.98 L1495.77 1078.98  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1495.77,1078.98 1495.77,1445.72 1569.61,1445.72 1569.61,1078.98 1495.77,1078.98 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M1569.61 1123.67 L1569.61 1445.72 L1643.44 1445.72 L1643.44 1123.67 L1569.61 1123.67 L1569.61 1123.67  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1569.61,1123.67 1569.61,1445.72 1643.44,1445.72 1643.44,1123.67 1569.61,1123.67 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M1643.44 1107.65 L1643.44 1445.72 L1717.28 1445.72 L1717.28 1107.65 L1643.44 1107.65 L1643.44 1107.65  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1643.44,1107.65 1643.44,1445.72 1717.28,1445.72 1717.28,1107.65 1643.44,1107.65 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M1717.28 919.971 L1717.28 1445.72 L1791.11 1445.72 L1791.11 919.971 L1717.28 919.971 L1717.28 919.971  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1717.28,919.971 1717.28,1445.72 1791.11,1445.72 1791.11,919.971 1717.28,919.971 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M1791.11 987.79 L1791.11 1445.72 L1864.94 1445.72 L1864.94 987.79 L1791.11 987.79 L1791.11 987.79  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1791.11,987.79 1791.11,1445.72 1864.94,1445.72 1864.94,987.79 1791.11,987.79 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M1864.94 869.846 L1864.94 1445.72 L1938.78 1445.72 L1938.78 869.846 L1864.94 869.846 L1864.94 869.846  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1864.94,869.846 1864.94,1445.72 1938.78,1445.72 1938.78,869.846 1864.94,869.846 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M1938.78 874.09 L1938.78 1445.72 L2012.61 1445.72 L2012.61 874.09 L1938.78 874.09 L1938.78 874.09  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1938.78,874.09 1938.78,1445.72 2012.61,1445.72 2012.61,874.09 1938.78,874.09 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M2012.61 855.355 L2012.61 1445.72 L2086.45 1445.72 L2086.45 855.355 L2012.61 855.355 L2012.61 855.355  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  2012.61,855.355 2012.61,1445.72 2086.45,1445.72 2086.45,855.355 2012.61,855.355 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M2086.45 812.327 L2086.45 1445.72 L2160.28 1445.72 L2160.28 812.327 L2086.45 812.327 L2086.45 812.327  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  2086.45,812.327 2086.45,1445.72 2160.28,1445.72 2160.28,812.327 2086.45,812.327 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip042)\" d=\"\n",
       "M2160.28 87.9763 L2160.28 1445.72 L2234.12 1445.72 L2234.12 87.9763 L2160.28 87.9763 L2160.28 87.9763  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip042)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  2160.28,87.9763 2160.28,1445.72 2234.12,1445.72 2234.12,87.9763 2160.28,87.9763 \n",
       "  \"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"351.333\" cy=\"807.304\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"425.167\" cy=\"1068.6\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"499.002\" cy=\"934.346\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"572.837\" cy=\"974.068\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"646.672\" cy=\"870.795\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"720.506\" cy=\"931.552\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"794.341\" cy=\"767.399\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"868.176\" cy=\"1023.58\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"942.011\" cy=\"971.114\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"1015.85\" cy=\"923.569\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"1089.68\" cy=\"904.004\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"1163.51\" cy=\"1101.7\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"1237.35\" cy=\"1143.55\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"1311.18\" cy=\"1133.42\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"1385.02\" cy=\"1147.31\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"1458.85\" cy=\"1070.93\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"1532.69\" cy=\"1078.98\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"1606.52\" cy=\"1123.67\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"1680.36\" cy=\"1107.65\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"1754.19\" cy=\"919.971\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"1828.03\" cy=\"987.79\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"1901.86\" cy=\"869.846\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"1975.7\" cy=\"874.09\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"2049.53\" cy=\"855.355\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"2123.37\" cy=\"812.327\" r=\"2\"/>\n",
       "<circle clip-path=\"url(#clip042)\" style=\"fill:#009af9; stroke:none; fill-opacity:0\" cx=\"2197.2\" cy=\"87.9763\" r=\"2\"/>\n",
       "<path clip-path=\"url(#clip040)\" d=\"\n",
       "M1988.76 198.898 L2280.86 198.898 L2280.86 95.2176 L1988.76 95.2176  Z\n",
       "  \" fill=\"#ffffff\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip040)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  1988.76,198.898 2280.86,198.898 2280.86,95.2176 1988.76,95.2176 1988.76,198.898 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip040)\" d=\"\n",
       "M2012.73 167.794 L2156.53 167.794 L2156.53 126.322 L2012.73 126.322 L2012.73 167.794  Z\n",
       "  \" fill=\"#009af9\" fill-rule=\"evenodd\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip040)\" style=\"stroke:#000000; stroke-linecap:butt; stroke-linejoin:round; stroke-width:4; stroke-opacity:1; fill:none\" points=\"\n",
       "  2012.73,167.794 2156.53,167.794 2156.53,126.322 2012.73,126.322 2012.73,167.794 \n",
       "  \"/>\n",
       "<path clip-path=\"url(#clip040)\" d=\"M2194.34 166.745 Q2192.53 171.375 2190.82 172.787 Q2189.1 174.199 2186.23 174.199 L2182.83 174.199 L2182.83 170.634 L2185.33 170.634 Q2187.09 170.634 2188.06 169.8 Q2189.04 168.967 2190.22 165.865 L2190.98 163.921 L2180.49 138.412 L2185.01 138.412 L2193.11 158.689 L2201.21 138.412 L2205.72 138.412 L2194.34 166.745 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /><path clip-path=\"url(#clip040)\" d=\"M2213.02 160.402 L2220.66 160.402 L2220.66 134.037 L2212.35 135.703 L2212.35 131.444 L2220.61 129.778 L2225.28 129.778 L2225.28 160.402 L2232.92 160.402 L2232.92 164.338 L2213.02 164.338 L2213.02 160.402 Z\" fill=\"#000000\" fill-rule=\"evenodd\" fill-opacity=\"1\" /></svg>\n"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "syn_weights = reshape(Array(Float64.(tm.distalSynapses.Dd)), :)\n",
    "\n",
    "histogram(syn_weights[syn_weights .> 1], yaxis=:log10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3411c83-a3d1-4c29-b77d-7f3955129983",
   "metadata": {},
   "source": [
    "The weights here were stored in UInt8. A nice idea. But this shows the distribution. I think that initially it was uniform."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1548c4b8-1bbc-4a0d-bab7-ae5efeba2c81",
   "metadata": {},
   "outputs": [],
   "source": [
    "function _process_sp(t,tN,data,encParams,sp,display_evaluation=identity)\n",
    "  # z: encoder activation\n",
    "  z, bucket= encode_powerDay(data.power_hourly_kw[t], data.hour[t], data.is_weekend[t];\n",
    "                       encParams...)\n",
    "  # a: SP activation\n",
    "  a= step!(sp,z)\n",
    "  t%(tN÷10)==0 && display_evaluation(t,sp,a,sp.params.szₛₚ)\n",
    "  return z,a,bucket\n",
    "end\n",
    "\n",
    "_process_tm(t,tN, tm,a) = step!(tm,a)\n",
    "\n",
    "identity(a...) = a\n",
    "\n",
    "function read_gympower(dataset_filename= \"test/test_data/gym_power_benchmark.csv\")\n",
    "  data= CSV.read(dataset_filename, DataFrame)\n",
    "  ((power_hourly_kw= data.power_hourly_kw,\n",
    "    hour= data.hour,\n",
    "    is_weekend= data.is_weekend\n",
    "   ), size(data,1))\n",
    "end\n",
    "\n",
    "\"\"\"\n",
    "Find the top-k% most similar SDRs to the last one from the entire history.\n",
    "\"\"\"\n",
    "function top_similar_sdr(sdrHist,k)\n",
    "  k= size(sdrHist,2)==1 ? 0 : ceil(Int,k/100*size(sdrHist,2))\n",
    "  # Shame: I can't find ANY high-level construct to even approach the efficiency of this!\n",
    "  overlap= zeros(Int32,size(sdrHist,2)-1)\n",
    "  for t in 1:length(overlap) for i in 1:size(sdrHist,1)\n",
    "      overlap[t]+= sdrHist[i,end] & sdrHist[i,t] end end\n",
    "  topIdx= @> overlap partialsortperm(1:k,rev=true)\n",
    "  (sdrHist[:,topIdx], topIdx)\n",
    "end\n",
    "\n",
    "# Calculate prediction \"mean absolute scaled error\"\n",
    "# (https://en.wikipedia.org/wiki/Mean_absolute_scaled_error)\n",
    "function mase(data,prediction,predict_timesteps)\n",
    "  aligned_data= data[1+predict_timesteps:end]\n",
    "  aligned_pred= prediction[1:end-predict_timesteps]\n",
    "  mean(abs.(aligned_data .- aligned_pred)) / mean(abs.(diff(aligned_data)))\n",
    "end\n",
    "\n",
    "plot_ts_similarEncSp(t,ts,encOnly,spOnly,encANDspHistory)=\n",
    "    if t>1 _plot_ts_similarEncSP(t,ts,encOnly,spOnly,encANDspHistory)\n",
    "    end\n",
    "\n",
    "function _plot_ts_similarEncSP(t,ts,encOnly,spOnly,encANDspHistory)\n",
    "  graph_ts()= begin\n",
    "    graph_ts= plot(ts, label=\"power\", ylims=(0.9minimum(ts),1.05maximum(ts)),\n",
    "              xlims=(1,1.3length(ts)))\n",
    "    scatter!(graph_ts, encOnly, ts[encOnly], label=\"encoding only\")\n",
    "    scatter!(graph_ts, spOnly, ts[spOnly], label=\"SP only\")\n",
    "    scatter!(graph_ts, encANDspHistory[t][1], ts[encANDspHistory[t][1]],\n",
    "              label=\"overlapping\", legendfontsize=6)\n",
    "    vline!(graph_ts,[t], label=\"\")\n",
    "    title!(graph_ts, \"Spatial Pooler mapping property evaluation\", titlefont=font(10))\n",
    "    return graph_ts\n",
    "  end\n",
    "  graph_ovp(overlapFraction)= begin\n",
    "    graph_ovp= plot(1:t,overlapFraction*100, label=\"\", linestyle=:dash,\n",
    "        ylabel=\"overlap%\",ylims=(-1,102),xlims=(1,1.3length(ts)))\n",
    "    title!(graph_ovp, \"Percentage of overlapping encoder and SP SDRs\", titlefont=font(10))\n",
    "    return graph_ovp\n",
    "  end\n",
    "  overlapFraction= map(x->length(x.encANDsp)/x.Nenc, encANDspHistory[1:t])\n",
    "  overlapFraction[isnan.(overlapFraction)].= 1.0\n",
    "  plot(graph_ts(),graph_ovp(overlapFraction), layout=(2,1))|> display\n",
    "end\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c2940c1-a9cf-4d66-b961-d04cd7be117c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameters for the test script (set in runtests.jl)\n",
    "#ENV[\"JULIA_DEBUG\"] = \"HierarchicalTemporalMemory\"\n",
    "plot_enabled= false\n",
    "\n",
    "using HierarchicalTemporalMemory, BenchmarkTools, CSV, Printf, Lazy, Test\n",
    "using Plots; gr()\n",
    "import Random: seed!, bitrand\n",
    "\n",
    "using StatsBase: median\n",
    "seed!(0)\n",
    "\n",
    "function plot_mase(data, pred, pred_timesteps)\n",
    "    windowLength= 240\n",
    "    n = length(data)-windowLength\n",
    "    errormetric= zeros(n)\n",
    "    # Compute error for each timepoint over a sliding window\n",
    "    for w in 1:n\n",
    "        errormetric[w]= mase(\n",
    "            data[w:w+windowLength-1], \n",
    "            pred[w:w+windowLength-1], \n",
    "            pred_timesteps)\n",
    "    end\n",
    "    \n",
    "    # Critical time values\n",
    "    crit_t1= 1:500\n",
    "    crit_t2= 3000:3200\n",
    "    \n",
    "    # Plots\n",
    "    p_mase = plot(errormetric, label=\"10-day MASE\")\n",
    "    p_pred = plot([data pred], label=[\"timeseries\",\"prediction\"], legend=:none)\n",
    "    p_crit = plot(\n",
    "                  plot(crit_t1, [data[crit_t1] pred[crit_t1]], legend= :none),\n",
    "                  plot(crit_t2, [data[crit_t2] pred[crit_t2]], legend= :none),\n",
    "                  layout= (1,2)\n",
    "    )\n",
    "    \n",
    "    p_all = plot(p_mase, p_pred, p_crit, layout= (3,1))\n",
    "    display(p_all)\n",
    "    @info @sprintf(\"Min 10-day MASE: %.2f\\n\",minimum(errormetric))\n",
    "end\n",
    "\n",
    "display_evaluation(t,sp,sp_activity,spDims)= @info(\"t=$t\")\n",
    "\n",
    "function process_data!(tN, data, encParams, sp, tm, decoder)\n",
    "    for t in 1:tN\n",
    "        # Retrieves something from spaial pooler\n",
    "        z, a, power_bucket = _process_sp(t, tN, data, encParams, sp, \n",
    "                                         display_evaluation)\n",
    "        # Retrives something from temporal memory\n",
    "        A, Π, B = _process_tm(t,tN, tm,a)\n",
    "        # Makes prediction based on neurons in predictive state, Π\n",
    "        prediction= predict!(decoder,Π,power_bucket)\n",
    "        # A likelilhood prediction?\n",
    "        likelyPred= reverse_simpleArithmetic(prediction, \"highmean\", encParams.power_p)\n",
    "        # Store everything. I think these are all global!\n",
    "        history_enc[:,t]= z\n",
    "        history_SP[:,t]= a\n",
    "        history_TMout[:,t]= A\n",
    "        history_TMpred[:,t]= Π\n",
    "        history_decodedPred[:,t]= prediction\n",
    "        history_likelyPred[t]= likelyPred\n",
    "        # Some kind of bursting parameter, I don't know what it is. Update for each t\n",
    "        global avg_burst= ((t-1)*avg_burst+count(B)/length(B))/t\n",
    "    end\n",
    "end\n",
    "\n",
    "prediction_timesteps=1\n",
    "inputDims= ((15,6,3).*25,)\n",
    "spDims= (1600,)\n",
    "k= 8\n",
    "\n",
    "@info \"creating Spatial Pooler\"\n",
    "sp= SpatialPooler(SPParams(\n",
    "      szᵢₙ= map(sum,inputDims), szₛₚ=spDims,\n",
    "      γ=600,\n",
    "      s=0.03,\n",
    "      prob_synapse=0.85,\n",
    "      θ_stimulus_activate=5,\n",
    "      p⁺_01= 0.20,\n",
    "      p⁻_01= 0.12,\n",
    "      β=3,\n",
    "      Tboost=350,\n",
    "      enable_local_inhibit=false,\n",
    "      enable_boosting=true))\n",
    "\n",
    "@info \"creating Temporal Memory\"\n",
    "tm= TemporalMemory(TMParams(\n",
    "      Nc=prod(spDims),\n",
    "      k=k,\n",
    "      θ_stimulus_activate=14,\n",
    "      θ_stimulus_learn=12,\n",
    "      synapseSampleSize=35,\n",
    "      p⁺_01=0.24,\n",
    "      p⁻_01=0.08,\n",
    "      LTD_p⁻_01= 0.012\n",
    "     ))\n",
    "\n",
    "# Number of columns\n",
    "Ncol= prod(spDims); \n",
    "# Number of cells\n",
    "Ncell= Ncol*k\n",
    "# Define input data\n",
    "data,tN= read_gympower(test_dir * \"test_data/gym_power_benchmark-extended.csv\")\n",
    "\n",
    "# Creates encoder\n",
    "encParams= initenc_powerDay(data.power_hourly_kw, data.hour, data.is_weekend,\n",
    "                            encoder_size=inputDims[1], w=(34,35,35))\n",
    "\n",
    "# Creates decoding classifier\n",
    "decoder= SDRClassifier(Ncell, encParams.power_p.buckets,\n",
    "                       α=0.09, buffer_length=prediction_timesteps)\n",
    "\n",
    "# Initialize empty histories\n",
    "history_enc= falses(map(sum,inputDims)|>prod,tN)\n",
    "history_SP= falses(Ncol,tN)\n",
    "history_TMout=  falses(Ncell,tN)\n",
    "history_TMpred= falses(Ncell,tN)\n",
    "history_decodedPred= zeros(encParams.power_p.buckets,tN)\n",
    "history_likelyPred= zeros(tN)\n",
    "avg_burst= 0\n",
    "\n",
    "# Does all training occur in here? I'd better grab it.\n",
    "process_data!(tN,data,encParams,sp,tm,decoder)\n",
    "\n",
    "errormetric= mase(data.power_hourly_kw[400:end], history_likelyPred[400:end],prediction_timesteps)\n",
    "@info @sprintf(\"Prediction MASE: %.3f\\n\", errormetric)\n",
    "\n",
    "avg_TMout_sparsity= mapslices(x->count(x)./length(x),history_TMout,dims=1)'|>median\n",
    "plot_enabled && plot_mase(data.power_hourly_kw, history_likelyPred, prediction_timesteps)\n",
    "@info @sprintf(\"avg_TMout_sparsity: %.3f%%\\n\", 100*avg_TMout_sparsity)\n",
    "@info @sprintf(\"avg_burst: %.3f%%\\n\", 100*avg_burst)\n",
    "\n",
    "# Sanity check: If this is not true, there's something very wrong with the temporal memory\n",
    "@test errormetric < 1.15"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.7.3",
   "language": "julia",
   "name": "julia-1.7"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}