Assignment1_Ell409.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Assignment1-Ell409.ipynb",
      "version": "0.3.2",
      "provenance": []
    },
    "kernelspec": {
      "name": "python2",
      "display_name": "Python 2"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "ji2rU4xyYigY",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# This Notebook contains python implementation of different statistical approaches for data classifcation. Several performance measures are also observed.                 Report: https://tinyurl.com/report-mil\n",
        "\n",
        "---\n",
        "The code is written in python and divided in many section\n"
      ]
    },
    {
      "metadata": {
        "id": "FhwSiQLUYz9W",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Section 1: \n",
        "Reading data from the mnist binary data file."
      ]
    },
    {
      "metadata": {
        "id": "9MXJxW6JXFk9",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def load_mnist(path, kind='train'):\n",
        "    import os\n",
        "    import gzip\n",
        "    import numpy as np\n",
        "\n",
        "    \"\"\"Load MNIST data from `path`\"\"\"\n",
        "    labels_path = os.path.join(path,\n",
        "                               '%s-labels-idx1-ubyte.gz'\n",
        "                               % kind)\n",
        "    images_path = os.path.join(path,\n",
        "                               '%s-images-idx3-ubyte.gz'\n",
        "                               % kind)\n",
        "\n",
        "    with gzip.open(labels_path, 'rb') as lbpath:\n",
        "        labels = np.frombuffer(lbpath.read(), dtype=np.uint8,\n",
        "                               offset=8)\n",
        "\n",
        "    with gzip.open(images_path, 'rb') as imgpath:\n",
        "        images = np.frombuffer(imgpath.read(), dtype=np.uint8,\n",
        "                               offset=16).reshape(len(labels), 784)\n",
        "\n",
        "    return images, labels\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "lfPfb5EYY77t",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Importing required python Module and load data"
      ]
    },
    {
      "metadata": {
        "id": "oHJIO1UeZNXX",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import numpy\n",
        "import math,copy\n",
        "import csv\n",
        "import collections\n",
        "import matplotlib.pyplot as plt\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "1bqDdQmyZX-w",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Section 2: \n",
        "Bayesian Classifier with Maximum Likelihood Estimation"
      ]
    },
    {
      "metadata": {
        "id": "OqdDKYPaZbAR",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# PART A\n",
        "Fashion Minst dataset"
      ]
    },
    {
      "metadata": {
        "id": "R5ybK7AUZSic",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "mus = []\n",
        "training_freq = {}\n",
        "training_data = []\n",
        "test_data = []\n",
        "sigs = []\n",
        "dets = 1\n",
        "priori = []\n",
        "m0 = 1\n",
        "s0 = 1\n",
        "n = 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Sxc3VgiaZe4v",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#\n",
        "def cal_priori():\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    global mus\n",
        "    dic = training_freq.copy()\n",
        "    for i in dic.keys():\n",
        "        dic[i] = float(dic.get(i))/float(len(training_data))\n",
        "    priori = dic.copy()\n",
        "    return dic"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "nLf_vF6vZg2N",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#\n",
        "def mean_mat():\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    global mus\n",
        "    global n\n",
        "    dic = {}\n",
        "    for y,x in training_data:\n",
        "        if y not in dic:\n",
        "            dic[y] = []\n",
        "        dic[y].append(x)\n",
        "    for j in dic.keys():\n",
        "        dic[j] = np.mean(dic.get(j),axis=0)\n",
        "    ns0 = np.array(n*s0)\n",
        "    sigs = sigs.astype(float)\n",
        "    ns0 = ns0.astype(float)\n",
        "    for va in dic.keys():\n",
        "        mn0 = np.matmul(np.matmul(n*s0,np.linalg.inv(np.array(ns0+sigs))),dic.get(va))\n",
        "        mn1 = np.matmul(np.matmul(sigs,np.linalg.inv(np.array(ns0+sigs))),m0)\n",
        "        mn = np.array(mn0+mn1)\n",
        "        dic[va] = mn\n",
        "    mus = dic\n",
        "    return dic"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Ju2sDxyrZifo",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def mulvar_nor(X,y):\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    global mus\n",
        "    #print dets\n",
        "    #c = math.log(math.pow(2*math.pi,-0.5*len(X)))+math.log(math.pow(np.real(dets),-0.5))\n",
        "    #math.log(2*math.pi) = 1.8378770664093453\n",
        "    c = -0.5*len(X)*1.8378770664093453+ -0.5*abs(dets)\n",
        "    xut = np.array([X-mus.get(y)])\n",
        "    sig_i = np.linalg.inv(sigs)\n",
        "    pd = np.matmul(xut , sig_i)\n",
        "    xu = np.array([X-mus.get(y)]).T\n",
        "    e = (-0.5)*np.linalg.det(np.matmul(pd,xu))\n",
        "    return c+e\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Ur4dqqt4ZkiH",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def mul_exp(X,y):\n",
        "    return lam.get(y)*math.exp(-1*np.matmul(lam.get(y),X))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "60wVAZGSZmdQ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def train_rn():\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    global mus\n",
        "    X_train, y_train = load_mnist('drive/ml1/data/fashion', kind='train')\n",
        "    #X_train = np.array(pca(X_train))\n",
        "    #X_train = np.array(X_train).real.astype(float)\n",
        "    sigs = np.cov(X_train,rowvar=0)\n",
        "    dets = np.linalg.slogdet(sigs)[-1]\n",
        "    X_test, y_test = load_mnist('drive/ml1/data/fashion', kind='t10k')\n",
        "    #X_test = pca(X_test)\n",
        "    #X_test = np.array(X_test).real.astype(float)\n",
        "    training_data = [x for x in zip(y_train,X_train[:-len(X_train)/10])]\n",
        "    val_data = [x for x in zip(y_train,X_train[-len(X_train)/10:])]\n",
        "    m0 = np.mean(X_train[-len(X_train)/10:],axis=0)\n",
        "    s0 = np.cov(X_train[-len(X_train)/10:],rowvar=0)\n",
        "    n=len(X_train)-len(X_train)/10\n",
        "    sn = np.matmul(np.matmul(sigs,np.linalg.inv(np.array(n*s0+sigs))),s0)\n",
        "    sigs = np.array(sn+sigs)\n",
        "    test_data = [x for x in zip(y_test,X_test)]\n",
        "    unique, counts = np.unique(y_train, return_counts=True)\n",
        "    training_freq = dict(zip(unique,counts))\n",
        "    priori = cal_priori()\n",
        "    mus = mean_mat()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "dU4Y6zEWZoJX",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def bayes_c(X):\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    clas = \"\"\n",
        "    max = 0\n",
        "    dik={}\n",
        "    for i in training_freq.keys():\n",
        "        dik[i] = mulvar_nor(X,i)+math.log(priori.get(i))\n",
        "    pm = -999999\n",
        "    for v in dik.keys():\n",
        "        if dik.get(v)>pm:\n",
        "            pm = dik.get(v)\n",
        "            clas = v\n",
        "    #print str(te)+ \" \" + str(mulvar_nor(X,j)*priori.get(j))\n",
        "    return clas\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "3qoE7spFZp7B",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def test_rn():\n",
        "    fp = 0\n",
        "    cp = 0\n",
        "    fn = 0\n",
        "    cn = 0\n",
        "    p = 0\n",
        "    f = 0\n",
        "    t = 0\n",
        "    for v,u in test_data:\n",
        "        t+=1\n",
        "        k = str(bayes_c(u))\n",
        "        if k==str(v):\n",
        "            p+=1\n",
        "        else:\n",
        "            f+=1\n",
        "    print \"accuracy = \" + str(float(p)/float(t))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "b6ioASADZrxs",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def pca(X):\n",
        "    a = []\n",
        "    for i in X:\n",
        "        M = np.mean(i)\n",
        "        i = i-M\n",
        "        b = i.reshape(28,28)\n",
        "        c = np.cov(b)\n",
        "        values,vectors = np.linalg.eig(c)\n",
        "        eig_pair = [(np.abs(values[i]),vectors[:,i]) for i in range(len(values))]\n",
        "        eig_pair.sort(key=lambda x:x[0])\n",
        "        eig_pair.reverse()\n",
        "        ab = []\n",
        "        for j in range():\n",
        "            ab = np.concatenate((ab,eig_pair[j][1]),axis = None)\n",
        "        a.append(ab)\n",
        "    return a\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8gG_mK7mZtmM",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "train_rn()\n",
        "test_rn()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "lWkQ_8rRkdX-",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "![alt text](https://2.bp.blogspot.com/-dvmGCHt1IsE/W57oy9mGLOI/AAAAAAAAKPE/FfqahezKlzgq0DvW_5n1JcMKfEWd4tJWQCLcBGAs/s320/Capture.PNG)"
      ]
    },
    {
      "metadata": {
        "id": "ZA_wTQMxaX9E",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Part B\n",
        "Medical Data"
      ]
    },
    {
      "metadata": {
        "id": "pMh7YgL8bZ4W",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "mus = []\n",
        "training_freq = {}\n",
        "training_data = []\n",
        "test_data = []\n",
        "sigs = []\n",
        "dets = 1\n",
        "priori = []"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "up0lLROeaoHA",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def cal_priori():\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    global mus\n",
        "    dic = training_freq.copy()\n",
        "    for i in dic.keys():\n",
        "        dic[i] = float(dic.get(i))/float(len(training_data))\n",
        "    priori = dic.copy()\n",
        "    return dic"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Uq26GgIbasOP",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def mean_mat():\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    global mus\n",
        "    dic = {}\n",
        "    for y,x in training_data:\n",
        "        if y not in dic:\n",
        "            dic[y] = []\n",
        "        dic[y].append(x)\n",
        "    for j in dic.keys():\n",
        "        dic[j] = np.mean(dic.get(j),axis=0)\n",
        "    mus = dic\n",
        "    return dic\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "roFsl5isaxO9",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def mulvar_nor(X,y):\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    global mus\n",
        "    c = math.log(math.pow(2*math.pi,-0.5*len(X)))+math.log(math.pow(np.real(dets),-0.5))\n",
        "    xut = np.array([X-mus.get(y)])\n",
        "    sig_i = np.linalg.inv(sigs)\n",
        "    pd = np.matmul(xut , sig_i)\n",
        "    xu = np.array([X-mus.get(y)]).T\n",
        "    e = (-0.5)*np.linalg.det(np.matmul(pd,xu))\n",
        "    return c+e\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ULeYJnWza1K-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def train_rn():\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    global mus\n",
        "    priori = cal_priori()\n",
        "    mus = mean_mat()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "GFprrV88a3I4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def bayes_c(X):\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    global mus\n",
        "    clas = \"\"\n",
        "    max = 0\n",
        "    dik={}\n",
        "    for i in training_freq.keys():\n",
        "        dik[i] = mulvar_nor(X,i)+math.log(priori.get(i))\n",
        "    pm = -999999\n",
        "    for v in dik.keys():\n",
        "        if dik.get(v)>pm:\n",
        "            pm = dik.get(v)\n",
        "            clas = v\n",
        "    #print str(te)+ \" \" + str(mulvar_nor(X,j)*priori.get(j))\n",
        "    return clas\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ixaBjUGwa5IU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def test_rn():\n",
        "    fp = 0\n",
        "    cp = 0\n",
        "    fn = 0\n",
        "    cn = 0\n",
        "    p = 0\n",
        "    f = 0\n",
        "    t = 0\n",
        "    for v,u in test_data:\n",
        "        t+=1\n",
        "        k = str(bayes_c(u))\n",
        "        if k==str(v):\n",
        "            p+=1\n",
        "        else:\n",
        "            f+=1\n",
        "    print \"accuracy = \" + str(float(p)/float(t))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "M5n8BtZza7e2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "71189eef-d644-4c38-8e11-59d87fa7b779"
      },
      "cell_type": "code",
      "source": [
        "df = pd.read_csv(\"drive/ml1/Medical_data.csv\")\n",
        "y_train = np.array(df['Health'].values.tolist())\n",
        "X_train = df[['TEST1','TEST2','TEST3']].values.tolist()\n",
        "df = pd.read_csv(\"drive/ml1/test_medical.csv\")\n",
        "y_test = np.array(df['Health'].values.tolist())\n",
        "X_test = df[['TEST1','TEST2','TEST3']].values.tolist()\n",
        "sigs = np.cov(X_train,rowvar=0)\n",
        "dets = np.linalg.det(sigs)\n",
        "training_data = [x for x in zip(y_train,X_train)]\n",
        "test_data = [x for x in zip(y_test,X_test)]\n",
        "unique, counts = np.unique(y_train, return_counts=True)\n",
        "training_freq = dict(zip(unique,counts))\n",
        "train_rn()\n",
        "test_rn()"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "accuracy = 0.816666666667\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Lmk2oj-fbH43",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Part C\n",
        "Railway Data"
      ]
    },
    {
      "metadata": {
        "id": "nHUsQ7s6bIrr",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "mus = []\n",
        "training_freq = {}\n",
        "training_data = []\n",
        "test_data = []\n",
        "sigs = []\n",
        "dets = 1\n",
        "priori = []\n",
        "priori_d = {}\n",
        "recal = 0.5\n",
        "precisio = 0.5\n",
        "fscore = 0.5\n",
        "k = 1\n",
        "fp = 0\n",
        "cp = 0\n",
        "fn = 0\n",
        "cn = 0\n",
        "accu = 0.92"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "fEf2mu5ObxGP",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def cal_priori_d():\n",
        "    global dets\n",
        "    global k\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global priori\n",
        "    global sigs\n",
        "    global mus\n",
        "    global priori_d\n",
        "    dicd = {}\n",
        "    for y,x,x1 in training_data:\n",
        "        if str(x1) not in dicd.keys():\n",
        "            #print x1\n",
        "            dicd[str(x1)] = 0\n",
        "        if y==1:\n",
        "            dicd[str(x1)]+=1\n",
        "    for a in dicd.keys():\n",
        "        dicd[a] = dicd.get(a)/float(len(training_data))\n",
        "    priori_d = dicd.copy()\n",
        "    return dicd"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "R-O0JXQXb17a",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def cal_priori():\n",
        "    global dets\n",
        "    global k\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global priori\n",
        "    global sigs\n",
        "    global mus\n",
        "    global priori_d\n",
        "    dic = training_freq.copy()\n",
        "    for i in dic.keys():\n",
        "        dic[i] = float(dic.get(i))/float(len(training_data))\n",
        "    priori = dic.copy()\n",
        "    return dic\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "S1TBMnqbb4-U",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def mean_mat():\n",
        "    global dets\n",
        "    global k\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global priori\n",
        "    global sigs\n",
        "    global mus\n",
        "    global priori_d\n",
        "    dic = {}\n",
        "    for y,x,x1 in training_data:\n",
        "        if y not in dic:\n",
        "            dic[y] = []\n",
        "        dic[y].append(x)\n",
        "    for j in dic.keys():\n",
        "        dic[j] = np.mean(dic.get(j),axis=0)\n",
        "    mus = dic.copy()\n",
        "    return dic\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "OafUmW6hb9o7",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def mulvar_nor(X,y):\n",
        "    global training_freq\n",
        "    global k\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    global mus\n",
        "    c = math.log(math.pow(2*math.pi,-0.5*len(X)))+math.log(math.pow(np.real(dets),-0.5))\n",
        "    xut = np.array([X-mus.get(y)])\n",
        "    sig_i = np.linalg.inv(sigs)\n",
        "    pd = np.matmul(xut , sig_i)\n",
        "    xu = np.array([X-mus.get(y)]).T\n",
        "    e = (-0.5)*np.linalg.det(np.matmul(pd,xu))\n",
        "    return c+e"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "gTaPWzRmcBib",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def train_rn():\n",
        "    global dets\n",
        "    global k\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global priori\n",
        "    global sigs\n",
        "    global mus\n",
        "    global priori_d\n",
        "    df = pd.read_csv(\"drive/ml1/railwayBookingList.csv\")\n",
        "    y_train = np.array(df['boarded'].values.tolist())\n",
        "    X_train = df[['budget', 'memberCount', 'age']].values.tolist()\n",
        "    sigs = np.cov(X_train,rowvar=0)\n",
        "    dets = np.linalg.det(sigs)\n",
        "    X_train1 = df[['preferredClass', 'sex']].values.tolist()\n",
        "    training_data = [x for x in zip(y_train[:1100],X_train[:1100],X_train1[:1100])]\n",
        "    #print \"length of training_data: \"+str(len(training_data))\n",
        "    test_data = [x for x in zip(y_train[1100:],X_train[1100:],X_train1[1100:])]\n",
        "    #print \"length of test_data: \"+str(len(test_data))\n",
        "    unique, counts = np.unique(y_train, return_counts=True)\n",
        "    training_freq = dict(zip(unique,counts))\n",
        "    cal_priori()\n",
        "    cal_priori_d()\n",
        "    mean_mat()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "N2aJmd8tcuss",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def bayes_c(X,X1):\n",
        "    global dets\n",
        "    global k\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global priori\n",
        "    global sigs\n",
        "    global mus\n",
        "    global priori_d\n",
        "    clas = 1\n",
        "    max = 0\n",
        "    dik ={}\n",
        "    if X1 not in priori_d:\n",
        "        return 1\n",
        "    for i in training_freq.keys():\n",
        "        if i==1:\n",
        "            post_d = priori_d.get(X1)\n",
        "        else:\n",
        "            post_d = 1-priori_d.get(X1)\n",
        "        dik[i] = mulvar_nor(X,i)*priori.get(i)*post_d\n",
        "    #print str(dik.get(0))+\" \"+str(k*dik.get(1))\n",
        "    if dik.get(0)>k*dik.get(1):\n",
        "        return 0\n",
        "    return 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "7OvF8MIScwMs",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def test_rn():\n",
        "    global fscore\n",
        "    global precisio\n",
        "    global recal\n",
        "    global k\n",
        "    global fp\n",
        "    global cp\n",
        "    global fn\n",
        "    global cn\n",
        "    global accu\n",
        "    accu = 0.85\n",
        "    p = 0\n",
        "    f = 0\n",
        "    t = 0\n",
        "    for v,u,c in test_data:\n",
        "        t+=1\n",
        "        st = str(v)\n",
        "        te = str(bayes_c(u,str(c)))\n",
        "        if (st==\"0\" and te==\"0\"):\n",
        "            cn+=1\n",
        "        if (st==\"0\" and te==\"1\"):\n",
        "            fp+=1\n",
        "        if (st==\"1\" and te==\"0\"):\n",
        "            fn+=1\n",
        "        if (st==\"1\" and te==\"1\"):\n",
        "            cp+=1\n",
        "        if te==st:\n",
        "            p+=1\n",
        "        else:\n",
        "            f+=1\n",
        "    recal = cp/(float(cp)+ float(fn))\n",
        "    precisio = cp/(float(cp)+ float(fp))\n",
        "    fscore = 2*precisio*recal/(precisio + recal)\n",
        "    accu = str(float(p)/float(t))\n",
        "    print \"Accuracy: \" + str(accu)\n",
        "    print \"Precision: \" + str(precisio)\n",
        "    print \"recall: \" + str(recal)\n",
        "    print \"F1 score: \" + str(fscore)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ulbhA6BTc0bw",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def roc():\n",
        "    global k\n",
        "    global fp\n",
        "    global cp\n",
        "    global fn\n",
        "    global cn\n",
        "    ro = [(0,0)]\n",
        "    for i in range(100):\n",
        "        k = -2.5 + i/float(10)\n",
        "        #train_rn()\n",
        "        test_rn()\n",
        "        le = len(test_data)\n",
        "        ro.append((fp/float(le),cp/float(le)))\n",
        "        fp = 0\n",
        "        cp = 0\n",
        "        fn = 0\n",
        "        cn = 0\n",
        "    #print ro\n",
        "    plt.scatter(*zip(*ro))\n",
        "    plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "G_5qRfOr455c",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "![alt text](https://2.bp.blogspot.com/-XzCZ_uO9UHY/W57x3Qpe9tI/AAAAAAAAKPY/HT0ULZWjyd8oSfpocvIJz6H_AufDolXfgCLcBGAs/s320/Froc.png)"
      ]
    },
    {
      "metadata": {
        "id": "lrFU58ikc6lu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        },
        "outputId": "ab4df8e1-4b67-422a-87d0-b3a6518448f3"
      },
      "cell_type": "code",
      "source": [
        "train_rn()\n",
        "test_rn()\n",
        "#roc()"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy: 0.828571428571\n",
            "Precision: 0.828571428571\n",
            "recall: 1.0\n",
            "F1 score: 0.90625\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "GOhTMGsphEQX",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#Section 3\n",
        "Bayesian Classifier with Bayesian Estimation"
      ]
    },
    {
      "metadata": {
        "id": "doe70KWfhiXr",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#Part A\n",
        "Fashion Mnist"
      ]
    },
    {
      "metadata": {
        "id": "3sP4-dBRhDKE",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "mus = []\n",
        "training_freq = {}\n",
        "training_data = []\n",
        "test_data = []\n",
        "sigs = []\n",
        "dets = 1\n",
        "priori = []\n",
        "m0 = 1\n",
        "s0 = 1\n",
        "n = 1\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "svjQqMNqiLxS",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def cal_priori():\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    global mus\n",
        "    dic = training_freq.copy()\n",
        "    for i in dic.keys():\n",
        "        dic[i] = float(dic.get(i))/float(len(training_data))\n",
        "    priori = dic.copy()\n",
        "    return dic"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "evXd8ZjriO_f",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def mean_mat():\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    global mus\n",
        "    global n\n",
        "    dic = {}\n",
        "    for y,x in training_data:\n",
        "        if y not in dic:\n",
        "            dic[y] = []\n",
        "        dic[y].append(x)\n",
        "    for j in dic.keys():\n",
        "        dic[j] = np.mean(dic.get(j),axis=0)\n",
        "    ns0 = np.array(n*s0)\n",
        "    sigs = sigs.astype(float)\n",
        "    ns0 = ns0.astype(float)\n",
        "    for va in dic.keys():\n",
        "        mn0 = np.matmul(np.matmul(n*s0,np.linalg.inv(np.array(ns0+sigs))),dic.get(va))\n",
        "        mn1 = np.matmul(np.matmul(sigs,np.linalg.inv(np.array(ns0+sigs))),m0)\n",
        "        mn = np.array(mn0+mn1)\n",
        "        dic[va] = mn\n",
        "    mus = dic\n",
        "    return dic\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "SrnVsv1fiR2p",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def mulvar_nor(X,y):\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    global mus\n",
        "    #print dets\n",
        "    #c = math.log(math.pow(2*math.pi,-0.5*len(X)))+math.log(math.pow(np.real(dets),-0.5))\n",
        "    #math.log(2*math.pi) = 1.8378770664093453\n",
        "    c = -0.5*len(X)*1.8378770664093453+ -0.5*abs(dets)\n",
        "    xut = np.array([X-mus.get(y)])\n",
        "    sig_i = np.linalg.inv(sigs)\n",
        "    pd = np.matmul(xut , sig_i)\n",
        "    xu = np.array([X-mus.get(y)]).T\n",
        "    e = (-0.5)*np.linalg.det(np.matmul(pd,xu))\n",
        "    return c+e\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jSsIgIEfiWdG",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def mul_exp(X,y):\n",
        "    return lam.get(y)*math.exp(-1*np.matmul(lam.get(y),X))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "EHdMitwxiXbc",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def train_rn():\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    global mus\n",
        "    X_train, y_train = load_mnist('drive/ml1/data/fashion', kind='train')\n",
        "    #X_train = np.array(pca(X_train))\n",
        "    #X_train = np.array(X_train).real.astype(float)\n",
        "    sigs = np.cov(X_train,rowvar=0)\n",
        "    dets = np.linalg.slogdet(sigs)[-1]\n",
        "    X_test, y_test = load_mnist('drive/ml1/data/fashion', kind='t10k')\n",
        "    #X_test = pca(X_test)\n",
        "    #X_test = np.array(X_test).real.astype(float)\n",
        "    training_data = [x for x in zip(y_train,X_train[:-len(X_train)/10])]\n",
        "    val_data = [x for x in zip(y_train,X_train[-len(X_train)/10:])]\n",
        "    m0 = np.mean(X_train[-len(X_train)/10:],axis=0)\n",
        "    s0 = np.cov(X_train[-len(X_train)/10:],rowvar=0)\n",
        "    n=len(X_train)-len(X_train)/10\n",
        "    sn = np.matmul(np.matmul(sigs,np.linalg.inv(np.array(n*s0+sigs))),s0)\n",
        "    sigs = np.array(sn+sigs)\n",
        "    test_data = [x for x in zip(y_test,X_test)]\n",
        "    unique, counts = np.unique(y_train, return_counts=True)\n",
        "    training_freq = dict(zip(unique,counts))\n",
        "    priori = cal_priori()\n",
        "    mus = mean_mat()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "bsHLovtNiaaE",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def bayes_c(X):\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    clas = \"\"\n",
        "    max = 0\n",
        "    dik={}\n",
        "    for i in training_freq.keys():\n",
        "        dik[i] = mulvar_nor(X,i)+math.log(priori.get(i))\n",
        "    pm = -999999\n",
        "    for v in dik.keys():\n",
        "        if dik.get(v)>pm:\n",
        "            pm = dik.get(v)\n",
        "            clas = v\n",
        "    #print str(te)+ \" \" + str(mulvar_nor(X,j)*priori.get(j))\n",
        "    return clas\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "37TrX60oidy-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def test_rn():\n",
        "    fp = 0\n",
        "    cp = 0\n",
        "    fn = 0\n",
        "    cn = 0\n",
        "    p = 0\n",
        "    f = 0\n",
        "    t = 0\n",
        "    for v,u in test_data:\n",
        "        t+=1\n",
        "        k = str(bayes_c(u))\n",
        "        if k==str(v):\n",
        "            p+=1\n",
        "        else:\n",
        "            f+=1\n",
        "    print \"accuracy = \" + str(float(p)/float(t))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "gIzfr_DPijKh",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def pca(X):\n",
        "    a = []\n",
        "    for i in X:\n",
        "        M = np.mean(i)\n",
        "        i = i-M\n",
        "        b = i.reshape(28,28)\n",
        "        c = np.cov(b)\n",
        "        values,vectors = np.linalg.eig(c)\n",
        "        eig_pair = [(np.abs(values[i]),vectors[:,i]) for i in range(len(values))]\n",
        "        eig_pair.sort(key=lambda x:x[0])\n",
        "        eig_pair.reverse()\n",
        "        ab = []\n",
        "        for j in range():\n",
        "            ab = np.concatenate((ab,eig_pair[j][1]),axis = None)\n",
        "        a.append(ab)\n",
        "    return a"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Lipzvntiilxl",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "train_rn()\n",
        "test_rn()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "6xsMpIqDnQvq",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "![alt text](https://2.bp.blogspot.com/-aDBDQgbZ5k8/W57o1JoQ9yI/AAAAAAAAKPI/8rW8eReCTDca9AnH--smS7d8QCm1fu_DQCLcBGAs/s320/Capture2.PNG)"
      ]
    },
    {
      "metadata": {
        "id": "I5OFxYKxiq13",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#Part B\n",
        "Medical Data"
      ]
    },
    {
      "metadata": {
        "id": "Ut2-o4oxiuV6",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "mus = []\n",
        "training_freq = {}\n",
        "training_data = []\n",
        "test_data = []\n",
        "sigs = []\n",
        "dets = 1\n",
        "priori = []\n",
        "m0 = 1\n",
        "s0 = 1\n",
        "n = 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "buVtfgZWi0EQ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def cal_priori():\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    global mus\n",
        "    dic = training_freq.copy()\n",
        "    for i in dic.keys():\n",
        "        dic[i] = float(dic.get(i))/float(len(training_data))\n",
        "    priori = dic.copy()\n",
        "    return dic\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "eWAvVYCWi4Wf",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def mean_mat():\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    global mus\n",
        "    global n\n",
        "    dic = {}\n",
        "    for y,x in training_data:\n",
        "        if y not in dic:\n",
        "            dic[y] = []\n",
        "        dic[y].append(x)\n",
        "    for j in dic.keys():\n",
        "        dic[j] = np.mean(dic.get(j),axis=0)\n",
        "    ns0 = np.array(n*s0)\n",
        "    sigs = sigs.astype(float)\n",
        "    ns0 = ns0.astype(float)\n",
        "    for va in dic.keys():\n",
        "        mn0 = np.matmul(np.matmul(n*s0,np.linalg.inv(np.array(ns0+sigs))),dic.get(va))\n",
        "        mn1 = np.matmul(np.matmul(sigs,np.linalg.inv(np.array(ns0+sigs))),m0)\n",
        "        mn = np.array(mn0+mn1)\n",
        "        dic[va] = mn\n",
        "    mus = dic\n",
        "    return dic\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ooxWfeAhi5h7",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def mulvar_nor(X,y):\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    global mus\n",
        "    #print dets\n",
        "    #c = math.log(math.pow(2*math.pi,-0.5*len(X)))+math.log(math.pow(np.real(dets),-0.5))\n",
        "    #math.log(2*math.pi) = 1.8378770664093453\n",
        "    c = -0.5*len(X)*1.8378770664093453+ -0.5*abs(dets)\n",
        "    xut = np.array([X-mus.get(y)])\n",
        "    sig_i = np.linalg.inv(sigs)\n",
        "    pd = np.matmul(xut , sig_i)\n",
        "    xu = np.array([X-mus.get(y)]).T\n",
        "    e = (-0.5)*np.linalg.det(np.matmul(pd,xu))\n",
        "    return c+e\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "CskyMfEpi8eN",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def mul_exp(X,y):\n",
        "    return lam.get(y)*math.exp(-1*np.matmul(lam.get(y),X))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "dFoI7U9LjCjR",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def train_rn():\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    global mus\n",
        "    priori = cal_priori()\n",
        "    mus = mean_mat()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ZTyUA7LmjEeb",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def bayes_c(X):\n",
        "    global training_freq\n",
        "    global training_data\n",
        "    global test_data\n",
        "    global m0,s0\n",
        "    global sigs\n",
        "    global dets\n",
        "    global priori\n",
        "    clas = \"\"\n",
        "    max = 0\n",
        "    dik={}\n",
        "    for i in training_freq.keys():\n",
        "        dik[i] = mulvar_nor(X,i)+math.log(priori.get(i))\n",
        "    pm = -999999\n",
        "    for v in dik.keys():\n",
        "        if dik.get(v)>pm:\n",
        "            pm = dik.get(v)\n",
        "            clas = v\n",
        "    #print str(te)+ \" \" + str(mulvar_nor(X,j)*priori.get(j))\n",
        "    return clas\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "BPYAWDLZjIOg",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def test_rn():\n",
        "    fp = 0\n",
        "    cp = 0\n",
        "    fn = 0\n",
        "    cn = 0\n",
        "    p = 0\n",
        "    f = 0\n",
        "    t = 0\n",
        "    for v,u in test_data:\n",
        "        t+=1\n",
        "        k = str(bayes_c(u))\n",
        "        if k==str(v):\n",
        "            p+=1\n",
        "        else:\n",
        "            f+=1\n",
        "    print \"accuracy = \" + str(float(p)/float(t))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Z9TxdkU0jK8W",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def pca(X):\n",
        "    a = []\n",
        "    for i in X:\n",
        "        M = np.mean(i)\n",
        "        i = i-M\n",
        "        b = i.reshape(28,28)\n",
        "        c = np.cov(b)\n",
        "        values,vectors = np.linalg.eig(c)\n",
        "        eig_pair = [(np.abs(values[i]),vectors[:,i]) for i in range(len(values))]\n",
        "        eig_pair.sort(key=lambda x:x[0])\n",
        "        eig_pair.reverse()\n",
        "        ab = []\n",
        "        for j in range():\n",
        "            ab = np.concatenate((ab,eig_pair[j][1]),axis = None)\n",
        "        a.append(ab)\n",
        "    return a\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "pr2RagHBjNnK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "6ced8432-4c4c-4c62-a13a-77bb4975c4e3"
      },
      "cell_type": "code",
      "source": [
        "df = pd.read_csv(\"drive/ml1/Medical_data.csv\")\n",
        "y_train = np.array(df['Health'].values.tolist())\n",
        "X_train = df[['TEST1','TEST2','TEST3']].values.tolist()\n",
        "df = pd.read_csv(\"drive/ml1/test_medical.csv\")\n",
        "y_test = np.array(df['Health'].values.tolist())\n",
        "X_test = df[['TEST1','TEST2','TEST3']].values.tolist()\n",
        "sigs = np.cov(X_train,rowvar=0)\n",
        "dets = np.linalg.slogdet(sigs)[-1]\n",
        "test_data = [x for x in zip(y_test,X_test)]\n",
        "training_data = [x for x in zip(y_train,X_train[:-len(X_train)/10])]\n",
        "val_data = [x for x in zip(y_train,X_train[-len(X_train)/10:])]\n",
        "m0 = np.mean(X_train[-len(X_train)/10:],axis=0)\n",
        "s0 = np.cov(X_train[-len(X_train)/10:],rowvar=0)\n",
        "n=len(X_train)-len(X_train)/10\n",
        "sn = np.matmul(np.matmul(sigs,np.linalg.inv(np.array(n*s0+sigs))),s0)\n",
        "sigs = np.array(sn+sigs)\n",
        "unique, counts = np.unique(y_train, return_counts=True)\n",
        "training_freq = dict(zip(unique,counts))\n",
        "train_rn()\n",
        "test_rn()"
      ],
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "accuracy = 0.817\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "QJvWGsF1n_l-",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Section 3\n",
        "Naive Bayes Classifier"
      ]
    },
    {
      "metadata": {
        "id": "GfX7Eln8u9Bm",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#Part A\n",
        "Medical Data"
      ]
    },
    {
      "metadata": {
        "id": "fRAdUbY6oXfn",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def loadCsv(filename):\n",
        "\tlines = csv.reader(open(filename, \"rb\"))\n",
        "\tdataset = list(lines)\n",
        "\tleng = len(dataset)\n",
        "\tdataset = dataset[1:leng]\n",
        "\treturn dataset\n",
        "def mean(arr):\n",
        "\tarr = arr.astype(np.float)\n",
        "\treturn np.sum(arr)/arr.size\n",
        "\n",
        "def standev(arr):\n",
        "\tarr = arr.astype(np.float)\n",
        "\tmea = mean(arr);\n",
        "\tsquaresum = sum(np.power(arr-mea,2))/float(arr.size)\n",
        "\treturn math.sqrt(squaresum) \n",
        "def trainprob(arr):\n",
        "\tarr = np.array(arr)\n",
        "\tunique, counts = np.unique(arr, return_counts=True)\n",
        "\tcounts = np.float_(counts)/float(arr.size)\n",
        "\treturn (unique,counts)\n",
        "def probdata(x,mean,stdev):\n",
        "\texponent = math.exp(-(math.pow(x-mean,2)/(2*math.pow(stdev,2))))\n",
        "\treturn (1 / (math.sqrt(2*math.pi) * stdev)) * exponent\n",
        "def dsepbyclass(dataset):\n",
        "\tdicte = {}\n",
        "\tfor i in dataset:\n",
        "\t\tif(i[0] not in dicte):\n",
        "\t\t\tdicte[i[0]] = []\n",
        "\t\tdicte[i[0]].append(i[1:4])\n",
        "\treturn dicte\n",
        "def meanvar(dataset):\n",
        "\tdicte = {}\n",
        "\tfor i in dataset:\n",
        "\t\ta = zip(*dataset[i])\n",
        "\t\tdicte[i] = []\n",
        "\t\tfor j in a:\n",
        "\t\t\tp = np.array(j)\n",
        "\t\t\tdicte[i].append([mean(p),standev(p)])\n",
        "\treturn dicte"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "rnaHnNSAtKpW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "629acab4-b38f-43eb-e939-60285210fdc2"
      },
      "cell_type": "code",
      "source": [
        "def predictclass(mdict,data,probmatrix):\n",
        "\tdicter = predictprob(mdict,data)\n",
        "\tbestprob,bestclass = -1,None\n",
        "\tt =0\n",
        "\tfor clas,prob in dicter.iteritems():\n",
        "\t\tif bestprob<prob*probmatrix[t] :\n",
        "\t\t\tbestprob = prob*probmatrix[t]\n",
        "\t\t\tbestclass = clas\n",
        "\t\tt+=1\n",
        "\n",
        "\treturn bestclass\n",
        "\n",
        "\n",
        "   \n",
        "def predictprob(mdict,data):\n",
        "\tdicter = {}\n",
        "\t\n",
        "\tfor classvalue,classdata in mdict.iteritems():\n",
        "\t\tprob = 1\n",
        "\t\tt=1\n",
        "\t\tfor j in classdata:\n",
        "\t\t\tprob *= probdata(float(data[t]),j[0],j[1])\n",
        "\t\t\tt +=1\n",
        "\t\tdicter[classvalue] = prob\n",
        "\treturn dicter\n",
        "\n",
        "\n",
        "            \n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\t#return [(mean(x),standev(x)) for x in a]\n",
        "def answerdata(file1,file2):\n",
        "\tdataset = loadCsv(file1)\n",
        "\tdic = dsepbyclass(dataset)\n",
        "\tmdict = meanvar(dic)\n",
        "\tdataset1 = loadCsv(file2)\n",
        "\tdataset1 = dataset1[400:700]\n",
        "\tunique ,counts = trainprob(dataset[1:500])\n",
        "\taccuracy =0\n",
        "\tfor j in dataset1:\n",
        "\t\ta =  predictclass(mdict,j,counts)\n",
        "\t\tif(a==j[0]):\n",
        "\t\t\taccuracy += 1\n",
        "\tprint (float(accuracy)/float(len(dataset1)))*100\n",
        "\n",
        "\n",
        "\n",
        "answerdata(\"drive/ml1/Medical_data.csv\",\"drive/ml1/test_medical.csv\")"
      ],
      "execution_count": 86,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "89.6666666667\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "v-pNstvmu4iP",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Part B\n",
        "Fashion Mnist Data"
      ]
    },
    {
      "metadata": {
        "id": "o0jhg6iDtXub",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def loadCsv(filename):\n",
        "\tlines = csv.reader(open(filename, \"rb\"))\n",
        "\tdataset = list(lines)\n",
        "\tleng = len(dataset)\n",
        "\tdataset = dataset[1:leng]\n",
        "\treturn dataset\n",
        "def mean(arr):\n",
        "\tarr = arr.astype(np.float)\n",
        "\n",
        "\treturn np.sum(arr)/arr.size\n",
        "\n",
        "def standev(arr):\n",
        "\tarr = arr.astype(np.float)\n",
        "\tmea = mean(arr)\n",
        "\tsquaresum = sum(np.power(arr-mea,2))/float(arr.size)\n",
        "\treturn math.sqrt(squaresum)\n",
        "def trainprob(dicte,X_train):\n",
        "\t#unique, counts = np.unique(arr, return_counts=True)\n",
        "\t#counts = np.float_(counts)/float(arr.size)\n",
        "\t#return (unique,counts)\n",
        "\ta ={}\n",
        "\tlent = len(X_train)\n",
        "\tfor j in dicte:\n",
        "\t\ta[j] =   float(len(dicte[j]))/float(lent)\n",
        "\treturn a\n",
        "def probdata(x,mean,stdev):\n",
        "\t#print mean\n",
        "\t#print stdev\n",
        "\tif stdev==0:\n",
        "\t\treturn 1\n",
        "\texponent = math.exp(-(math.pow(x-mean,2)/(2*math.pow(stdev,2))))\n",
        "\treturn (1 / (math.sqrt(2*math.pi) * stdev)) * exponent\n",
        "def dsepbyclass(X_train,y_train):\n",
        "\tdicte = {}\n",
        "\tfor i in range(len(X_train)):\n",
        "\t\tif(y_train[i] not in dicte):\n",
        "\t\t\tdicte[y_train[i]] = []\n",
        "\t\tdicte[y_train[i]].append(X_train[i])\n",
        "\treturn dicte\n",
        "def meanvar(dataset):\n",
        "\tdicte = {}\n",
        "\tfor i in dataset:\n",
        "\t\ta = zip(*dataset[i])\n",
        "\t\t#print len(a)\n",
        "\t\tdicte[i] = []\n",
        "\t\tfor j in a:\n",
        "\t\t\tp = np.array(j)\n",
        "\t\t\tdicte[i].append([mean(p),standev(p)])\n",
        "\treturn dicte\n",
        "def predictclass(mdict,data,probmatrix):\n",
        "\tdicter = predictprob(mdict,data)\n",
        "\tbestprob,bestclass = -1,None\n",
        "\tt =0\n",
        "\tfor clas,prob in dicter.iteritems():\n",
        "\t\tif bestprob<prob*probmatrix[clas] :\n",
        "\t\t\tbestprob = prob*probmatrix[clas]\n",
        "\t\t\tbestclass = clas\n",
        "\t\tt+=1\n",
        "\n",
        "\treturn bestclass\n",
        "\n",
        "\n",
        "\n",
        "def predictprob(mdict,data):\n",
        "\tdicter = {}\n",
        "\n",
        "\tfor classvalue,classdata in mdict.iteritems():\n",
        "\t\tprob = 1\n",
        "\t\tt=0\n",
        "\t\tfor j in classdata:\n",
        "\t\t\t#print j\n",
        "\t\t\tprob *= probdata(float(data[t]),j[0],j[1])\n",
        "\t\t\tt +=1\n",
        "\t\tdicter[classvalue] = prob\n",
        "\treturn dicter\n",
        "\n",
        "\t#return [(mean(x),standev(x)) for x in a]\n",
        "def answerdata():\n",
        "\tX_train, y_train = load_mnist(\"drive/ml1/data/fashion\", kind='train')\n",
        "\tX_test, y_test = load_mnist(\"drive/ml1/data/fashion\", kind='t10k')\n",
        "\t#pcareduction([X_train[0]])\n",
        "\t#return\n",
        "\tX_test = pcareduction(X_test)\n",
        "\tX_train = pcareduction(X_train)\n",
        "\t#dataset = loadCsv(file1)\n",
        "\tdic = dsepbyclass(X_train,y_train)\n",
        "\tmdict = meanvar(dic)\n",
        "\tcounts = trainprob(dic,X_train)\n",
        "\taccuracy =0\n",
        "\tfor j in range(len(X_test)):\n",
        "\t\ta =  predictclass(mdict,X_test[j],counts)\n",
        "\t\tif(a==y_test[j]):\n",
        "\t\t\taccuracy += 1\n",
        "\t\t\t#print \"correct\"\n",
        "\t\t#else:\n",
        "\t\t\t#print \"incorrect\"\n",
        "\tprint \"Accuracy: \" + str((float(accuracy)/float(len(X_test)))*100)\n",
        "def pcareduction(X_train):\n",
        "\ta = []\n",
        "\tfor i in X_train:\n",
        "\t    M = np.mean(i)\n",
        "\t    i = i-M\n",
        "\t    b = i.reshape(28,28)\n",
        "\t    #m = np.mean(b.T,axis =1)\n",
        "\t    c = np.cov(b)\n",
        "\t    values,vectors = np.linalg.eig(c)\n",
        "\t    eig_pair = [(np.abs(values[i]),vectors[:,i]) for i in range(len(values))]\n",
        "\t    eig_pair.sort(key=lambda x:x[0])\n",
        "\t    eig_pair.reverse()\n",
        "\t    #print eig_pair\n",
        "\t    ab = []\n",
        "\t    #print eig_pair[0][0]\n",
        "\t    for j in range(6):\n",
        "\t    \tab = np.concatenate((ab,eig_pair[j][1]),axis = None)\n",
        "\t    #print ab\n",
        "\t    a.append(ab)\n",
        "\treturn a\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "def pca1(X_train):\n",
        "\ta = []\n",
        "\tb = X_train.T\n",
        "\tM = np.mean(b,axis=1)\n",
        "\tprint M.shape\n",
        "\tac = X_train- M\n",
        "\tc = np.cov(ac.T)\n",
        "\tprint c.shape\n",
        "\tvalues,vectors = np.linalg.eig(c)\n",
        "\teig_pair = [(np.abs(values[i]),vectors[:,i]) for i in range(len(values))]\n",
        "\teig_pair.sort(key=lambda x:x[0])\n",
        "\teig_pair.reverse()\n",
        "\tfor i,j in eig_pair:\n",
        "\t\tprint i\n",
        "\tab =[]\n",
        "\tfor j in range(6):\n",
        "\t\tab.append(eig_pair[j][1])\n",
        "\tab = np.array(ab)\n",
        "\tprint ab.shape\n",
        "\treturn\n",
        "\tp = ab.dot(ac.T)\n",
        "\tprint p.shape\n",
        "\treturn p.T\n",
        "def a(x):\n",
        "\tx.sort(key=lambda x:x[0])\n",
        "\t"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "uGzqvXRIvQXA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        },
        "outputId": "2d1a471f-d844-4caa-bcba-38c899966c33"
      },
      "cell_type": "code",
      "source": [
        "answerdata()"
      ],
      "execution_count": 95,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:8: ComplexWarning: Casting complex values to real discards the imaginary part\n",
            "  \n",
            "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:13: ComplexWarning: Casting complex values to real discards the imaginary part\n",
            "  del sys.path[0]\n",
            "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:72: ComplexWarning: Casting complex values to real discards the imaginary part\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Accuracy: 52.48\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "GWVlBSMxwvJt",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#Part C\n",
        "Railway Data"
      ]
    },
    {
      "metadata": {
        "id": "n5mKGc7awyy2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        },
        "outputId": "5efa0a0f-2b94-4f51-fbdb-c84e41de8db0"
      },
      "cell_type": "code",
      "source": [
        "def loadCsv(filename):\n",
        "\tlines = csv.reader(open(filename, \"rb\"))\n",
        "\tdataset = list(lines)\n",
        "\tleng = len(dataset)\n",
        "\tdataset = np.array(dataset[1:leng])\n",
        "\tdataset1 = dataset[:,[1,2,3,6]]\n",
        "\tdataset2 = dataset[:,[1,4,5]]\n",
        "\treturn dataset1,dataset2\n",
        "def mean(arr):\n",
        "\tarr = arr.astype(np.float)\n",
        "\treturn np.sum(arr)/arr.size\n",
        "\n",
        "def standev(arr):\n",
        "\tarr = arr.astype(np.float)\n",
        "\tmea = mean(arr);\n",
        "\tsquaresum = sum(np.power(arr-mea,2))/float(arr.size)\n",
        "\treturn math.sqrt(squaresum) \n",
        "def trainprob(arr):\n",
        "\tarr = np.array(arr)\n",
        "\tunique, counts = np.unique(arr, return_counts=True)\n",
        "\tcounts = np.float_(counts)/float(arr.size)\n",
        "\treturn (unique,counts)\n",
        "def probdata(x,mean,stdev):\n",
        "\texponent = math.exp(-(math.pow(x-mean,2)/(2*math.pow(stdev,2))))\n",
        "\treturn (1 / (math.sqrt(2*math.pi) * stdev)) * exponent\n",
        "def dsepbyclass(dataset):\n",
        "\tdicte = {}\n",
        "\tfor i in dataset:\n",
        "\t\tif(i[0] not in dicte):\n",
        "\t\t\tdicte[i[0]] = []\n",
        "\t\tdicte[i[0]].append(i[1:4])\n",
        "\treturn dicte\n",
        "def meanvar(dataset):\n",
        "\tdicte = {}\n",
        "\tfor i in dataset:\n",
        "\t\ta = zip(*dataset[i])\n",
        "\t\tdicte[i] = []\n",
        "\t\tfor j in a:\n",
        "\t\t\tp = np.array(j)\n",
        "\t\t\tdicte[i].append([mean(p),standev(p)])\n",
        "\treturn dicte\n",
        "def newprob(dicter):\n",
        "\tdic ={}\n",
        "\tfor j in dicter:\n",
        "\t\ta = zip(*dicter[j])\n",
        "\t\tunique1,counts1 = np.unique(a[0], return_counts=True)\n",
        "\t\tunique2,counts2 = np.unique(a[1], return_counts=True) \n",
        "\t\tb = {}\n",
        "\t\tc =0\n",
        "\t\td= 0 \n",
        "\t\tfor k in counts1:\n",
        "\t\t\tc +=k\n",
        "\t\tfor l in counts2:\n",
        "\t\t\td +=l\n",
        "\t\tfor qw in range(len(unique1)):\n",
        "\t\t\tb[unique1[qw]] = float(counts1[qw])/float(c)\n",
        "\t\tfor qa in range(len(unique2)):\n",
        "\t\t\tb[unique2[qa]] = float(counts2[qa])/float(c)\n",
        "\t\tdic[j] = b\n",
        "\tif len(dic['0']) != len(dic['1']):\n",
        "\t\tfor j in dic['0']:\n",
        "\t\t\tif not j in dic['1']:\n",
        "\t\t\t\tdic['1'][j] =0\n",
        "\t\tfor j in dic['1']:\n",
        "\t\t\tif not j in dic['0']:\n",
        "\t\t\t\tdic['0'][j] =0\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "    \n",
        "\treturn dic\n",
        "def predictclass(mdict,data,data1,probmatrix,ndict):\n",
        "\tdicter = predictprob(mdict,data,probmatrix)\n",
        "\tbestprob,bestclass = -1,None\n",
        "\t#print dicter\n",
        "\tfor clas,prob in dicter.iteritems():\n",
        "\t\t\n",
        "\t\tab = ndict[clas]\n",
        "\t\tif data1[1] not in ab:\n",
        "\t\t\tp=0\n",
        "\t\telif data1[2] not in ab:\n",
        "\t\t\tp = 0\n",
        "\t\telse:\n",
        "\t\t\tp = prob*ab[data1[1]]*ab[data1[2]]\n",
        "\t\t\n",
        "\t\tif bestprob< p:\n",
        "\t\t\tbestprob = p\n",
        "\t\t\tbestclass = clas\n",
        "\n",
        "\treturn bestclass\n",
        "\n",
        "\n",
        "   \n",
        "def predictprob(mdict,data,cd):\n",
        "\tdicter = {}\n",
        "\tre = 0\n",
        "\tfor classvalue,classdata in mdict.iteritems():\n",
        "\t\tprob = 1\n",
        "\t\tt=1\n",
        "\t\tfor j in classdata:\n",
        "\t\t\tprob *= probdata(float(data[t]),j[0],j[1])\n",
        "\t\t\tt +=1\n",
        "\t\tdicter[classvalue] = prob*cd[re]\n",
        "\t\tre += 1\n",
        "\treturn dicter\n",
        "\n",
        "\t#return [(mean(x),standev(x)) for x in a]\n",
        "def answerdata(file1,file2):\n",
        "\tdataset1,dataset2 = loadCsv(file1)\n",
        "\tdic = dsepbyclass(dataset1)\n",
        "\tdic1 = dsepbyclass(dataset2)\n",
        "\tmdict = meanvar(dic)\n",
        "\tndict = newprob(dic1)\n",
        "\tdataset3 ,dataset4 = loadCsv(file2)\n",
        "\t#print trainprob\n",
        "\tunique ,counts = trainprob(dataset1[:,1])\n",
        "\taccuracy =0\n",
        "\ttp = 0\n",
        "\tfp =0\n",
        "\tfn =0\n",
        "\tfor s in range(len(dataset3)):\n",
        "\t\tj = dataset3[s]\n",
        "\t\tk = dataset4[s]\n",
        "\t\ta =  predictclass(mdict,j,k,counts,ndict)\n",
        "\t\tif(a==j[0]):\n",
        "\t\t\taccuracy += 1\n",
        "\t\t\tif(a ==\"1\"):\n",
        "\t\t\t\ttp +=1\n",
        "\t\telse:\n",
        "\t\t\tif(a==\"1\"):\n",
        "\t\t\t\tfp +=1\n",
        "\t\t\tif(a==\"0\"):\n",
        "\t\t\t\tfn += 1\n",
        "\tprecision = float(tp)/float(tp+fp)\n",
        "\trecall = float(tp)/float(tp+fn)\n",
        "\tprint \"accuaracy = \" + str((float(accuracy)/float(len(dataset1)))*100)\n",
        "\tprint \"precision = \" + str(precision)\n",
        "\tprint \"recall = \" + str(recall)\n",
        "\tprint \"F1 score = \" +str(float(2*precision*recall)/float(precision+recall))\n",
        "\n",
        "\n",
        "\n",
        "answerdata(\"drive/ml1/railwayBookingList.csv\",\"drive/ml1/railwayBookingList.csv\")"
      ],
      "execution_count": 96,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "accuaracy = 76.0305343511\n",
            "precision = 0.745059288538\n",
            "recall = 0.930864197531\n",
            "F1 score = 0.827661909989\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "OW_64bLDxLWf",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#Section 4\n",
        "K Nearest Neighbor"
      ]
    },
    {
      "metadata": {
        "id": "Nj1CUy19xsSO",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#Part A\n",
        "Medical Data"
      ]
    },
    {
      "metadata": {
        "id": "me7jAEFDxByr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "7eabedc4-00bd-4690-9e41-e3c19db3f14b"
      },
      "cell_type": "code",
      "source": [
        "def knnclassifier(file1,file2,k):\n",
        "\tdataset1 = np.array(loadCsv(file1))\n",
        "\tdataset1 = dataset1[1:1500]\n",
        "\tdataset2 = np.array(loadCsv(file2))\n",
        "\tdataset2 = dataset2[1500:3000]\n",
        "\n",
        "\tunique,counts  = np.unique(dataset1[:,0],return_counts=True)\n",
        "\taccuracy =0\n",
        "\tfor i in dataset2:\n",
        "\t\t#print i\n",
        "\t\tclas = find(i,dataset1,k)\n",
        "\t\t#print \"---\"\n",
        "\t\t#print clas\n",
        "\t\t#print\"-----\"\n",
        "\t\tif clas == i[0]:\n",
        "\t\t\taccuracy +=1\n",
        "\tprint (float(accuracy)/float(len(dataset2)))*100\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "def find(data,file,k):\n",
        "\ta = []\n",
        "\tfor j in file:\n",
        "\t\tdis = finddis(data,j)\n",
        "\t\ta.append([j[0],dis])\n",
        "\t#print len(a)\n",
        "\tb = findclass(a,k)\n",
        "\treturn b\n",
        "\n",
        "def findclass(a,k):\n",
        "\tm = {}\n",
        "\tb =[]\n",
        "\tn = len(a)\n",
        "\tfor i in range(k):\n",
        "\t\tp =float(\"inf\")\n",
        "\t\tc = ''\n",
        "\t\tfor j in range(i,n-1):\n",
        "\t\t\tif a[j][1] <p:\n",
        "\t\t\t\tp = a[j][1]\n",
        "\t\t\t\tc = a[j][0] \n",
        "\t\tb.append([c,p])\n",
        "\t#print b\n",
        "\tfor d in range(k-1):\n",
        "\t\tif b[d][0] not in m:\n",
        "\t\t\tm[b[d][0]] =1\n",
        "\t\telse:\n",
        "\t\t\tm[b[d][0]] +=1\n",
        "\tmaxclass = None\n",
        "\tmaxvalue = -1\n",
        "\tfor clas,value in m.iteritems():\n",
        "\t\tif maxvalue<value:\n",
        "\t\t\tmaxvalue = value\n",
        "\t\t\tmaxclass = clas\n",
        "\treturn maxclass\n",
        "\n",
        "def finddis(data1,data2):\n",
        "\ta = np.array(data1[1:3])\n",
        "\ta = a.astype(np.float)\n",
        "\tb = np.array(data2[1:3])\n",
        "\tb = b.astype(np.float)\n",
        "\treturn math.sqrt(np.sum((a-b)**2))\n",
        "\n",
        "def loadCsv(filename):\n",
        "\tlines = csv.reader(open(filename, \"rb\"))\n",
        "\tdataset = list(lines)\n",
        "\tleng = len(dataset)\n",
        "\tdataset = dataset[1:leng]\n",
        "\treturn dataset\n",
        "knnclassifier(\"drive/ml1/Medical_data.csv\",\"drive/ml1/test_medical.csv\",50)"
      ],
      "execution_count": 99,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "79.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "MSdeYLL4yPfx",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#Part B\n",
        "Railway Data"
      ]
    },
    {
      "metadata": {
        "id": "P4MxAXNvymuJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        },
        "outputId": "b86a08bf-e352-40e8-d7d2-907e4764b5e1"
      },
      "cell_type": "code",
      "source": [
        "def knnclassifier(file1,file2,k):\n",
        "\tdataset1 = np.array(loadCsv(file1))\n",
        "\tdataset2 = np.array(loadCsv(file2))\n",
        "\tdataset11 = dataset1[:,[1,2,3,6]]\n",
        "\tdataset12 = dataset1[:,[1,4,5]]\n",
        "\tdataset21 = dataset2[:,[1,2,3,6]]\n",
        "\tdataset22 = dataset2[:,[1,4,5]]\n",
        "\n",
        "\tunique,counts  = np.unique(dataset1[:,0],return_counts=True)\n",
        "\taccuracy =0\n",
        "\tfp =0\n",
        "\tfn=0\n",
        "\ttp = 0\n",
        "\tfor i in range(len(dataset2)):\n",
        "\t\t#print i\n",
        "\t\tclas = find(dataset21[i],dataset22[i],dataset11,dataset22,k)\n",
        "\t\t#print \"---\"\n",
        "\t\t#print clas\n",
        "\t\t#print\"-----\"\n",
        "\t\tif clas == dataset21[i][0]:\n",
        "\t\t\taccuracy +=1\n",
        "\t\t\tif(clas ==\"1\"):\n",
        "\t\t\t\ttp +=1\n",
        "\t\telse:\n",
        "\t\t\tif(clas==\"1\"):\n",
        "\t\t\t\tfp +=1\n",
        "\t\t\tif(clas==\"0\"):\n",
        "\t\t\t\tfn += 1\n",
        "\n",
        "\taccuracy =  (float(accuracy)/float(len(dataset2)))*100\n",
        "\tprecision = float(tp)/float(tp+fp)\n",
        "\trecall = float(tp)/float(tp+fn)\n",
        "\tprint \"accuaracy = \" + str(accuracy)\n",
        "\tprint \"precision = \" + str(precision)\n",
        "\tprint \"recall = \" + str(recall)\n",
        "\tprint \"F1 score = \" +str(float(2*precision*recall)/float(precision+recall))\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "def find(data,data1,file,file1,k):\n",
        "\ta = []\n",
        "\tfor j in range(len(file)):\n",
        "\t\tdis = finddis(data,data1,file[j],file1[j])\n",
        "\t\ta.append([file[j][0],dis])\n",
        "\t#print len(a)\n",
        "\tb = findclass(a,k)\n",
        "\treturn b\n",
        "\n",
        "def findclass(a,k):\n",
        "\tm = {}\n",
        "\tb =[]\n",
        "\tn = len(a)\n",
        "\tfor i in range(k):\n",
        "\t\tp =float(\"inf\")\n",
        "\t\tc = ''\n",
        "\t\tfor j in range(i,n-1):\n",
        "\t\t\tif a[j][1] <p:\n",
        "\t\t\t\tp = a[j][1]\n",
        "\t\t\t\tc = a[j][0] \n",
        "\t\tb.append([c,p])\n",
        "\t#print b\n",
        "\tfor d in range(k-1):\n",
        "\t\tif b[d][0] not in m:\n",
        "\t\t\tm[b[d][0]] =1\n",
        "\t\telse:\n",
        "\t\t\tm[b[d][0]] +=1\n",
        "\tmaxclass = None\n",
        "\tmaxvalue = -1\n",
        "\tfor clas,value in m.iteritems():\n",
        "\t\tif maxvalue<value:\n",
        "\t\t\tmaxvalue = value\n",
        "\t\t\tmaxclass = clas\n",
        "\treturn maxclass\n",
        "\n",
        "def finddis(data1,data3,data2,data4):\n",
        "\ta = np.array(data1[1:3])\n",
        "\ta = a.astype(np.float)\n",
        "\tb = np.array(data2[1:3])\n",
        "\tb = b.astype(np.float)\n",
        "\tc = math.sqrt(np.sum((a-b)**2))\n",
        "\tif data3[1] != data4[1]:\n",
        "\t\tc+=2\n",
        "\tif data3[2] != data4[2]:\n",
        "\t\tc+=2\n",
        "\treturn c\n",
        "\n",
        "def loadCsv(filename):\n",
        "\tlines = csv.reader(open(filename, \"rb\"))\n",
        "\tdataset = list(lines)\n",
        "\tleng = len(dataset)\n",
        "\tdataset = dataset[1:leng]\n",
        "\treturn dataset\n",
        "knnclassifier(\"drive/ml1/railwayBookingList.csv\",\"drive/ml1/railwayBookingList.csv\",25)"
      ],
      "execution_count": 100,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "accuaracy = 79.7709923664\n",
            "precision = 0.827130852341\n",
            "recall = 0.850617283951\n",
            "F1 score = 0.838709677419\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "EPamaqley6Rx",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#Part C\n",
        "Fashion Mnist"
      ]
    },
    {
      "metadata": {
        "id": "xQAWGZVAy0Vf",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def pcareduction(X_train):\n",
        "\ta = []\n",
        "\tfor i in X_train:\n",
        "\t    M = np.mean(i)\n",
        "\t    i = i-M\n",
        "\t    b = i.reshape(28,28)\n",
        "\t    c = np.cov(b)\n",
        "\t    values,vectors = np.linalg.eig(c)\n",
        "\t    eig_pair = [(np.abs(values[i]),vectors[:,i]) for i in range(len(values))]\n",
        "\t    eig_pair.sort(key=lambda x:x[0])\n",
        "\t    eig_pair.reverse()\n",
        "\t    #print eig_pair\n",
        "\t    ab = []\n",
        "\t    #print eig_pair[0][0]\n",
        "\t    for j in range(6):\n",
        "\t    \tab = np.concatenate((ab,eig_pair[j][1]),axis = None)\n",
        "\t    #print ab\n",
        "\t    a.append(ab)\n",
        "\t#print a\n",
        "\n",
        "\n",
        "\n",
        "\treturn a\n",
        "\n",
        "def knnclassifier(k):\n",
        "\tX_train, y_train = mnist_reader.load_mnist(\"drive/ml1/data/fashion\", kind='train')\n",
        "\tX_test, y_test = mnist_reader.load_mnist(\"drive/ml1/data/fashion\", kind='t10k')\n",
        "\tX_train  = pcareduction(X_train)\n",
        "\tX_test = pcareduction(X_test)\n",
        "\ty_train = np.reshape(y_train, (-1, 1))\n",
        "\ty_test = np.reshape(y_test, (-1, 1))\n",
        "\tdataset1 = np.hstack((y_train,X_train))\n",
        "\tdataset2 = np.hstack((y_test,X_test))\n",
        "\tdataset1 = dataset1\n",
        "\tdataset2 = dataset2\n",
        "\tunique,counts  = np.unique(dataset1[:,0],return_counts=True)\n",
        "\taccuracy =0\n",
        "\tfor i in dataset2:\n",
        "\t\t#print i\n",
        "\t\tclas = find(i,dataset1,k)\n",
        "\t\t#print \"---\"\n",
        "\t\t#print clas\n",
        "\t\t#print\"-----\"\n",
        "\t\tif clas == i[0]:\n",
        "\t\t\taccuracy +=1\n",
        "\t\t\t#print \"correct\"\n",
        "\t\t#else :\n",
        "\t\t\t#print \"incorrect\"\n",
        "\tprint \"accurracy=\" + str((float(accuracy)/float(len(dataset2)))*100)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "def find(data,file,k):\n",
        "\ta = []\n",
        "\tfor j in file:\n",
        "\t\tdis = finddis(data,j)\n",
        "\t\ta.append([j[0],dis])\n",
        "\t#print len(a)\n",
        "\tb = findclass(a,k)\n",
        "\treturn b\n",
        "\n",
        "def findclass(a,k):\n",
        "\tm = {}\n",
        "\tb =[]\n",
        "\tn = len(a)\n",
        "\tfor i in range(k):\n",
        "\t\tp =float(\"inf\")\n",
        "\t\tc = ''\n",
        "\t\tfor j in range(i,n-1):\n",
        "\t\t\tif a[j][1] <p:\n",
        "\t\t\t\tp = a[j][1]\n",
        "\t\t\t\tc = a[j][0]\n",
        "\t\tb.append([c,p])\n",
        "\t#print b\n",
        "\tfor d in range(k-1):\n",
        "\t\tif b[d][0] not in m:\n",
        "\t\t\tm[b[d][0]] =1\n",
        "\t\telse:\n",
        "\t\t\tm[b[d][0]] +=1\n",
        "\tmaxclass = None\n",
        "\tmaxvalue = -1\n",
        "\tfor clas,value in m.iteritems():\n",
        "\t\tif maxvalue<value:\n",
        "\t\t\tmaxvalue = value\n",
        "\t\t\tmaxclass = clas\n",
        "\treturn maxclass\n",
        "\n",
        "def finddis(data1,data2):\n",
        "\ta = np.array(data1[1:len(data1)])\n",
        "\ta = a.astype(np.float)\n",
        "\tb = np.array(data2[1:len(data2)])\n",
        "\tb = b.astype(np.float)\n",
        "\treturn math.sqrt(np.sum((a-b)**2))\n",
        "\n",
        "def loadCsv(filename):\n",
        "\tlines = csv.reader(open(filename, \"rb\"))\n",
        "\tdataset = list(lines)\n",
        "\tleng = len(dataset)\n",
        "\tdataset = dataset[1:leng]\n",
        "\treturn dataset\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "YYL9gAIA3MUX",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "knnclassifier(100)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Dr2KnpvO3OWX",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "![alt text](https://3.bp.blogspot.com/-VaR_5oYII5U/W57oNE53MPI/AAAAAAAAKO0/s1FnDpGl07g2yDGNl0ve6lWdXQCliP_3gCLcBGAs/s320/Screenshot_5.png)"
      ]
    },
    {
      "metadata": {
        "id": "Z_UwDAgfzoZo",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#Section 5\n",
        "K Means Clustering"
      ]
    },
    {
      "metadata": {
        "id": "zsOhyZrLzm_y",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "\n",
        "def precision(predicted_test,test):\n",
        "    TP = 0\n",
        "    FP = 0\n",
        "    for i in range(len(test)):\n",
        "        if predicted_test[i][1] == test[i][1]:\n",
        "            if predicted_test[i][1]==1:\n",
        "                TP = TP + 1\n",
        "        else:\n",
        "            if predicted_test[i][1] == 1:\n",
        "                FP = FP + 1\n",
        "    #print (str(TP)+\" \"+str(FP))\n",
        "    return TP*(1.0/(TP+FP))\n",
        "def accuracy(predicted_test,test):\n",
        "    T = 0\n",
        "    for i in range(len(test)):\n",
        "        if predicted_test[i][1] == test[i][1]:\n",
        "            T = T + 1\n",
        "    #print (str(T)+\" \")\n",
        "    return T*(1.0/len(test))\n",
        "def recall(predicted_test,test):\n",
        "    TP = 0\n",
        "    FN = 0\n",
        "    for i in range(len(test)):\n",
        "        if predicted_test[i][1] == test[i][1]:\n",
        "            if predicted_test[i][1]==1:\n",
        "                TP = TP + 1\n",
        "        else:\n",
        "            if predicted_test[i][1] == 0:\n",
        "                FN = FN + 1\n",
        "\n",
        "    return TP*(1.0/(TP+FN))\n",
        "def F1(precision,recall):\n",
        "    return 2.0*(precision*recall)*(1.0/(precision+recall))\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "class kmean:\n",
        "    name = \"\"\n",
        "    k=2\n",
        "    def __init__(self,name=\"K-Means Clustering\",k=2):\n",
        "        self.name = name\n",
        "        self.k = k\n",
        "\n",
        "    def initialize(self,train):\n",
        "\n",
        "        l=[]\n",
        "        i=0\n",
        "        for t in train:\n",
        "            if t[1]==i:\n",
        "                l.append(t[0])\n",
        "                i+=1\n",
        "            if i==self.k:\n",
        "                break\n",
        "        print(\"number of classes: \"+str(len(l)))\n",
        "        return l\n",
        "\n",
        "    def make_cluster(self,train,centroids):\n",
        "        clusters = {c: [] for c in range(self.k)}\n",
        "        #initialze cluster with key as label\n",
        "\n",
        "        for X,Y in train:\n",
        "            min = float('inf')\n",
        "            its_cluster = 0\n",
        "\n",
        "            for i in range(self.k):\n",
        "                centroid_i = centroids[i]\n",
        "                euclidean_dis = np.linalg.norm(X-centroid_i)\n",
        "                if euclidean_dis<min:\n",
        "                    its_cluster = i\n",
        "                    min = euclidean_dis\n",
        "            clusters[its_cluster].append((Y,X))\n",
        "        return clusters.values()\n",
        "\n",
        "    def avg(self,cluster):\n",
        "        #print(len(cluster))\n",
        "        l = list(map(lambda x: x[1], cluster))\n",
        "        return np.array(l).mean(axis=0)\n",
        "    def change_centroids(self,clusters):\n",
        "        centroids = []\n",
        "        #print(len(clusters))\n",
        "        for cluster in clusters:\n",
        "            #print(len(cluster))\n",
        "            centroids.append(self.avg(cluster))\n",
        "\n",
        "        return centroids\n",
        "\n",
        "    def train_the_model(self,train):\n",
        "        centroids = self.initialize(train)\n",
        "        clusters = self.make_cluster(train,centroids)\n",
        "        diff = 0\n",
        "        i=0\n",
        "        while True:\n",
        "            old_centroids = centroids\n",
        "            centroids = self.change_centroids(clusters)\n",
        "            clusters = self.make_cluster(train,centroids)\n",
        "            #print(\"diff\",end=\"\\r\")\n",
        "            #print(list(map(lambda a,b : np.linalg.norm(a-b),old_centroids,centroids)))\n",
        "            new_diff = max(list(map(lambda a,b : np.linalg.norm(a-b),old_centroids,centroids)))\n",
        "            #print (\"new_diff-: \"+ str(new_diff),end=\"\\r\")\n",
        "            try:\n",
        "                change_in_diff = abs((new_diff-diff)/np.mean([diff,new_diff])) * 100\n",
        "                #print (\"iteration-:\"+str(i) + \" \" +str(change_in_diff) ,end=\"\\r\")\n",
        "            except:\n",
        "                break\n",
        "            if np.isnan(change_in_diff):\n",
        "                break\n",
        "\n",
        "            diff = new_diff\n",
        "            i += 1\n",
        "        return clusters,centroids\n",
        "\n",
        "    def labelling(self,clusters,centroids):\n",
        "        new_centroids = []\n",
        "        i=0\n",
        "        for cluster in clusters:\n",
        "            labels = [y for y,x in cluster]\n",
        "            high_freq = max(set(labels),key=labels.count)\n",
        "            centroid = (high_freq,centroids[i])\n",
        "            new_centroids.append(centroid)\n",
        "            i = i + 1\n",
        "        return new_centroids\n",
        "\n",
        "    def test(self,testimage,centroids):\n",
        "        min = float(\"inf\")\n",
        "        y = 0\n",
        "        for (label, centroid) in centroids:\n",
        "            distance = np.linalg.norm(centroid - testimage)\n",
        "            if distance < min:\n",
        "                min = distance\n",
        "                y = label\n",
        "        #print(y)\n",
        "        return y\n",
        "def main(train,test,k):\n",
        "    #print(\"Creating classifier Object...\")\n",
        "    \n",
        "    classifier = kmean(k=k)\n",
        "    #print(\"classifier name: \" + classifier.name)\n",
        "    #print(\"Training Start Here...\")\n",
        "    \n",
        "    clusters,centroids=classifier.train_the_model(train)\n",
        "    labelled_centroids = classifier.labelling(clusters,centroids)\n",
        "    predicted_test = []\n",
        "    #print(\"Prediction Start Here...\")\n",
        "    \n",
        "    for x,y in test:\n",
        "        yy = classifier.test(x,labelled_centroids)\n",
        "        predicted_test.append((x,yy))\n",
        "\n",
        "    predicted_train = []\n",
        "    for x,y in train:\n",
        "        yy = classifier.test(x,labelled_centroids)\n",
        "        predicted_train.append((x,yy))\n",
        "\n",
        "    print(\"Test Accuracy: \" + str(accuracy(predicted_test,test)))\n",
        "    #print(\"Training Accuracy: \" + str(accuracy(predicted_train,test)))\n",
        "\n",
        "    if k==2:\n",
        "        p=precision(predicted_test,test)\n",
        "        r=recall(predicted_test,test)\n",
        "        print(\"Precision \"+str(p))\n",
        "        print(\"Recall \"+str(r))\n",
        "        print(\"F1 Score \"+ str(F1(p,r)))\n",
        "\n",
        "\n",
        "\n",
        "def mnist():\n",
        "    trainX , trainY = load_mnist('drive/ml1/data/fashion', kind='train')\n",
        "    testX , testY = load_mnist('drive/ml1/data/fashion', kind='t10k')\n",
        "    train = [list(a) for a in zip(trainX, trainY)]\n",
        "    test = [list(a) for a in zip(testX, testY)]\n",
        "\n",
        "    main(train,test,10)\n",
        "\n",
        "def medical():\n",
        "    df = pd.read_csv('drive/ml1/Medical_data.csv')\n",
        "    df2 = pd.read_csv('drive/ml1/test_medical.csv')\n",
        "    df['Health'] = df['Health'].map({'HEALTHY':0,'MEDICATION':1,'SURGERY':2})\n",
        "    df2['Health'] = df2['Health'].map({'HEALTHY':0,'MEDICATION':1,'SURGERY':2})\n",
        "    trainY = df['Health'].values.tolist()\n",
        "    trainXX = df[['TEST1','TEST2','TEST3']].values.tolist()\n",
        "    trainX = list(map(lambda x: np.array(x),trainXX))\n",
        "    train = [list(a) for a in zip(trainX, trainY)]\n",
        "    testY = df2['Health'].values.tolist()\n",
        "    testXX = df2[['TEST1','TEST2','TEST3']].values.tolist()\n",
        "    testX = list(map(lambda x: np.array(x),testXX))\n",
        "    test = [list(a) for a in zip(testX, testY)]\n",
        "    #print(type(train))\n",
        "    main(train,test,3)\n",
        "\n",
        "def rail():\n",
        "    df = pd.read_csv('drive/ml1/railwayBookingList.csv')\n",
        "    dataY = df['boarded'].values.tolist()\n",
        "    df.memberCount = df.memberCount*df['budget'].mean()\n",
        "    df.age = df.age*df['budget'].mean()\n",
        "    df['preferredClass'] = df['preferredClass'].map({'FIRST_AC':100000,'SECOND_AC':500000,'THIRD_AC':100000,'NO_PREF':0})\n",
        "    df['sex'] = df['sex'].map({'female':300000,'male':300000})\n",
        "    dataXX = df[['budget','memberCount','preferredClass','sex','age']].values.tolist()\n",
        "    dataX = list(map(lambda x: np.array(x),dataXX))\n",
        "    data = [list(a) for a in zip(dataX, dataY)]\n",
        "    train = data[:int(len(data)*0.75)]\n",
        "    test = data[int(len(data)*0.75):]\n",
        "    #print(df[['budget','memberCount','preferredClass','sex','age']])\n",
        "\n",
        "    main(train,test,2)\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ZXGFuKlP0GoI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "981c6835-a5da-4717-e658-86ea0c344ed9"
      },
      "cell_type": "code",
      "source": [
        "medical()\n"
      ],
      "execution_count": 115,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "number of classes: 3\n",
            "Test Accuracy: 0.534666666667\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:103: RuntimeWarning: invalid value encountered in double_scalars\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "rY_1bnSA0-Eh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        },
        "outputId": "80ad67e9-7e73-4043-d6ee-4a0cde8d26db"
      },
      "cell_type": "code",
      "source": [
        "rail()"
      ],
      "execution_count": 116,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "number of classes: 2\n",
            "Test Accuracy: 0.743902439024\n",
            "Precision 0.746177370031\n",
            "Recall 0.995918367347\n",
            "F1 Score 0.853146853147\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:103: RuntimeWarning: invalid value encountered in double_scalars\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "EqrltOIj0_ky",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "670a2282-35c3-404e-9841-5c29969fbdb5"
      },
      "cell_type": "code",
      "source": [
        "mnist()"
      ],
      "execution_count": 117,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "number of classes: 10\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python2.7/dist-packages/ipykernel_launcher.py:103: RuntimeWarning: invalid value encountered in double_scalars\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Test Accuracy: 0.5275\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "ipKxKL7x1EJv",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}