LSTM_WITH_GATING_WordLevel.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "LSTM_WITH_GATING_WordLevel.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/notmanan/Depression-Detection-Through-Multi-Modal-Data/blob/master/LSTM_WITH_GATING_WordLevel.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i9WrTCsMSj0-",
        "colab_type": "text"
      },
      "source": [
        "Mount to Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T8W21nrARt0P",
        "colab_type": "code",
        "outputId": "0a927015-5090-471d-ce63-c0a756e8cbde",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 211
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive', force_remount=True)\n",
        "\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from gensim.models.keyedvectors import KeyedVectors\n",
        "import gc\n",
        "import math\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Conv2D\n",
        "from keras.layers import MaxPooling2D\n",
        "from keras.layers import Flatten\n",
        "from keras.layers import Dense\n",
        "from smart_open import open\n",
        "from nltk.corpus import stopwords\n",
        "from sklearn.metrics import classification_report\n",
        "from keras.layers import Dropout\n",
        "from matplotlib import pyplot as plt\n",
        "\n",
        "dev_location = \"dev_data\"\n",
        "test_location = \"test_data\"\n",
        "train_location = \"train_data\"\n",
        "\n",
        "devData = np.array(pd.read_csv('/content/drive/My Drive/MCA Dataset/dev_split_Depression_AVEC2017.csv',delimiter=',',encoding='utf-8'))[:, 0:2]\n",
        "testData = np.array(pd.read_csv('/content/drive/My Drive/MCA Dataset/full_test_split.csv',delimiter=',',encoding='utf-8'))[:, 0:2]\n",
        "trainData = np.array(pd.read_csv('/content/drive/My Drive/MCA Dataset/train_split_Depression_AVEC2017.csv',delimiter=',',encoding='utf-8'))[:, 0:2]\n",
        "\n",
        "dataset = np.concatenate((devData, np.concatenate((testData, trainData))))\n",
        "\n",
        "gc.collect()      \n",
        "model = KeyedVectors.load_word2vec_format('/content/drive/My Drive/MCA Dataset/Copy of GoogleNews-vectors-negative300.bin', binary=True)\n",
        "stop_words = set(stopwords.words('english'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n",
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:253: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZfzScFaPTd93",
        "colab_type": "code",
        "outputId": "9e54172a-0f3c-4981-ebfc-108d997f8e8a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "def checkDataPointExistence(patientID, split):\n",
        "  for i in split:\n",
        "    if(patientID == i[0]):\n",
        "      return True\n",
        "  return False\n",
        "\n",
        "def getData(patientID, location):\n",
        "  print(\"PatientID: \" + str(int(patientID)))\n",
        "  retData = [int(patientID)]\n",
        "  textD = getTextData(patientID, location)\n",
        "  textD = np.array(textD)\n",
        "  audioD = getAudioData(patientID, location, textD)\n",
        "  videoD = getVideoData(str(int(patientID)), location, textD)\n",
        "  # patientD = np.concatenate((textD, audioD, videoD), axis = 1)\n",
        "  # print(\"Final Patient Data: \" + str(patientD.shape))\n",
        "  # patientD = textD\n",
        "\n",
        "  return textD,audioD,videoD\n",
        "\n",
        "def getTextData(patientID, location):\n",
        "  fileName = \"/content/drive/My Drive/MCA Dataset/\"+ str(location) + \"/\" + str(int(patientID)) + \"_TRANSCRIPT.csv\"\n",
        "  file = np.array(pd.read_csv(fileName,delimiter='\\t',encoding='utf-8', engine='python'))\n",
        "  outputList = []\n",
        "\n",
        "  for i in range(len(file)):\n",
        "    sentence = file[i][3]\n",
        "    words = str(sentence).split(\" \")\n",
        "    totalWordLength = 0\n",
        "    for word in words: \n",
        "      totalWordLength += len(word)\n",
        "\n",
        "    sentenceStart = file[i][0] \n",
        "    sentenceEnd = file[i][1]\n",
        "    speaker = file[i][2]\n",
        "    if speaker == 'Ellie':\n",
        "      continue\n",
        "    else:\n",
        "      speaker = 1\n",
        "\n",
        "    totalTime = sentenceEnd - sentenceStart\n",
        "    timePerChar = totalTime/totalWordLength\n",
        "\n",
        "    wordStart = sentenceStart\n",
        "    for word in words:\n",
        "      wordEnd = wordStart + (timePerChar * len(word))\n",
        "      appender = [wordStart, wordEnd, speaker]\n",
        "      vector = list(returnWordToVec(word))\n",
        "      for v in vector:\n",
        "        appender.append(v)\n",
        "      # print(appender)\n",
        "      # print(len(appender))\n",
        "      outputList.append(appender)\n",
        "      wordStart = wordEnd\n",
        "\n",
        "  return outputList\n",
        "\n",
        "def remove_StopWords(sentence):\n",
        "    filtered_sentence = [] \n",
        "    for w in sentence: \n",
        "        if w not in stop_words: \n",
        "            filtered_sentence.append(w)\n",
        "    return filtered_sentence\n",
        "\n",
        "def returnWordToVec(word):\n",
        "  word = str(word)\n",
        "  try:\n",
        "    if word[0] == '<':\n",
        "        word = word[1:]\n",
        "    if word[-1] == '>':\n",
        "        word = word[0:-1]\n",
        "  except:\n",
        "    dfdsd = 3\n",
        "  if(word in model):\n",
        "      return np.array(model[word])\n",
        "  else:\n",
        "    return np.zeros((300))\n",
        "\n",
        "def audioDataHelper(X):\n",
        "    for i in range(X.shape[0]):\n",
        "        if(X[i,1] == 0):\n",
        "            X[i,0] = 0\n",
        "            for j in range(7):\n",
        "                X[i,j+1] = 0\n",
        "    X = np.array(X)\n",
        "    return X\n",
        "    \n",
        "def getAudioData(patientID, location, textD):\n",
        "  fileName = \"/content/drive/My Drive/MCA Dataset/\"+ str(location) + \"/\" + str(int(patientID)) + \"_COVAREP.csv\"\n",
        "  data = pd.read_csv(fileName,header = None)\n",
        "  data = data.iloc[:,:].values\n",
        "  data = audioDataHelper(data)\n",
        "  # print(\"Audio Raw Data:\" + str(data.shape))\n",
        "  sentenceDatas = []\n",
        "  for sentence in textD:\n",
        "    sentenceStartime = sentence[0]\n",
        "    sentenceEndTime = sentence[1]\n",
        "    startIndex = math.floor(sentenceStartime/0.01)\n",
        "    endIndex = math.ceil(sentenceEndTime/0.01)\n",
        "    sentenceData = data[startIndex: endIndex]\n",
        "    sentenceData = np.average(sentenceData, axis = 0)\n",
        "    sentenceData = np.array(sentenceData.reshape(1, -1))\n",
        "    sentenceDatas.append(sentenceData)\n",
        "  sentenceDatas = np.array(sentenceDatas)\n",
        "  sentenceDatas = np.reshape(sentenceDatas, (textD.shape[0],-1))\n",
        "\n",
        "  return sentenceDatas\n",
        "\n",
        "def getVideoDataHelper(patientID, location):\n",
        "  root = \"/content/drive/My Drive/MCA Dataset/\"+ str(location) + \"/\" \n",
        "  file1 = root + (patientID)+\"_CLNF_AUs.txt\"\n",
        "  file2 = root + (patientID)+\"_CLNF_features.txt\"\n",
        "  file3 = root + (patientID)+\"_CLNF_features3D.txt\"\n",
        "  file4 = root + (patientID)+\"_CLNF_gaze.txt\"\n",
        "  file6 = root + (patientID)+\"_CLNF_pose.txt\"\n",
        "  data = processVideoData(file1)\n",
        "  data = np.concatenate((data, processVideoData(file2)), 1)\n",
        "  data = np.concatenate((data, processVideoData(file3)), 1)\n",
        "  data = np.concatenate((data, processVideoData(file4)), 1)\n",
        "  data = np.concatenate((data, processVideoData(file6)), 1)\n",
        "\n",
        "  # try:\n",
        "  #   hogFile = open(file5, 'r')\n",
        "  # except:\n",
        "  #   print(\"Hog tXT file\")\n",
        "  #   hogFile = open(root + (patientID)+\"_CLNF_hog.txt\", 'r')\n",
        "  # hogData = np.fromfile(hogFile, dtype = np.uint32)\n",
        "  # hogData = np.reshape(hogData, (-1, 4468))\n",
        "  \n",
        "  # data = np.concatenate((data, hogData), 1)\n",
        "  return data\n",
        "\n",
        "def processVideoData(filename):\n",
        "  try:\n",
        "    data = pd.read_csv(filename,delimiter=',', dtype=float)\n",
        "    X = data.iloc[:,:].values\n",
        "    X = np.delete(X, 0, 1)\n",
        "    X = np.delete(X, 1, 1)\n",
        "  except:\n",
        "    print(\"Video Data corrupt, fixing.\")\n",
        "    data = pd.read_csv(filename,delimiter=',')\n",
        "    X = data.iloc[:,:].values\n",
        "    X = np.delete(X, 0, 1)\n",
        "    X = np.delete(X, 1, 1)\n",
        "    for i in range(len(X)):\n",
        "        if(isinstance(X[i][5],str) or isinstance(X[i][7],str)):\n",
        "            X[i] = np.zeros((1, X.shape[1]))\n",
        "  return X\n",
        "\n",
        "def getVideoData(patientID, location, textD):\n",
        "  data = getVideoDataHelper(patientID, location)\n",
        "  sentenceDatas = []\n",
        "  for sentence in textD:\n",
        "    sentenceStartime = sentence[0]\n",
        "    sentenceEndTime = sentence[1]\n",
        "    startIndex = math.floor(sentenceStartime/0.333)\n",
        "    endIndex = math.ceil(sentenceEndTime/0.333)\n",
        "    sentenceData = data[startIndex: endIndex]\n",
        "    sentenceData = np.average(sentenceData, axis = 0)\n",
        "    sentenceData = np.array(sentenceData.reshape(1, -1))\n",
        "    sentenceDatas.append(sentenceData)\n",
        "  \n",
        "  sentenceDatas = np.array(sentenceDatas)\n",
        "  sentenceDatas = np.reshape(sentenceDatas, (textD.shape[0],-1))\n",
        "  return sentenceDatas\n",
        "\n",
        "# Xtrain = []\n",
        "Ytrain = []\n",
        "# Xtest = []\n",
        "Ytest = []\n",
        "\n",
        "audio_train = []\n",
        "video_train = []\n",
        "text_train = []\n",
        "audio_test = []\n",
        "video_test = []\n",
        "text_test = []\n",
        "\n",
        "\n",
        "lengths = []\n",
        "for datapoint in dataset:\n",
        "  # print(datapoint[0])\n",
        "  if(checkDataPointExistence(datapoint[0], devData)):\n",
        "\n",
        "    Data Point in Dev Set\n",
        "    text,audio,video = getData(datapoint[0], dev_location)\n",
        "    text = text[:,2:]\n",
        "    audio_train.append(audio)\n",
        "    video_train.append(video)\n",
        "    text_train.append(text)\n",
        "    Ytrain.append(datapoint[1])\n",
        "    # print(data)\n",
        "  if(checkDataPointExistence(datapoint[0], testData)):\n",
        "    # Data Point in Test Set\n",
        "    text,audio,video = getData(datapoint[0], test_location)\n",
        "    text = text[:,2:]\n",
        "    audio_test.append(audio)\n",
        "    video_test.append(video)\n",
        "    text_test.append(text)\n",
        "    Ytest.append(datapoint[1])\n",
        "  elif(checkDataPointExistence(datapoint[0], trainData)):\n",
        "    # Data Point in Train Set\n",
        "    text,audio,video = getData(datapoint[0], train_location)\n",
        "    text = text[:,2:]\n",
        "    audio_train.append(audio)\n",
        "    video_train.append(video)\n",
        "    text_train.append(text)\n",
        "    Ytrain.append(datapoint[1])\n",
        "  # lengths.append(data.shape[0])\n",
        "model = []\n",
        "gc.collect()\n",
        "\n",
        "\n",
        "def refactor(arr, size):\n",
        "  temp = arr[0:min(len(arr),size), :]\n",
        "  if (len(temp) < size):\n",
        "    temp = np.concatenate((temp, np.zeros(((size - len(temp)), arr.shape[1]))), axis = 0 )\n",
        "  return temp\n",
        "\n",
        "numberOfSentences = 1700\n",
        "\n",
        "for i in range(len(audio_train)):\n",
        "  audio_train[i] = refactor(audio_train[i], numberOfSentences)\n",
        "  video_train[i] = refactor(video_train[i], numberOfSentences)\n",
        "  text_train[i] = refactor(text_train[i], numberOfSentences)\n",
        "\n",
        "for i in range(len(audio_test)):\n",
        "  audio_test[i] = refactor(audio_train[i], numberOfSentences)\n",
        "  video_test[i] = refactor(video_train[i], numberOfSentences)\n",
        "  text_test[i] = refactor(text_train[i], numberOfSentences)\n",
        "\n",
        "audio_test = np.array(audio_test)\n",
        "video_test = np.array(video_test)\n",
        "text_test = np.array(text_test)\n",
        "\n",
        "audio_train = np.array(audio_train)\n",
        "video_train = np.array(video_train)\n",
        "text_train = np.array(text_train)\n",
        "\n",
        "gc.collect()\n",
        "\n",
        "print(audio_test.shape,video_test.shape,text_test.shape)\n",
        "print(audio_train.shape,video_train.shape,text_train.shape)\n",
        "\n",
        "dataset = []\n",
        "devData = []\n",
        "trainData = []\n",
        "testdata = []\n",
        "gc.collect()\n",
        "\n",
        "Ytrain = np.array(Ytrain)\n",
        "Ytest = np.array(Ytest)\n",
        "\n",
        "gc.collect()\n",
        "\n",
        "import sklearn\n",
        "from sklearn import preprocessing\n",
        "\n",
        "def upsample(X_train,Y_train):\n",
        "  X_train_0 = X_train[Y_train==0]\n",
        "  X_train_1 = X_train[Y_train==1]\n",
        "\n",
        "  Y_train_1 = Y_train[Y_train==1]\n",
        "  size = X_train_0.shape[0] - X_train_1.shape[0]\n",
        "  X = []\n",
        "  Y = []\n",
        "  X_train = list(X_train)\n",
        "  Y_train = list(Y_train)\n",
        "  while(size>0):\n",
        "    size -= 1\n",
        "    index = np.random.randint(0,X_train_1.shape[0]-1)\n",
        "    leave_index = np.random.randint(0,len(X_train)-1)\n",
        "    X_add = X_train_1[index]\n",
        "    X_leave = X_train[leave_index]\n",
        "\n",
        "    Y_add = Y_train_1[index]\n",
        "    Y_leave = Y_train[leave_index]\n",
        "\n",
        "    X_train[leave_index] = X_add\n",
        "    X_train.append(X_leave)\n",
        "\n",
        "    Y_train[leave_index] = Y_add\n",
        "    Y_train.append(Y_leave)\n",
        "\n",
        "\n",
        "  X_train = np.array(X_train)\n",
        "  Y_train = np.array(Y_train)\n",
        "  return X_train,Y_train\n",
        "\n",
        "\n",
        "audio_train = np.nan_to_num(audio_train)\n",
        "video_train = np.nan_to_num(video_train)\n",
        "text_train = np.nan_to_num(text_train)\n",
        "\n",
        "\n",
        "audio_train, _ = upsample(audio_train,Ytrain)\n",
        "video_train, _ = upsample(video_train,Ytrain)\n",
        "text_train, Ytrain = upsample(text_train,Ytrain)\n",
        "\n",
        "gc.collect()\n",
        "\n",
        "for i in range(audio_train.shape[0]):\n",
        "  audio_train[i] = sklearn.preprocessing.normalize(audio_train[i])\n",
        "  video_train[i] = sklearn.preprocessing.normalize(video_train[i])\n",
        "  text_train[i] = sklearn.preprocessing.normalize(text_train[i])\n",
        "\n",
        "\n",
        "audio_test = np.nan_to_num(audio_test)\n",
        "video_test = np.nan_to_num(video_test)\n",
        "text_test = np.nan_to_num(text_test)\n",
        "\n",
        "\n",
        "for i in range(audio_test.shape[0]):\n",
        "  audio_test[i] = sklearn.preprocessing.normalize(audio_test[i])\n",
        "  video_test[i] = sklearn.preprocessing.normalize(video_test[i])\n",
        "  text_test[i] = sklearn.preprocessing.normalize(text_test[i])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "PatientID: 300\n",
            "PatientID: 301\n",
            "PatientID: 306\n",
            "PatientID: 308\n",
            "PatientID: 309\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/numpy/lib/function_base.py:393: RuntimeWarning: Mean of empty slice.\n",
            "  avg = a.mean(axis)\n",
            "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:154: RuntimeWarning: invalid value encountered in true_divide\n",
            "  ret, rcount, out=ret, casting='unsafe', subok=False)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "PatientID: 311\n",
            "PatientID: 314\n",
            "PatientID: 323\n",
            "PatientID: 329\n",
            "PatientID: 332\n",
            "PatientID: 334\n",
            "PatientID: 337\n",
            "PatientID: 349\n",
            "PatientID: 354\n",
            "PatientID: 359\n",
            "PatientID: 361\n",
            "PatientID: 365\n",
            "PatientID: 373\n",
            "PatientID: 378\n",
            "PatientID: 384\n",
            "PatientID: 387\n",
            "PatientID: 396\n",
            "Video Data corrupt, fixing.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:13: DtypeWarning: Columns (4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139) have mixed types.Specify dtype option on import or set low_memory=False.\n",
            "  del sys.path[0]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Video Data corrupt, fixing.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:13: DtypeWarning: Columns (4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207) have mixed types.Specify dtype option on import or set low_memory=False.\n",
            "  del sys.path[0]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Video Data corrupt, fixing.\n",
            "PatientID: 399\n",
            "PatientID: 405\n",
            "PatientID: 421\n",
            "PatientID: 424\n",
            "PatientID: 431\n",
            "PatientID: 432\n",
            "Video Data corrupt, fixing.\n",
            "Video Data corrupt, fixing.\n",
            "Video Data corrupt, fixing.\n",
            "PatientID: 435\n",
            "PatientID: 438\n",
            "PatientID: 442\n",
            "PatientID: 452\n",
            "PatientID: 461\n",
            "PatientID: 462\n",
            "PatientID: 465\n",
            "PatientID: 466\n",
            "PatientID: 467\n",
            "PatientID: 469\n",
            "PatientID: 470\n",
            "PatientID: 480\n",
            "PatientID: 481\n",
            "PatientID: 303\n",
            "PatientID: 304\n",
            "PatientID: 305\n",
            "PatientID: 310\n",
            "PatientID: 312\n",
            "PatientID: 313\n",
            "PatientID: 315\n",
            "PatientID: 316\n",
            "PatientID: 317\n",
            "PatientID: 318\n",
            "PatientID: 319\n",
            "PatientID: 320\n",
            "PatientID: 321\n",
            "PatientID: 322\n",
            "PatientID: 324\n",
            "PatientID: 325\n",
            "PatientID: 326\n",
            "PatientID: 327\n",
            "PatientID: 328\n",
            "PatientID: 330\n",
            "PatientID: 333\n",
            "PatientID: 336\n",
            "PatientID: 338\n",
            "PatientID: 339\n",
            "PatientID: 340\n",
            "PatientID: 341\n",
            "PatientID: 343\n",
            "PatientID: 344\n",
            "PatientID: 345\n",
            "PatientID: 347\n",
            "PatientID: 348\n",
            "PatientID: 350\n",
            "PatientID: 351\n",
            "PatientID: 352\n",
            "PatientID: 353\n",
            "PatientID: 355\n",
            "PatientID: 356\n",
            "PatientID: 357\n",
            "PatientID: 358\n",
            "PatientID: 360\n",
            "PatientID: 362\n",
            "PatientID: 363\n",
            "PatientID: 364\n",
            "PatientID: 366\n",
            "PatientID: 368\n",
            "PatientID: 369\n",
            "PatientID: 370\n",
            "PatientID: 371\n",
            "PatientID: 372\n",
            "PatientID: 374\n",
            "PatientID: 375\n",
            "PatientID: 376\n",
            "PatientID: 379\n",
            "PatientID: 380\n",
            "PatientID: 383\n",
            "PatientID: 385\n",
            "PatientID: 386\n",
            "PatientID: 391\n",
            "PatientID: 392\n",
            "PatientID: 393\n",
            "PatientID: 397\n",
            "PatientID: 400\n",
            "PatientID: 401\n",
            "PatientID: 402\n",
            "PatientID: 416\n",
            "PatientID: 419\n",
            "PatientID: 423\n",
            "PatientID: 425\n",
            "PatientID: 426\n",
            "PatientID: 427\n",
            "PatientID: 428\n",
            "PatientID: 429\n",
            "PatientID: 430\n",
            "PatientID: 433\n",
            "PatientID: 434\n",
            "PatientID: 437\n",
            "PatientID: 441\n",
            "PatientID: 443\n",
            "PatientID: 444\n",
            "PatientID: 445\n",
            "PatientID: 446\n",
            "PatientID: 447\n",
            "PatientID: 448\n",
            "PatientID: 449\n",
            "PatientID: 454\n",
            "PatientID: 455\n",
            "PatientID: 456\n",
            "PatientID: 457\n",
            "PatientID: 459\n",
            "PatientID: 463\n",
            "PatientID: 464\n",
            "PatientID: 468\n",
            "PatientID: 471\n",
            "PatientID: 473\n",
            "PatientID: 474\n",
            "PatientID: 475\n",
            "PatientID: 478\n",
            "PatientID: 479\n",
            "PatientID: 485\n",
            "PatientID: 486\n",
            "PatientID: 487\n",
            "PatientID: 488\n",
            "PatientID: 491\n",
            "(41, 1700, 74) (41, 1700, 388) (41, 1700, 301)\n",
            "(103, 1700, 74) (103, 1700, 388) (103, 1700, 301)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xk_9D5mWmTB7",
        "colab_type": "code",
        "outputId": "c66e1ee0-4cd5-4ec0-d91c-66abbc48448d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "#Word Level\n",
        "\n",
        "\n",
        "#Text+Audio+Video\n",
        "import tensorflow as tf\n",
        "\n",
        "from tensorflow import keras\n",
        "\n",
        "from tensorflow.keras.layers import LSTM, Dense\n",
        "from tensorflow.keras import layers\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "class Highway(layers.Layer):\n",
        "\n",
        "  def __init__(self):\n",
        "    super(Highway, self).__init__()\n",
        "\n",
        "  def build(self, input_shape):\n",
        "    n_sentences = input_shape[1]\n",
        "    n_features = input_shape[2]\n",
        "    carry_bias = keras.initializers.Constant(value=-1.0)\n",
        "    random_dist = keras.initializers.RandomNormal(mean=0.0, stddev=0.1)\n",
        "\n",
        "    carry_bias_2 = keras.initializers.Constant(value= 0.1)\n",
        "\n",
        "    self.W_T = self.add_weight(shape=(n_features, n_features),initializer = random_dist,trainable=True)\n",
        "    self.b_T = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias, trainable=True)\n",
        "    # self.W_T = tf.Variable(tf.random.truncated_normal([n_data_points,n_features, n_features], stddev=0.1),trainable=True)\n",
        "    # self.b_T = tf.Variable(tf.constant(-1.0, shape=[n_data_points, n_sentences, n_features]),trainable=True)\n",
        "   \n",
        "    self.W = self.add_weight(shape=( n_features, n_features),initializer = random_dist, trainable=True)\n",
        "    self.b = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias_2, trainable=True)\n",
        "    # self.W = tf.Variable(tf.random.truncated_normal([n_data_points,n_features, n_features], stddev=0.1), trainable=True)\n",
        "    # self.b = tf.Variable(tf.constant(0.1, shape=[n_data_points, n_sentences, n_features]), trainable=True)  \n",
        "\n",
        "\n",
        "  def call(self, inputs):\n",
        "    x = inputs\n",
        "    T = tf.sigmoid(tf.matmul(x, self.W_T) + self.b_T, name=\"transform_gate\")\n",
        "    H = tf.nn.relu(tf.matmul(x, self.W) + self.b, name=\"activation\")\n",
        "    C = tf.subtract(1.0, T, name=\"carry_gate\")\n",
        "    \n",
        "    return tf.add(tf.multiply(H, T), tf.multiply(x, C), \"y\")\n",
        "\n",
        "\n",
        "# Multiple Inputs\n",
        "from tensorflow.keras.utils import plot_model\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.layers import Input\n",
        "from tensorflow.keras.layers import Dense\n",
        "from tensorflow.keras.layers import concatenate\n",
        "\n",
        "# first input model\n",
        "input1 = Input(shape=(1700,74), name = 'Audio_input')\n",
        "highway1 = Highway()(input1)\n",
        "highway5 = Highway()(highway1)\n",
        "highway6 = Highway()(highway5)\n",
        "dense1 = Dense(74)(highway6)\n",
        "\n",
        "# second input model\n",
        "input2 = Input(shape=(1700,388), name = 'Video_input')\n",
        "highway2 = Highway()(input2)\n",
        "highway3 = Highway()(highway2)\n",
        "highway4 = Highway()(highway3)\n",
        "# dense8 = Dense(1000)(highway4)\n",
        "# dense9 = Dense(500)(dense8)\n",
        "dense7 = Dense(200)(highway4)\n",
        "dense2 = Dense(74)(dense7) \n",
        "\n",
        "input3 = Input(shape = (1700,301), name = 'Text_input')\n",
        "# dense4 = Dense(1000)(input3)\n",
        "# dense5 = Dense(500)(dense4)\n",
        "dense6 = Dense(150)(input3)\n",
        "dense3 = Dense(74)(dense6)\n",
        "# merge input models\n",
        "merge = concatenate([dense1,dense2,dense3], axis = 1)\n",
        "# interpretation model\n",
        "lstm = LSTM(128, dropout = 0.2, recurrent_dropout = 0.2)(merge)\n",
        "output = Dense(1, activation='sigmoid')(lstm)\n",
        "model = Model(inputs=[input1, input2, input3], outputs=output)\n",
        "# summarize layers\n",
        "print(model.summary())\n",
        "# plot graph\n",
        "plot_model(model)\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate= 0.0001)\n",
        "model.compile(optimizer=optimizer, loss='binary_crossentropy')\n",
        "\n",
        "# inp = np.array([audio_train,video_train,text_train], dtype =\n",
        "model.fit([audio_train,video_train,text_train],Ytrain, validation_split = 0.2, callbacks = tf.keras.callbacks.EarlyStopping(\n",
        "    monitor='val_loss', min_delta=0, patience=15, verbose=0, mode='min',\n",
        "    baseline=None, restore_best_weights=True),epochs=50, batch_size = 2)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "def Thresholding(Y_pred, threshold):\n",
        "  Y_pred2 = []\n",
        "  print(\"Y_pred: \", Y_pred.shape)\n",
        "  for i in range(len(Y_pred)):\n",
        "    if(Y_pred[i] < threshold):\n",
        "      Y_pred2.append(0)\n",
        "    else:\n",
        "      Y_pred2.append(1)\n",
        "\n",
        "  return np.array(Y_pred2)\n",
        "\n",
        "\n",
        "from sklearn.metrics import classification_report\n",
        "pred = model.predict([audio_test,video_test,text_test])\n",
        "pred2 = model.predict([audio_train,video_train,text_train])\n",
        "\n",
        "\n",
        "print(classification_report(Ytest,Thresholding(pred,0.5)))\n",
        "print(\"TRAINING ACCCCCC        \" , classification_report(Ytrain,Thresholding(pred2,0.5)))\n",
        "\n",
        "print(classification_report(Ytest,Thresholding(pred,0.6)))\n",
        "\n",
        "print(classification_report(Ytest,Thresholding(pred,0.4)))\n",
        "print(classification_report(Ytest,Thresholding(pred,0.3)))\n",
        "print(classification_report(Ytest,Thresholding(pred,0.7)))\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "Video_input (InputLayer)        [(None, 1700, 388)]  0                                            \n",
            "__________________________________________________________________________________________________\n",
            "Audio_input (InputLayer)        [(None, 1700, 74)]   0                                            \n",
            "__________________________________________________________________________________________________\n",
            "highway_9 (Highway)             (None, 1700, 388)    1620288     Video_input[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "highway_6 (Highway)             (None, 1700, 74)     262552      Audio_input[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "highway_10 (Highway)            (None, 1700, 388)    1620288     highway_9[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "highway_7 (Highway)             (None, 1700, 74)     262552      highway_6[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "highway_11 (Highway)            (None, 1700, 388)    1620288     highway_10[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "Text_input (InputLayer)         [(None, 1700, 301)]  0                                            \n",
            "__________________________________________________________________________________________________\n",
            "highway_8 (Highway)             (None, 1700, 74)     262552      highway_7[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "dense_7 (Dense)                 (None, 1700, 200)    77800       highway_11[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "dense_9 (Dense)                 (None, 1700, 150)    45300       Text_input[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "dense_6 (Dense)                 (None, 1700, 74)     5550        highway_8[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "dense_8 (Dense)                 (None, 1700, 74)     14874       dense_7[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_10 (Dense)                (None, 1700, 74)     11174       dense_9[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_1 (Concatenate)     (None, 5100, 74)     0           dense_6[0][0]                    \n",
            "                                                                 dense_8[0][0]                    \n",
            "                                                                 dense_10[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "lstm_1 (LSTM)                   (None, 128)          103936      concatenate_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "dense_11 (Dense)                (None, 1)            129         lstm_1[0][0]                     \n",
            "==================================================================================================\n",
            "Total params: 5,907,283\n",
            "Trainable params: 5,907,283\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n",
            "None\n",
            "Epoch 1/50\n",
            "60/60 [==============================] - 417s 7s/step - loss: 0.7000 - val_loss: 0.7365\n",
            "Epoch 2/50\n",
            "60/60 [==============================] - 427s 7s/step - loss: 0.6518 - val_loss: 0.6888\n",
            "Epoch 3/50\n",
            "60/60 [==============================] - 426s 7s/step - loss: 0.6147 - val_loss: 0.8022\n",
            "Epoch 4/50\n",
            "60/60 [==============================] - 427s 7s/step - loss: 0.5837 - val_loss: 0.6333\n",
            "Epoch 5/50\n",
            "60/60 [==============================] - 424s 7s/step - loss: 0.5642 - val_loss: 0.7487\n",
            "Epoch 6/50\n",
            "60/60 [==============================] - 422s 7s/step - loss: 0.5467 - val_loss: 0.7203\n",
            "Epoch 7/50\n",
            "60/60 [==============================] - 423s 7s/step - loss: 0.5167 - val_loss: 0.7540\n",
            "Epoch 8/50\n",
            "60/60 [==============================] - 425s 7s/step - loss: 0.4945 - val_loss: 0.6486\n",
            "Epoch 9/50\n",
            "60/60 [==============================] - 430s 7s/step - loss: 0.4810 - val_loss: 0.6803\n",
            "Epoch 10/50\n",
            "60/60 [==============================] - 430s 7s/step - loss: 0.4779 - val_loss: 0.6791\n",
            "Epoch 11/50\n",
            "60/60 [==============================] - 431s 7s/step - loss: 0.4764 - val_loss: 0.6881\n",
            "Epoch 12/50\n",
            "60/60 [==============================] - 428s 7s/step - loss: 0.4755 - val_loss: 0.6890\n",
            "Epoch 13/50\n",
            "60/60 [==============================] - 430s 7s/step - loss: 0.4677 - val_loss: 0.7247\n",
            "Epoch 14/50\n",
            "60/60 [==============================] - 430s 7s/step - loss: 0.4620 - val_loss: 0.8504\n",
            "Epoch 15/50\n",
            "60/60 [==============================] - 437s 7s/step - loss: 0.4654 - val_loss: 0.7805\n",
            "Epoch 16/50\n",
            "60/60 [==============================] - 436s 7s/step - loss: 0.4691 - val_loss: 0.7328\n",
            "Epoch 17/50\n",
            "60/60 [==============================] - 437s 7s/step - loss: 0.4657 - val_loss: 0.7369\n",
            "Epoch 18/50\n",
            "60/60 [==============================] - 436s 7s/step - loss: 0.4696 - val_loss: 0.7767\n",
            "Epoch 19/50\n",
            "60/60 [==============================] - 436s 7s/step - loss: 0.4610 - val_loss: 0.8022\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.68      0.86      0.76        29\n",
            "         1.0       0.00      0.00      0.00        12\n",
            "\n",
            "    accuracy                           0.61        41\n",
            "   macro avg       0.34      0.43      0.38        41\n",
            "weighted avg       0.48      0.61      0.54        41\n",
            "\n",
            "Y_pred:  (150, 1)\n",
            "TRAINING ACCCCCC                       precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.58      0.89      0.70        75\n",
            "         1.0       0.76      0.35      0.48        75\n",
            "\n",
            "    accuracy                           0.62       150\n",
            "   macro avg       0.67      0.62      0.59       150\n",
            "weighted avg       0.67      0.62      0.59       150\n",
            "\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.69      0.93      0.79        29\n",
            "         1.0       0.00      0.00      0.00        12\n",
            "\n",
            "    accuracy                           0.66        41\n",
            "   macro avg       0.35      0.47      0.40        41\n",
            "weighted avg       0.49      0.66      0.56        41\n",
            "\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       1.00      0.10      0.19        29\n",
            "         1.0       0.32      1.00      0.48        12\n",
            "\n",
            "    accuracy                           0.37        41\n",
            "   macro avg       0.66      0.55      0.33        41\n",
            "weighted avg       0.80      0.37      0.27        41\n",
            "\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       1.00      0.07      0.13        29\n",
            "         1.0       0.31      1.00      0.47        12\n",
            "\n",
            "    accuracy                           0.34        41\n",
            "   macro avg       0.65      0.53      0.30        41\n",
            "weighted avg       0.80      0.34      0.23        41\n",
            "\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.69      0.93      0.79        29\n",
            "         1.0       0.00      0.00      0.00        12\n",
            "\n",
            "    accuracy                           0.66        41\n",
            "   macro avg       0.35      0.47      0.40        41\n",
            "weighted avg       0.49      0.66      0.56        41\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}