LSTM_With_Gating_Sentence_Level.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "LSTM_With_Gating_Sentence_Level.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/notmanan/Depression-Detection-Through-Multi-Modal-Data/blob/master/LSTM_With_Gating_Sentence_Level.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GJLN_Guyi9E8",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KJyWRMcLCjbB",
        "colab_type": "code",
        "outputId": "06f332b7-18c9-45c5-d524-8731cc9c552b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive', force_remount=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7pclNO84uR_0",
        "colab_type": "code",
        "outputId": "f413c29c-f2bc-4408-b536-f59fd7f35e00",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "import gc\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "import math\n",
        "\n",
        "from smart_open import open\n",
        "from nltk.corpus import stopwords\n",
        "import sklearn\n",
        "from sklearn.metrics import classification_report\n",
        "from keras.layers import Dropout\n",
        "from matplotlib import pyplot as plt\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from gensim.models.keyedvectors import KeyedVectors\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "from tensorflow import keras\n",
        "\n",
        "from tensorflow.keras.layers import LSTM, Dense, Bidirectional\n",
        "from tensorflow.keras import layers\n",
        "\n",
        "\n",
        "from tensorflow.keras.utils import plot_model\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.layers import Input\n",
        "from tensorflow.keras.layers import concatenate\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gzWbjnf8EruV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Text + Audio \n",
        "from tensorflow.keras import layers\n",
        "\n",
        "class Highway(layers.Layer):\n",
        "\n",
        "  def __init__(self):\n",
        "    super(Highway, self).__init__()\n",
        "\n",
        "  def build(self, input_shape):\n",
        "    n_sentences = input_shape[1]\n",
        "    n_features = input_shape[2]\n",
        "    carry_bias = keras.initializers.Constant(value=-1.0)\n",
        "    random_dist = keras.initializers.RandomNormal(mean=0.0, stddev=0.1)\n",
        "\n",
        "    carry_bias_2 = keras.initializers.Constant(value= 0.1)\n",
        "\n",
        "    self.W_T = self.add_weight(shape=(n_features, n_features),initializer = random_dist,trainable=True)\n",
        "    self.b_T = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias, trainable=True)\n",
        "    # self.W_T = tf.Variable(tf.random.truncated_normal([n_data_points,n_features, n_features], stddev=0.1),trainable=True)\n",
        "    # self.b_T = tf.Variable(tf.constant(-1.0, shape=[n_data_points, n_sentences, n_features]),trainable=True)\n",
        "   \n",
        "    self.W = self.add_weight(shape=( n_features, n_features),initializer = random_dist, trainable=True)\n",
        "    self.b = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias_2, trainable=True)\n",
        "    # self.W = tf.Variable(tf.random.truncated_normal([n_data_points,n_features, n_features], stddev=0.1), trainable=True)\n",
        "    # self.b = tf.Variable(tf.constant(0.1, shape=[n_data_points, n_sentences, n_features]), trainable=True)  \n",
        "\n",
        "\n",
        "  def call(self, inputs):\n",
        "    x = inputs\n",
        "    T = tf.sigmoid(tf.matmul(x, self.W_T) + self.b_T, name=\"transform_gate\")\n",
        "    H = tf.nn.relu(tf.matmul(x, self.W) + self.b, name=\"activation\")\n",
        "    C = tf.subtract(1.0, T, name=\"carry_gate\")\n",
        "    \n",
        "    return tf.add(tf.multiply(H, T), tf.multiply(x, C), \"y\")\n",
        "\n",
        "\n",
        "# Multiple Inputs\n",
        "class text_audio:\n",
        "  # first input model\n",
        "  input1 = Input(shape=(250,74), name = 'Audio_input')\n",
        "  highway1 = Highway()(input1)\n",
        "  highway5 = Highway()(highway1)\n",
        "  highway6 = Highway()(highway5)\n",
        "  dense1 = Dense(74)(highway6)\n",
        "\n",
        "  input3 = Input(shape = (250,5100), name = 'Text_input')\n",
        "  dense4 = Dense(1000)(input3)\n",
        "  dense5 = Dense(500)(dense4)\n",
        "  dense6 = Dense(250)(dense5)\n",
        "  dense3 = Dense(74)(dense6)\n",
        "  # merge input models\n",
        "  merge = concatenate([dense1,dense3], axis = 1)\n",
        "  # interpretation model\n",
        "  lstm = LSTM(128, dropout = 0.2, recurrent_dropout = 0.2)(merge)\n",
        "  output = Dense(1, activation='sigmoid')(lstm)\n",
        "  model = Model(inputs=[input1, input3], outputs=output)\n",
        "  # summarize layers\n",
        "  # print(model.summary())\n",
        "  # plot graph\n",
        "  # plot_model(model)\n",
        "  optimizer = tf.keras.optimizers.Adam(learning_rate= 0.0001)\n",
        "  def run_model(self):\n",
        "    self.model.compile(optimizer=self.optimizer, loss='binary_crossentropy')\n",
        "\n",
        "    return self.model\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fsHsKkisFMM7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Text+Video\n",
        "\n",
        "\n",
        "class Highway(layers.Layer):\n",
        "\n",
        "  def __init__(self):\n",
        "    super(Highway, self).__init__()\n",
        "\n",
        "  def build(self, input_shape):\n",
        "    n_sentences = input_shape[1]\n",
        "    n_features = input_shape[2]\n",
        "    carry_bias = keras.initializers.Constant(value=-1.0)\n",
        "    random_dist = keras.initializers.RandomNormal(mean=0.0, stddev=0.1)\n",
        "\n",
        "    carry_bias_2 = keras.initializers.Constant(value= 0.1)\n",
        "\n",
        "    self.W_T = self.add_weight(shape=(n_features, n_features),initializer = random_dist,trainable=True)\n",
        "    self.b_T = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias, trainable=True)\n",
        "   \n",
        "    self.W = self.add_weight(shape=( n_features, n_features),initializer = random_dist, trainable=True)\n",
        "    self.b = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias_2, trainable=True)\n",
        "\n",
        "\n",
        "  def call(self, inputs):\n",
        "    x = inputs\n",
        "    T = tf.sigmoid(tf.matmul(x, self.W_T) + self.b_T, name=\"transform_gate\")\n",
        "    H = tf.nn.relu(tf.matmul(x, self.W) + self.b, name=\"activation\")\n",
        "    C = tf.subtract(1.0, T, name=\"carry_gate\")\n",
        "    \n",
        "    return tf.add(tf.multiply(H, T), tf.multiply(x, C), \"y\")\n",
        "\n",
        "class text_video:\n",
        "  # second input model\n",
        "  input2 = Input(shape=(250,388), name = 'Video_input')\n",
        "  highway2 = Highway()(input2)\n",
        "  highway3 = Highway()(highway2)\n",
        "  highway4 = Highway()(highway3)\n",
        "  # dense7 = Dense(200)(highway4)\n",
        "  dense2 = Dense(250)(highway4)\n",
        "\n",
        "  input3 = Input(shape = (250,5100), name = 'Text_input')\n",
        "  dense4 = Dense(1000)(input3)\n",
        "  dense5 = Dense(500)(dense4)\n",
        "  # dense6 = Dense(250)(dense5)\n",
        "  dense3 = Dense(250)(dense5)\n",
        "  # merge input models\n",
        "  merge = concatenate([dense2,dense3], axis = 1)\n",
        "  # interpretation model\n",
        "  lstm = LSTM(128, dropout = 0.2, recurrent_dropout = 0.2)(merge)\n",
        "  output = Dense(1, activation='sigmoid')(lstm)\n",
        "  model = Model(inputs=[input2, input3], outputs=output)\n",
        "  # summarize layers\n",
        "  # print(model.summary())\n",
        "  # # plot graph\n",
        "  # plot_model(model)\n",
        "  def run_model(self):\n",
        "    optimizer = tf.keras.optimizers.Adam(learning_rate= 0.0001)\n",
        "    self.model.compile(optimizer=optimizer, loss='binary_crossentropy')\n",
        "    return self.model\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PNelM26pDEiN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Text+Audio+Video\n",
        "\n",
        "class Highway(layers.Layer):\n",
        "\n",
        "  def __init__(self):\n",
        "    super(Highway, self).__init__()\n",
        "\n",
        "  def build(self, input_shape):\n",
        "    n_sentences = input_shape[1]\n",
        "    n_features = input_shape[2]\n",
        "    carry_bias = keras.initializers.Constant(value=-1.0)\n",
        "    random_dist = keras.initializers.RandomNormal(mean=0.0, stddev=0.1)\n",
        "\n",
        "    carry_bias_2 = keras.initializers.Constant(value= 0.1)\n",
        "\n",
        "    self.W_T = self.add_weight(shape=(n_features, n_features),initializer = random_dist,trainable=True)\n",
        "    self.b_T = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias, trainable=True)\n",
        "   \n",
        "    self.W = self.add_weight(shape=( n_features, n_features),initializer = random_dist, trainable=True)\n",
        "    self.b = self.add_weight(shape=( n_sentences, n_features),initializer = carry_bias_2, trainable=True) \n",
        "\n",
        "\n",
        "  def call(self, inputs):\n",
        "    x = inputs\n",
        "    T = tf.sigmoid(tf.matmul(x, self.W_T) + self.b_T, name=\"transform_gate\")\n",
        "    H = tf.nn.relu(tf.matmul(x, self.W) + self.b, name=\"activation\")\n",
        "    C = tf.subtract(1.0, T, name=\"carry_gate\")\n",
        "    \n",
        "    return tf.add(tf.multiply(H, T), tf.multiply(x, C), \"y\")\n",
        "\n",
        "class text_vieo_audio:\n",
        "  # first input model\n",
        "  input1 = Input(shape=(250,74), name = 'Audio_input')\n",
        "  highway1 = Highway()(input1)\n",
        "  highway5 = Highway()(highway1)\n",
        "  highway6 = Highway()(highway5)\n",
        "  dense1 = Dense(74)(highway6)\n",
        "\n",
        "  # second input model\n",
        "  input2 = Input(shape=(250,388), name = 'Video_input')\n",
        "  highway2 = Highway()(input2)\n",
        "  highway3 = Highway()(highway2)\n",
        "  highway4 = Highway()(highway3)\n",
        "  dense7 = Dense(200)(highway4)\n",
        "  dense2 = Dense(74)(dense7)\n",
        "\n",
        "  input3 = Input(shape = (250,5100), name = 'Text_input')\n",
        "  dense4 = Dense(1000)(input3)\n",
        "  dense5 = Dense(500)(dense4)\n",
        "  dense6 = Dense(250)(dense5)\n",
        "  dense3 = Dense(74)(dense6)\n",
        "  # merge input models\n",
        "  merge = concatenate([dense1,dense2,dense3], axis = 1)\n",
        "  lstm = LSTM(128, dropout = 0.2, recurrent_dropout = 0.2)(merge)\n",
        "  # bidire = Bidirectional(lstm)\n",
        "  output = Dense(1, activation='sigmoid')(lstm)\n",
        "  model = Model(inputs=[input1, input2, input3], outputs=output)\n",
        "  def run_model(self):\n",
        "\n",
        "    optimizer = tf.keras.optimizers.Adam(learning_rate= 0.0001)\n",
        "    self.model.compile(optimizer=optimizer, loss='binary_crossentropy')\n",
        "    return self.model\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gHCKLXfLx3Kc",
        "colab_type": "code",
        "outputId": "6c90ec02-e8ec-4a64-e245-5622b5fbdba8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 297
        }
      },
      "source": [
        "dev_location = \"dev_data\"\n",
        "test_location = \"test_data\"\n",
        "train_location = \"train_data\"\n",
        "\n",
        "devData = np.array(pd.read_csv('/content/drive/My Drive/MCA Dataset/dev_split_Depression_AVEC2017.csv',delimiter=',',encoding='utf-8'))[:, 0:2]\n",
        "testData = np.array(pd.read_csv('/content/drive/My Drive/MCA Dataset/full_test_split.csv',delimiter=',',encoding='utf-8'))[:, 0:2]\n",
        "trainData = np.array(pd.read_csv('/content/drive/My Drive/MCA Dataset/train_split_Depression_AVEC2017.csv',delimiter=',',encoding='utf-8'))[:, 0:2]\n",
        "\n",
        "\n",
        "dataset = np.concatenate((devData, np.concatenate((testData, trainData))))\n",
        "\n",
        "gc.collect()      \n",
        "max_num_words = 17\n",
        "model = KeyedVectors.load_word2vec_format('/content/drive/My Drive/MCA Dataset/Copy of GoogleNews-vectors-negative300.bin', binary=True)\n",
        "stop_words = set(stopwords.words('english'))\n",
        "\n",
        "\n",
        "\n",
        "def checkDataPointExistence(patientID, split):\n",
        "  for i in split:\n",
        "    if(patientID == i[0]):\n",
        "      return True\n",
        "  return False\n",
        "\n",
        "def getData(patientID, location):\n",
        "  # print(\"PatientID: \" + str(int(patientID)))\n",
        "  retData = [int(patientID)]\n",
        "  textD = getTextData(patientID, location)\n",
        "  audioD = getAudioData(patientID, location, textD)\n",
        "  videoD = getVideoData(str(int(patientID)), location, textD)\n",
        "  # patientD = np.concatenate((textD, audioD, videoD), axis = 1)\n",
        "  # print(\"Final Patient Data: \" + str(patientD.shape))\n",
        "  return textD,audioD,videoD\n",
        "\n",
        "def getTextData(patientID, location):\n",
        "  fileName = \"/content/drive/My Drive/MCA Dataset/\"+ str(location) + \"/\" + str(int(patientID)) + \"_TRANSCRIPT.csv\"\n",
        "  file = np.array(pd.read_csv(fileName,delimiter='\\t',encoding='utf-8', engine='python'))\n",
        "\n",
        "  # Remove All Utterences By Ellie:\n",
        "  for i in range(len(file)):\n",
        "    if(file[i][2] != 'Participant'):\n",
        "      np.delete(file, i)\n",
        "      i-=1\n",
        "\n",
        "  # Remove Speaker Columnn\n",
        "  file = np.delete(file, 2, 1)\n",
        "  \n",
        "  # Convert Text Into Word Vectors:\n",
        "  w2vs = np.zeros((1, max_num_words*300))\n",
        "  for i in range(len(file)):\n",
        "    sentence = file[i][2]\n",
        "    w2v = returnWordToVec(sentence)\n",
        "    w2vs = np.concatenate((w2vs, w2v), axis = 0)\n",
        "  w2vs = np.delete(w2vs, 0, 0)  \n",
        "\n",
        "  # Delete Sentences and Replace With W2Vs\n",
        "  file = np.delete(file, 2, 1)\n",
        "  file = np.concatenate((file, w2vs), axis = 1)\n",
        "  return file\n",
        "\n",
        "def remove_StopWords(sentence):\n",
        "    filtered_sentence = [] \n",
        "    for w in sentence: \n",
        "        if w not in stop_words: \n",
        "            filtered_sentence.append(w)\n",
        "    return filtered_sentence\n",
        "\n",
        "def returnWordToVec(sentence):\n",
        "  global max_num_words, stop_words, model\n",
        "  sentence = str(sentence).split(\" \")\n",
        "  sentence = remove_StopWords(sentence)\n",
        "  index_word = 0\n",
        "  wordMatrix = np.zeros(max_num_words*300)\n",
        "  for j in range(min(max_num_words, len(sentence))):\n",
        "    try:\n",
        "      word = sentence[j]\n",
        "      if(word[0] == '<'):\n",
        "        if(word.find('>')!=-1):\n",
        "          word = word[1:-1]\n",
        "        else:\n",
        "          word = word[1:]\n",
        "      else:\n",
        "        if(word.find('>')!=-1):\n",
        "          word = word[0:-1]\n",
        "      ss = np.array(model[word])\n",
        "      wordMatrix[index_word*300:(index_word+1)*300] = ss\n",
        "      index_word+=1\n",
        "    except Exception as e:\n",
        "      continue\n",
        "  wordMatrix = np.array(wordMatrix.reshape(1,-1))\n",
        "  return wordMatrix\n",
        "\n",
        "def audioDataHelper(X):\n",
        "    for i in range(X.shape[0]):\n",
        "        if(X[i,1] == 0):\n",
        "            X[i,0] = 0\n",
        "            for j in range(7):\n",
        "                X[i,j+1] = 0\n",
        "    X = np.array(X)\n",
        "    return X\n",
        "    \n",
        "def getAudioData(patientID, location, textD):\n",
        "  fileName = \"/content/drive/My Drive/MCA Dataset/\"+ str(location) + \"/\" + str(int(patientID)) + \"_COVAREP.csv\"\n",
        "  data = pd.read_csv(fileName,header = None)\n",
        "  data = data.iloc[:,:].values\n",
        "  data = audioDataHelper(data)\n",
        "  # print(\"Audio Raw Data:\" + str(data.shape))\n",
        "  sentenceDatas = []\n",
        "  for sentence in textD:\n",
        "    sentenceStartime = sentence[0]\n",
        "    sentenceEndTime = sentence[1]\n",
        "    startIndex = math.floor(sentenceStartime/0.01)\n",
        "    endIndex = math.ceil(sentenceEndTime/0.01)\n",
        "    # print(\"Start Time: \" + str(startIndex))\n",
        "    # print(\"End Time: \" + str(endIndex))\n",
        "    sentenceData = data[startIndex: endIndex]\n",
        "    sentenceData = np.average(sentenceData, axis = 0)\n",
        "    # This might be a possible error\n",
        "    sentenceData = np.array(sentenceData.reshape(1, -1))\n",
        "    sentenceDatas.append(sentenceData)\n",
        "  \n",
        "  sentenceDatas = np.array(sentenceDatas)\n",
        "  sentenceDatas = np.reshape(sentenceDatas, (textD.shape[0],-1))\n",
        "  # print(\"Audio Final Data:\" + str(sentenceDatas.shape))\n",
        "\n",
        "  return sentenceDatas\n",
        "\n",
        "def getVideoDataHelper(patientID, location):\n",
        "  root = \"/content/drive/My Drive/MCA Dataset/\"+ str(location) + \"/\" \n",
        "  file1 = root + (patientID)+\"_CLNF_AUs.txt\"\n",
        "  file2 = root + (patientID)+\"_CLNF_features.txt\"\n",
        "  file3 = root + (patientID)+\"_CLNF_features3D.txt\"\n",
        "  file4 = root + (patientID)+\"_CLNF_gaze.txt\"\n",
        "  file5 = root + (patientID)+\"_CLNF_hog.txt\"\n",
        "  file6 = root + (patientID)+\"_CLNF_pose.txt\"\n",
        "  data = processVideoData(file1)\n",
        "  data = np.concatenate((data, processVideoData(file2)), 1)\n",
        "  data = np.concatenate((data, processVideoData(file3)), 1)\n",
        "  data = np.concatenate((data, processVideoData(file4)), 1)\n",
        "  data = np.concatenate((data, processVideoData(file6)), 1)\n",
        "  # print(\"Video Raw Data:\" + str(data.shape))\n",
        "  return data\n",
        "\n",
        "def processVideoData(filename):\n",
        "  try:\n",
        "    data = pd.read_csv(filename,delimiter=',', dtype=float)\n",
        "    X = data.iloc[:,:].values\n",
        "    X = np.delete(X, 0, 1)\n",
        "    X = np.delete(X, 1, 1)\n",
        "  except:\n",
        "    # print(\"Video Data corrupt, fixing.\")\n",
        "    data = pd.read_csv(filename,delimiter=',')\n",
        "    X = data.iloc[:,:].values\n",
        "    X = np.delete(X, 0, 1)\n",
        "    X = np.delete(X, 1, 1)\n",
        "    for i in range(len(X)):\n",
        "        if(isinstance(X[i][5],str) or isinstance(X[i][7],str)):\n",
        "            X[i] = np.zeros((1, X.shape[1]))\n",
        "            # print(\"se\" , end = \" \")    \n",
        "  return X\n",
        "\n",
        "def getVideoData(patientID, location, textD):\n",
        "  data = getVideoDataHelper(patientID, location)\n",
        "  sentenceDatas = []\n",
        "  for sentence in textD:\n",
        "    sentenceStartime = sentence[0]\n",
        "    sentenceEndTime = sentence[1]\n",
        "    startIndex = math.floor(sentenceStartime/0.333)\n",
        "    endIndex = math.ceil(sentenceEndTime/0.333)\n",
        "    # print(\"Start Time: \" + str(startIndex))\n",
        "    sentenceData = data[startIndex: endIndex]\n",
        "    sentenceData = np.average(sentenceData, axis = 0)\n",
        "    # This might be a possible error\n",
        "    sentenceData = np.array(sentenceData.reshape(1, -1))\n",
        "    sentenceDatas.append(sentenceData)\n",
        "  \n",
        "  sentenceDatas = np.array(sentenceDatas)\n",
        "  sentenceDatas = np.reshape(sentenceDatas, (textD.shape[0],-1))\n",
        "  # print(\"Video Final Data:\" + str(sentenceDatas.shape))\n",
        "  return sentenceDatas\n",
        "\n",
        "# Xtrain = []\n",
        "Ytrain = []\n",
        "# Xtest = []\n",
        "Ytest = []\n",
        "\n",
        "\n",
        "audio_train = []\n",
        "video_train = []\n",
        "text_train = []\n",
        "\n",
        "audio_test = []\n",
        "video_test = []\n",
        "text_test = []\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "for datapoint in dataset:\n",
        "  # print(datapoint[0])\n",
        "  if(checkDataPointExistence(datapoint[0], devData)):\n",
        "\n",
        "    # Data Point in Dev Set\n",
        "    text,audio,video = getData(datapoint[0], dev_location)\n",
        "    audio_train.append(audio)\n",
        "    video_train.append(video)\n",
        "    text_train.append(text)\n",
        "    # Xtest.append(data)\n",
        "    Ytrain.append(datapoint[1])\n",
        "    # print(data)\n",
        "  elif(checkDataPointExistence(datapoint[0], testData)):\n",
        "    # Data Point in Test Set\n",
        "    text,audio,video = getData(datapoint[0], test_location)\n",
        "    audio_test.append(audio)\n",
        "    video_test.append(video)\n",
        "    text_test.append(text)\n",
        "    # Xtest.append(data)\n",
        "    Ytest.append(datapoint[1])\n",
        "  elif(checkDataPointExistence(datapoint[0], trainData)):\n",
        "    # Data Point in Train Set\n",
        "    text,audio,video = getData(datapoint[0], train_location)\n",
        "    audio_train.append(audio)\n",
        "    video_train.append(video)\n",
        "    text_train.append(text)\n",
        "    # Xtest.append(data)\n",
        "    Ytrain.append(datapoint[1])\n",
        "\n",
        "def refactor(arr, size):\n",
        "  arrsize = arr.shape[0]\n",
        "  temp = np.zeros((size, arr.shape[1]))\n",
        "  for i in range(min(len(arr), size)):\n",
        "    temp[i] = arr[i]\n",
        "  return temp\n",
        "\n",
        "numberOfSentences = 250\n",
        "\n",
        "devData = []\n",
        "trainData = []\n",
        "testData = []\n",
        "gc.collect()\n",
        "\n",
        "for i in range(len(audio_train)):\n",
        "  audio_train[i] = refactor(audio_train[i], numberOfSentences)\n",
        "  video_train[i] = refactor(video_train[i], numberOfSentences)\n",
        "  text_train[i] = refactor(text_train[i], numberOfSentences)\n",
        "  # print(Xtrain[i].shape)\n",
        "\n",
        "\n",
        "for i in range(len(audio_test)):\n",
        "  audio_test[i] = refactor(audio_train[i], numberOfSentences)\n",
        "  video_test[i] = refactor(video_train[i], numberOfSentences)\n",
        "  text_test[i] = refactor(text_train[i], numberOfSentences)\n",
        "  # print(Xtest[i].shape)\n",
        "audio_test = np.array(audio_test)\n",
        "video_test = np.array(video_test)\n",
        "text_test = np.array(text_test)\n",
        "text_test = text_test[:,:,2:]\n",
        "\n",
        "audio_train = np.array(audio_train)\n",
        "video_train = np.array(video_train)\n",
        "text_train = np.array(text_train)\n",
        "text_train = text_train[:,:,2:]\n",
        "\n",
        "\n",
        "dataset = []\n",
        "gc.collect()\n",
        "\n",
        "print(audio_test.shape,video_test.shape,text_test.shape)\n",
        "print(audio_train.shape,video_train.shape,text_train.shape)\n",
        "# print(audio_dev.shape,video_dev.shape,text_dev.shape)\n",
        "\n",
        "Ytrain = np.array(Ytrain)\n",
        "Ytest = np.array(Ytest)\n",
        "\n",
        "\n",
        "import sklearn\n",
        "from sklearn import preprocessing\n",
        "\n",
        "\n",
        "def upsample(X_train,Y_train):\n",
        "  X_train_0 = X_train[Y_train==0]\n",
        "  X_train_1 = X_train[Y_train==1]\n",
        "\n",
        "  Y_train_1 = Y_train[Y_train==1]\n",
        "  # print(Y_train_1.shape)\n",
        "  # print(X_train_1.shape)\n",
        "  size = X_train_0.shape[0] - X_train_1.shape[0]\n",
        "  X = []\n",
        "  Y = []\n",
        "  X_train = list(X_train)\n",
        "  Y_train = list(Y_train)\n",
        "  while(size>0):\n",
        "    size -= 1\n",
        "    index = np.random.randint(0,X_train_1.shape[0]-1)\n",
        "    leave_index = np.random.randint(0,len(X_train)-1)\n",
        "    X_add = X_train_1[index]\n",
        "    X_leave = X_train[leave_index]\n",
        "\n",
        "    Y_add = Y_train_1[index]\n",
        "    Y_leave = Y_train[leave_index]\n",
        "\n",
        "    X_train[leave_index] = X_add\n",
        "    X_train.append(X_leave)\n",
        "\n",
        "    Y_train[leave_index] = Y_add\n",
        "    Y_train.append(Y_leave)\n",
        "\n",
        "\n",
        "  X_train = np.array(X_train)\n",
        "  Y_train = np.array(Y_train)\n",
        "  return X_train,Y_train\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "audio_train = np.nan_to_num(audio_train)\n",
        "video_train = np.nan_to_num(video_train)\n",
        "text_train = np.nan_to_num(text_train)\n",
        "\n",
        "audio_train, _ = upsample(audio_train,Ytrain)\n",
        "video_train, _ = upsample(video_train,Ytrain)\n",
        "text_train, Ytrain = upsample(text_train,Ytrain)\n",
        "\n",
        "print(audio_train.shape)\n",
        "print(video_train.shape)\n",
        "print(text_train.shape)\n",
        "print(Ytrain.shape)\n",
        "\n",
        "for i in range(audio_train.shape[0]):\n",
        "  audio_train[i] = sklearn.preprocessing.normalize(audio_train[i])\n",
        "  video_train[i] = sklearn.preprocessing.normalize(video_train[i])\n",
        "  text_train[i] = sklearn.preprocessing.normalize(text_train[i])\n",
        "\n",
        "\n",
        "print(Ytest.shape)\n",
        "\n",
        "\n",
        "\n",
        "audio_test = np.nan_to_num(audio_test)\n",
        "video_test = np.nan_to_num(video_test)\n",
        "text_test = np.nan_to_num(text_test)\n",
        "\n",
        "\n",
        "for i in range(audio_test.shape[0]):\n",
        "  audio_test[i] = sklearn.preprocessing.normalize(audio_test[i])\n",
        "  video_test[i] = sklearn.preprocessing.normalize(video_test[i])\n",
        "  text_test[i] = sklearn.preprocessing.normalize(text_test[i])\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:253: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:30: DtypeWarning: Columns (4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139) have mixed types.Specify dtype option on import or set low_memory=False.\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:30: DtypeWarning: Columns (4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207) have mixed types.Specify dtype option on import or set low_memory=False.\n",
            "/usr/local/lib/python3.6/dist-packages/numpy/lib/function_base.py:393: RuntimeWarning: Mean of empty slice.\n",
            "  avg = a.mean(axis)\n",
            "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:154: RuntimeWarning: invalid value encountered in true_divide\n",
            "  ret, rcount, out=ret, casting='unsafe', subok=False)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "(41, 250, 74) (41, 250, 388) (41, 250, 5100)\n",
            "(137, 250, 74) (137, 250, 388) (137, 250, 5100)\n",
            "(196, 250, 74)\n",
            "(196, 250, 388)\n",
            "(196, 250, 5100)\n",
            "(196,)\n",
            "(41,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e1wTtgg4yOeK",
        "colab_type": "code",
        "outputId": "617d7828-282c-4c53-9750-566102a6fd54",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "def Thresholding(Y_pred, threshold):\n",
        "  Y_pred2 = []\n",
        "  print(\"Y_pred: \", Y_pred.shape)\n",
        "  for i in range(len(Y_pred)):\n",
        "    if(Y_pred[i] < threshold):\n",
        "      Y_pred2.append(0)\n",
        "    else:\n",
        "      Y_pred2.append(1)\n",
        "\n",
        "  return np.array(Y_pred2)\n",
        "\n",
        "gc.collect()\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "87"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QbT2jzhhu4Om",
        "colab_type": "code",
        "outputId": "c8167c52-836c-4822-baf2-3355ea041838",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "print(\"--------------------------------------AUDIO + TEXT (W GATING) SENTENCE LEVEL-------------------------------------------------\")\n",
        "model1 = text_audio()\n",
        "model = model1.run_model()\n",
        "\n",
        "model.fit([audio_train,text_train],Ytrain, validation_split = 0.2, callbacks = tf.keras.callbacks.EarlyStopping(\n",
        "        monitor='val_loss', min_delta=0, patience=15, verbose=0, mode='min',\n",
        "        baseline=None, restore_best_weights=True),epochs=50, batch_size = 137)\n",
        "\n",
        "\n",
        "from sklearn.metrics import classification_report\n",
        "pred = model.predict([audio_test,text_test])\n",
        "pred2 = model.predict([audio_train,text_train])\n",
        "\n",
        "print(classification_report(Ytest,Thresholding(pred,0.5)))\n",
        "print(\"TRAINING ACCCCCC        \" , classification_report(Ytrain,Thresholding(pred2,0.5)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.6)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.4)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.3)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.7)))\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--------------------------------------AUDIO + TEXT (W GATING) SENTENCE LEVEL-------------------------------------------------\n",
            "Epoch 1/50\n",
            "2/2 [==============================] - 3s 2s/step - loss: 0.6932 - val_loss: 0.6918\n",
            "Epoch 2/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.6905 - val_loss: 0.6919\n",
            "Epoch 3/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.6880 - val_loss: 0.6919\n",
            "Epoch 4/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.6850 - val_loss: 0.6919\n",
            "Epoch 5/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.6821 - val_loss: 0.6920\n",
            "Epoch 6/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.6785 - val_loss: 0.6913\n",
            "Epoch 7/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.6742 - val_loss: 0.6896\n",
            "Epoch 8/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6692 - val_loss: 0.6879\n",
            "Epoch 9/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.6631 - val_loss: 0.6861\n",
            "Epoch 10/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.6565 - val_loss: 0.6845\n",
            "Epoch 11/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.6491 - val_loss: 0.6819\n",
            "Epoch 12/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.6394 - val_loss: 0.6778\n",
            "Epoch 13/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.6281 - val_loss: 0.6730\n",
            "Epoch 14/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.6166 - val_loss: 0.6671\n",
            "Epoch 15/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.6029 - val_loss: 0.6591\n",
            "Epoch 16/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.5893 - val_loss: 0.6486\n",
            "Epoch 17/50\n",
            "2/2 [==============================] - 3s 2s/step - loss: 0.5734 - val_loss: 0.6365\n",
            "Epoch 18/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.5587 - val_loss: 0.6233\n",
            "Epoch 19/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.5416 - val_loss: 0.6093\n",
            "Epoch 20/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.5228 - val_loss: 0.5981\n",
            "Epoch 21/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.5064 - val_loss: 0.5887\n",
            "Epoch 22/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.4878 - val_loss: 0.5799\n",
            "Epoch 23/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.4719 - val_loss: 0.5715\n",
            "Epoch 24/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.4581 - val_loss: 0.5604\n",
            "Epoch 25/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.4465 - val_loss: 0.5493\n",
            "Epoch 26/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.4314 - val_loss: 0.5474\n",
            "Epoch 27/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.4250 - val_loss: 0.5368\n",
            "Epoch 28/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.4150 - val_loss: 0.5105\n",
            "Epoch 29/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.4076 - val_loss: 0.4861\n",
            "Epoch 30/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.4017 - val_loss: 0.4730\n",
            "Epoch 31/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3978 - val_loss: 0.4636\n",
            "Epoch 32/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3968 - val_loss: 0.4541\n",
            "Epoch 33/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3918 - val_loss: 0.4573\n",
            "Epoch 34/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3872 - val_loss: 0.4659\n",
            "Epoch 35/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3813 - val_loss: 0.4684\n",
            "Epoch 36/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3810 - val_loss: 0.4830\n",
            "Epoch 37/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3793 - val_loss: 0.4999\n",
            "Epoch 38/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3785 - val_loss: 0.5022\n",
            "Epoch 39/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3767 - val_loss: 0.4866\n",
            "Epoch 40/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3762 - val_loss: 0.4648\n",
            "Epoch 41/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3780 - val_loss: 0.4523\n",
            "Epoch 42/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3723 - val_loss: 0.4407\n",
            "Epoch 43/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3786 - val_loss: 0.4374\n",
            "Epoch 44/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3801 - val_loss: 0.4371\n",
            "Epoch 45/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3752 - val_loss: 0.4434\n",
            "Epoch 46/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3685 - val_loss: 0.4486\n",
            "Epoch 47/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3669 - val_loss: 0.4554\n",
            "Epoch 48/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3672 - val_loss: 0.4564\n",
            "Epoch 49/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3675 - val_loss: 0.4494\n",
            "Epoch 50/50\n",
            "2/2 [==============================] - 3s 1s/step - loss: 0.3629 - val_loss: 0.4452\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.76      0.86      0.81        29\n",
            "         1.0       0.50      0.33      0.40        12\n",
            "\n",
            "    accuracy                           0.71        41\n",
            "   macro avg       0.63      0.60      0.60        41\n",
            "weighted avg       0.68      0.71      0.69        41\n",
            "\n",
            "Y_pred:  (196, 1)\n",
            "TRAINING ACCCCCC                       precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.65      1.00      0.79        98\n",
            "         1.0       1.00      0.46      0.63        98\n",
            "\n",
            "    accuracy                           0.73       196\n",
            "   macro avg       0.82      0.73      0.71       196\n",
            "weighted avg       0.82      0.73      0.71       196\n",
            "\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.76      0.86      0.81        29\n",
            "         1.0       0.50      0.33      0.40        12\n",
            "\n",
            "    accuracy                           0.71        41\n",
            "   macro avg       0.63      0.60      0.60        41\n",
            "weighted avg       0.68      0.71      0.69        41\n",
            "\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.73      0.38      0.50        29\n",
            "         1.0       0.31      0.67      0.42        12\n",
            "\n",
            "    accuracy                           0.46        41\n",
            "   macro avg       0.52      0.52      0.46        41\n",
            "weighted avg       0.61      0.46      0.48        41\n",
            "\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.73      0.38      0.50        29\n",
            "         1.0       0.31      0.67      0.42        12\n",
            "\n",
            "    accuracy                           0.46        41\n",
            "   macro avg       0.52      0.52      0.46        41\n",
            "weighted avg       0.61      0.46      0.48        41\n",
            "\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.76      0.86      0.81        29\n",
            "         1.0       0.50      0.33      0.40        12\n",
            "\n",
            "    accuracy                           0.71        41\n",
            "   macro avg       0.63      0.60      0.60        41\n",
            "weighted avg       0.68      0.71      0.69        41\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0HXsKQ0kyc_0",
        "colab_type": "code",
        "outputId": "2bbe3b10-9dd6-4fd6-c981-7d1853f112b0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "print(\"--------------------------------------VIDEO + TEXT (W GATING) SENTENCE LEVEL-------------------------------------------------\")\n",
        "model2 = text_video()\n",
        "model = model2.run_model()\n",
        "\n",
        "model.fit([video_train,text_train],Ytrain, validation_split = 0.2, callbacks = tf.keras.callbacks.EarlyStopping(\n",
        "        monitor='val_loss', min_delta=0, patience=15, verbose=0, mode='min',\n",
        "        baseline=None, restore_best_weights=True),epochs=50, batch_size = 137)\n",
        "\n",
        "\n",
        "from sklearn.metrics import classification_report\n",
        "pred = model.predict([video_test,text_test])\n",
        "pred2 = model.predict([video_train,text_train])\n",
        "\n",
        "print(classification_report(Ytest,Thresholding(pred,0.5)))\n",
        "print(\"TRAINING ACCCCCC        \" , classification_report(Ytrain,Thresholding(pred2,0.5)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.6)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.4)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.3)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.7)))\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--------------------------------------VIDEO + TEXT (W GATING) SENTENCE LEVEL-------------------------------------------------\n",
            "Epoch 1/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6934 - val_loss: 0.6925\n",
            "Epoch 2/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6905 - val_loss: 0.6922\n",
            "Epoch 3/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6876 - val_loss: 0.6910\n",
            "Epoch 4/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6847 - val_loss: 0.6898\n",
            "Epoch 5/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6815 - val_loss: 0.6883\n",
            "Epoch 6/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6787 - val_loss: 0.6872\n",
            "Epoch 7/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6748 - val_loss: 0.6864\n",
            "Epoch 8/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6708 - val_loss: 0.6857\n",
            "Epoch 9/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6658 - val_loss: 0.6852\n",
            "Epoch 10/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6607 - val_loss: 0.6846\n",
            "Epoch 11/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6539 - val_loss: 0.6842\n",
            "Epoch 12/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6479 - val_loss: 0.6838\n",
            "Epoch 13/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6400 - val_loss: 0.6832\n",
            "Epoch 14/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6311 - val_loss: 0.6823\n",
            "Epoch 15/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6209 - val_loss: 0.6806\n",
            "Epoch 16/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6089 - val_loss: 0.6786\n",
            "Epoch 17/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.5975 - val_loss: 0.6747\n",
            "Epoch 18/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.5827 - val_loss: 0.6690\n",
            "Epoch 19/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.5663 - val_loss: 0.6644\n",
            "Epoch 20/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.5513 - val_loss: 0.6593\n",
            "Epoch 21/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.5382 - val_loss: 0.6527\n",
            "Epoch 22/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.5259 - val_loss: 0.6447\n",
            "Epoch 23/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.5149 - val_loss: 0.6345\n",
            "Epoch 24/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.5028 - val_loss: 0.6169\n",
            "Epoch 25/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4864 - val_loss: 0.5956\n",
            "Epoch 26/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4708 - val_loss: 0.5857\n",
            "Epoch 27/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4644 - val_loss: 0.5806\n",
            "Epoch 28/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4573 - val_loss: 0.5691\n",
            "Epoch 29/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4457 - val_loss: 0.5556\n",
            "Epoch 30/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4328 - val_loss: 0.5467\n",
            "Epoch 31/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4228 - val_loss: 0.5457\n",
            "Epoch 32/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4176 - val_loss: 0.5524\n",
            "Epoch 33/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4130 - val_loss: 0.5484\n",
            "Epoch 34/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4059 - val_loss: 0.5312\n",
            "Epoch 35/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3981 - val_loss: 0.5128\n",
            "Epoch 36/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3935 - val_loss: 0.5015\n",
            "Epoch 37/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3896 - val_loss: 0.4978\n",
            "Epoch 38/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3875 - val_loss: 0.5016\n",
            "Epoch 39/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3881 - val_loss: 0.4968\n",
            "Epoch 40/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3837 - val_loss: 0.4819\n",
            "Epoch 41/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3804 - val_loss: 0.4749\n",
            "Epoch 42/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3763 - val_loss: 0.4709\n",
            "Epoch 43/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3755 - val_loss: 0.4684\n",
            "Epoch 44/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3743 - val_loss: 0.4676\n",
            "Epoch 45/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3742 - val_loss: 0.4640\n",
            "Epoch 46/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3723 - val_loss: 0.4650\n",
            "Epoch 47/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3726 - val_loss: 0.4720\n",
            "Epoch 48/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3715 - val_loss: 0.4795\n",
            "Epoch 49/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3731 - val_loss: 0.4902\n",
            "Epoch 50/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3748 - val_loss: 0.4919\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.75      0.83      0.79        29\n",
            "         1.0       0.44      0.33      0.38        12\n",
            "\n",
            "    accuracy                           0.68        41\n",
            "   macro avg       0.60      0.58      0.58        41\n",
            "weighted avg       0.66      0.68      0.67        41\n",
            "\n",
            "Y_pred:  (196, 1)\n",
            "TRAINING ACCCCCC                       precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.65      0.99      0.78        98\n",
            "         1.0       0.98      0.46      0.62        98\n",
            "\n",
            "    accuracy                           0.72       196\n",
            "   macro avg       0.81      0.72      0.70       196\n",
            "weighted avg       0.81      0.72      0.70       196\n",
            "\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.75      0.83      0.79        29\n",
            "         1.0       0.44      0.33      0.38        12\n",
            "\n",
            "    accuracy                           0.68        41\n",
            "   macro avg       0.60      0.58      0.58        41\n",
            "weighted avg       0.66      0.68      0.67        41\n",
            "\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.76      0.45      0.57        29\n",
            "         1.0       0.33      0.67      0.44        12\n",
            "\n",
            "    accuracy                           0.51        41\n",
            "   macro avg       0.55      0.56      0.50        41\n",
            "weighted avg       0.64      0.51      0.53        41\n",
            "\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.73      0.38      0.50        29\n",
            "         1.0       0.31      0.67      0.42        12\n",
            "\n",
            "    accuracy                           0.46        41\n",
            "   macro avg       0.52      0.52      0.46        41\n",
            "weighted avg       0.61      0.46      0.48        41\n",
            "\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.76      0.86      0.81        29\n",
            "         1.0       0.50      0.33      0.40        12\n",
            "\n",
            "    accuracy                           0.71        41\n",
            "   macro avg       0.63      0.60      0.60        41\n",
            "weighted avg       0.68      0.71      0.69        41\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KJ0Bx0rdy75t",
        "colab_type": "code",
        "outputId": "baf36907-ffde-48be-ea7d-1950c08c8bce",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "print(\"--------------------------------------AUDIO + VIDEO + TEXT (W GATING) SENTENCE LEVEL-------------------------------------------------\")\n",
        "model3 = text_vieo_audio()\n",
        "model = model3.run_model()\n",
        "\n",
        "model.fit([audio_train,video_train,text_train],Ytrain, validation_split = 0.2, callbacks = tf.keras.callbacks.EarlyStopping(\n",
        "    monitor='val_loss', min_delta=0, patience=15, verbose=0, mode='min',\n",
        "    baseline=None, restore_best_weights=True),epochs=50, batch_size = 137)\n",
        "\n",
        "\n",
        "from sklearn.metrics import classification_report\n",
        "pred = model.predict([audio_test,video_test,text_test])\n",
        "pred2 = model.predict([audio_train,video_train,text_train])\n",
        "\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.5)))\n",
        "print(\"TRAINING ACCCCCC        \" , classification_report(Ytrain,Thresholding(pred2,0.5)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.6)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.4)))\n",
        "print(classification_report(Ytest,Thresholding(pred,0.3)))\n",
        "# print(classification_report(Ytest,Thresholding(pred,0.7)))\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--------------------------------------AUDIO + VIDEO + TEXT (W GATING) SENTENCE LEVEL-------------------------------------------------\n",
            "Epoch 1/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6925 - val_loss: 0.6930\n",
            "Epoch 2/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6901 - val_loss: 0.6931\n",
            "Epoch 3/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6887 - val_loss: 0.6934\n",
            "Epoch 4/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6867 - val_loss: 0.6933\n",
            "Epoch 5/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6842 - val_loss: 0.6930\n",
            "Epoch 6/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6816 - val_loss: 0.6924\n",
            "Epoch 7/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6786 - val_loss: 0.6914\n",
            "Epoch 8/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6744 - val_loss: 0.6903\n",
            "Epoch 9/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6695 - val_loss: 0.6897\n",
            "Epoch 10/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6647 - val_loss: 0.6893\n",
            "Epoch 11/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6593 - val_loss: 0.6881\n",
            "Epoch 12/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6509 - val_loss: 0.6851\n",
            "Epoch 13/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6439 - val_loss: 0.6811\n",
            "Epoch 14/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6333 - val_loss: 0.6764\n",
            "Epoch 15/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6214 - val_loss: 0.6723\n",
            "Epoch 16/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.6099 - val_loss: 0.6677\n",
            "Epoch 17/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.5948 - val_loss: 0.6606\n",
            "Epoch 18/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.5799 - val_loss: 0.6529\n",
            "Epoch 19/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.5644 - val_loss: 0.6462\n",
            "Epoch 20/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.5474 - val_loss: 0.6395\n",
            "Epoch 21/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.5318 - val_loss: 0.6280\n",
            "Epoch 22/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.5155 - val_loss: 0.6161\n",
            "Epoch 23/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4979 - val_loss: 0.6070\n",
            "Epoch 24/50\n",
            "2/2 [==============================] - 6s 3s/step - loss: 0.4820 - val_loss: 0.5970\n",
            "Epoch 25/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4672 - val_loss: 0.5884\n",
            "Epoch 26/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4544 - val_loss: 0.5778\n",
            "Epoch 27/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4378 - val_loss: 0.5601\n",
            "Epoch 28/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4263 - val_loss: 0.5397\n",
            "Epoch 29/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4151 - val_loss: 0.5174\n",
            "Epoch 30/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4103 - val_loss: 0.4972\n",
            "Epoch 31/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4095 - val_loss: 0.4814\n",
            "Epoch 32/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4130 - val_loss: 0.4682\n",
            "Epoch 33/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4135 - val_loss: 0.4603\n",
            "Epoch 34/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.4069 - val_loss: 0.4583\n",
            "Epoch 35/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3954 - val_loss: 0.4572\n",
            "Epoch 36/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3862 - val_loss: 0.4600\n",
            "Epoch 37/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3841 - val_loss: 0.4662\n",
            "Epoch 38/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3824 - val_loss: 0.4779\n",
            "Epoch 39/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3796 - val_loss: 0.4941\n",
            "Epoch 40/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3752 - val_loss: 0.5032\n",
            "Epoch 41/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3695 - val_loss: 0.4898\n",
            "Epoch 42/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3691 - val_loss: 0.4870\n",
            "Epoch 43/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3639 - val_loss: 0.5057\n",
            "Epoch 44/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3621 - val_loss: 0.5085\n",
            "Epoch 45/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3598 - val_loss: 0.4614\n",
            "Epoch 46/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3573 - val_loss: 0.4497\n",
            "Epoch 47/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3547 - val_loss: 0.4545\n",
            "Epoch 48/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3496 - val_loss: 0.4598\n",
            "Epoch 49/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3455 - val_loss: 0.4927\n",
            "Epoch 50/50\n",
            "2/2 [==============================] - 4s 2s/step - loss: 0.3327 - val_loss: 0.5235\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.74      0.69      0.71        29\n",
            "         1.0       0.36      0.42      0.38        12\n",
            "\n",
            "    accuracy                           0.61        41\n",
            "   macro avg       0.55      0.55      0.55        41\n",
            "weighted avg       0.63      0.61      0.62        41\n",
            "\n",
            "Y_pred:  (196, 1)\n",
            "TRAINING ACCCCCC                       precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.84      0.84      0.84        98\n",
            "         1.0       0.84      0.84      0.84        98\n",
            "\n",
            "    accuracy                           0.84       196\n",
            "   macro avg       0.84      0.84      0.84       196\n",
            "weighted avg       0.84      0.84      0.84       196\n",
            "\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.74      0.79      0.77        29\n",
            "         1.0       0.40      0.33      0.36        12\n",
            "\n",
            "    accuracy                           0.66        41\n",
            "   macro avg       0.57      0.56      0.57        41\n",
            "weighted avg       0.64      0.66      0.65        41\n",
            "\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.77      0.69      0.73        29\n",
            "         1.0       0.40      0.50      0.44        12\n",
            "\n",
            "    accuracy                           0.63        41\n",
            "   macro avg       0.58      0.59      0.59        41\n",
            "weighted avg       0.66      0.63      0.64        41\n",
            "\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.77      0.59      0.67        29\n",
            "         1.0       0.37      0.58      0.45        12\n",
            "\n",
            "    accuracy                           0.59        41\n",
            "   macro avg       0.57      0.58      0.56        41\n",
            "weighted avg       0.65      0.59      0.60        41\n",
            "\n",
            "Y_pred:  (41, 1)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "         0.0       0.74      0.79      0.77        29\n",
            "         1.0       0.40      0.33      0.36        12\n",
            "\n",
            "    accuracy                           0.66        41\n",
            "   macro avg       0.57      0.56      0.57        41\n",
            "weighted avg       0.64      0.66      0.65        41\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}