NLP_Project.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Objective of this assignment is to extract some sections (which are mentioned below) from SEC / EDGAR financial reports and perform text analysis to compute variables. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CIK</th>\n",
       "      <th>CONAME</th>\n",
       "      <th>FYRMO</th>\n",
       "      <th>FDATE</th>\n",
       "      <th>FORM</th>\n",
       "      <th>SECFNAME</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199803</td>\n",
       "      <td>1998-03-06</td>\n",
       "      <td>10-K405</td>\n",
       "      <td>edgar/data/3662/0000950170-98-000413.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199805</td>\n",
       "      <td>1998-05-15</td>\n",
       "      <td>10-Q</td>\n",
       "      <td>edgar/data/3662/0000950170-98-001001.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199808</td>\n",
       "      <td>1998-08-13</td>\n",
       "      <td>NT 10-Q</td>\n",
       "      <td>edgar/data/3662/0000950172-98-000783.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199811</td>\n",
       "      <td>1998-11-12</td>\n",
       "      <td>10-K/A</td>\n",
       "      <td>edgar/data/3662/0000950170-98-002145.txt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199811</td>\n",
       "      <td>1998-11-16</td>\n",
       "      <td>NT 10-Q</td>\n",
       "      <td>edgar/data/3662/0000950172-98-001203.txt</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    CIK            CONAME   FYRMO      FDATE     FORM  \\\n",
       "0  3662  SUNBEAM CORP/FL/  199803 1998-03-06  10-K405   \n",
       "1  3662  SUNBEAM CORP/FL/  199805 1998-05-15     10-Q   \n",
       "2  3662  SUNBEAM CORP/FL/  199808 1998-08-13  NT 10-Q   \n",
       "3  3662  SUNBEAM CORP/FL/  199811 1998-11-12   10-K/A   \n",
       "4  3662  SUNBEAM CORP/FL/  199811 1998-11-16  NT 10-Q   \n",
       "\n",
       "                                   SECFNAME  \n",
       "0  edgar/data/3662/0000950170-98-000413.txt  \n",
       "1  edgar/data/3662/0000950170-98-001001.txt  \n",
       "2  edgar/data/3662/0000950172-98-000783.txt  \n",
       "3  edgar/data/3662/0000950170-98-002145.txt  \n",
       "4  edgar/data/3662/0000950172-98-001203.txt  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file = pd.read_excel('cik_list.xlsx')\n",
    "file.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combining the url domain with 'secfname' column to access the financial reports in each link\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "y = 'https://www.sec.gov/Archives/'\n",
    "links = [y+x for x in file['SECFNAME']] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "152"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating a report list that contains all the text data from the financial reports \n",
    "\n",
    "## Each element of the list contains is the text from each financial report that is read form the links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "report = []\n",
    "for url in links:\n",
    "    r = requests.get(url)\n",
    "    data=r.text\n",
    "    words=BeautifulSoup(data,'html.parser')\n",
    "    report.append(words.get_text())\n",
    "    \n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import sent_tokenize, word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "152\n"
     ]
    }
   ],
   "source": [
    "print(len(report))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading the stopwords from the given url and storing it in a variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "with open ('StopWords_GenericLong.txt', 'r') as p:\n",
    "    stop_words = p.read()\n",
    "    stop_words = stop_words.split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "571"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(stop_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading the Master dictionary from url, which contains the year in which a positive or a negative words were added\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word</th>\n",
       "      <th>Sequence Number</th>\n",
       "      <th>Word Count</th>\n",
       "      <th>Word Proportion</th>\n",
       "      <th>Average Proportion</th>\n",
       "      <th>Std Dev</th>\n",
       "      <th>Doc Count</th>\n",
       "      <th>Negative</th>\n",
       "      <th>Positive</th>\n",
       "      <th>Uncertainty</th>\n",
       "      <th>Litigious</th>\n",
       "      <th>Constraining</th>\n",
       "      <th>Superfluous</th>\n",
       "      <th>Interesting</th>\n",
       "      <th>Modal</th>\n",
       "      <th>Irr_Verb</th>\n",
       "      <th>Harvard_IV</th>\n",
       "      <th>Syllables</th>\n",
       "      <th>Source</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AARDVARK</td>\n",
       "      <td>1</td>\n",
       "      <td>277</td>\n",
       "      <td>1.480368e-08</td>\n",
       "      <td>1.239377e-08</td>\n",
       "      <td>3.564730e-06</td>\n",
       "      <td>84</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>12of12inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AARDVARKS</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1.603287e-10</td>\n",
       "      <td>9.725110e-12</td>\n",
       "      <td>9.863549e-09</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>12of12inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ABACI</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>4.275431e-10</td>\n",
       "      <td>1.386497e-10</td>\n",
       "      <td>6.225591e-08</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>12of12inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ABACK</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>6.413147e-10</td>\n",
       "      <td>3.159061e-10</td>\n",
       "      <td>9.383557e-08</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>12of12inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ABACUS</td>\n",
       "      <td>5</td>\n",
       "      <td>7250</td>\n",
       "      <td>3.874610e-07</td>\n",
       "      <td>3.681624e-07</td>\n",
       "      <td>3.366553e-05</td>\n",
       "      <td>914</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>12of12inf</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Word  Sequence Number  Word Count  Word Proportion  \\\n",
       "0   AARDVARK                1         277     1.480368e-08   \n",
       "1  AARDVARKS                2           3     1.603287e-10   \n",
       "2      ABACI                3           8     4.275431e-10   \n",
       "3      ABACK                4          12     6.413147e-10   \n",
       "4     ABACUS                5        7250     3.874610e-07   \n",
       "\n",
       "   Average Proportion       Std Dev  Doc Count  Negative  Positive  \\\n",
       "0        1.239377e-08  3.564730e-06         84         0         0   \n",
       "1        9.725110e-12  9.863549e-09          1         0         0   \n",
       "2        1.386497e-10  6.225591e-08          7         0         0   \n",
       "3        3.159061e-10  9.383557e-08         12         0         0   \n",
       "4        3.681624e-07  3.366553e-05        914         0         0   \n",
       "\n",
       "   Uncertainty  Litigious  Constraining  Superfluous  Interesting  Modal  \\\n",
       "0            0          0             0            0            0      0   \n",
       "1            0          0             0            0            0      0   \n",
       "2            0          0             0            0            0      0   \n",
       "3            0          0             0            0            0      0   \n",
       "4            0          0             0            0            0      0   \n",
       "\n",
       "   Irr_Verb  Harvard_IV  Syllables     Source  \n",
       "0         0           0          2  12of12inf  \n",
       "1         0           0          2  12of12inf  \n",
       "2         0           0          3  12of12inf  \n",
       "3         0           0          2  12of12inf  \n",
       "4         0           0          3  12of12inf  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "MD = pd.read_excel('LoughranMcDonald_MasterDictionary_2018.xlsx')\n",
    "MD.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating a postive dictionary list contains the postive words from the Master dictionary\n",
    "\n",
    "## Postive words with value 0 in master dictionary, depicts that the word is neutral"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "354"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "pos_dict = [x for x in MD[MD['Positive']!=0]['Word']]\n",
    "len(pos_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating a negative dictionary list contains the negative words from the Master dictionary\n",
    "\n",
    "## Negative words with value 0 in master dictionary, depicts that the word is neutral"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2355"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "neg_dict =  [x for x in MD[MD['Negative'] != 0]['Word']]\n",
    "\n",
    "len(neg_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading the uncertainity words from the uncertainity_dictionary excel sheet "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "uncertainity = pd.read_excel('uncertainity_dictionary.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ABEYANCE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ABEYANCES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ALMOST</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ALTERATION</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ALTERATIONS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>292</th>\n",
       "      <td>VARY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293</th>\n",
       "      <td>VARYING</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>294</th>\n",
       "      <td>VOLATILE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295</th>\n",
       "      <td>VOLATILITIES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>296</th>\n",
       "      <td>VOLATILITY</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>297 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Word\n",
       "0        ABEYANCE\n",
       "1       ABEYANCES\n",
       "2          ALMOST\n",
       "3      ALTERATION\n",
       "4     ALTERATIONS\n",
       "..            ...\n",
       "292          VARY\n",
       "293       VARYING\n",
       "294      VOLATILE\n",
       "295  VOLATILITIES\n",
       "296    VOLATILITY\n",
       "\n",
       "[297 rows x 1 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uncertainity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "uncertainity_words = list(uncertainity['Word'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading the constraining words from the constraining_dictionary excel sheet "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "constraining = pd.read_excel('constraining_dictionary.xlsx')\n",
    "constraining_words = list(constraining['Word'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating a user defined function to tokenize the text from the url into words\n",
    "\n",
    "## r'[^A-Za-z]',' ' -> replacing all the substrings other than a-z or A-z with ' ' blank space\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def tokenize(text):\n",
    "    text = re.sub(r'[^A-Za-z]',' ',text.upper())\n",
    "    tokenized_words = word_tokenize(text)\n",
    "    return tokenized_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## User defined function to remove all the stop words like a, the, he etc from the text\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def remove_stopwords(words, stop_words):\n",
    "    return [x for x in words if x not in stop_words]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## User defined function to count the number of positive and negative words in the text  which matches with those in the positive and negative dictionary list created form master dictionary \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def countfunc(store, words):\n",
    "    score = 0\n",
    "    for x in words:\n",
    "        if(x in store):\n",
    "            score = score+1\n",
    "    return score\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# user defined function to calculate the polarity score value of the text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def polarity(positive_score, negative_score):\n",
    "    polarity_score = (positive_score - negative_score)/((positive_score + negative_score)+ 0.000001)\n",
    "    return polarity_score\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## User defined function for sentiment score categorization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def sentiment(score):\n",
    "    if(score < -0.5):\n",
    "        return 'Most Negative'\n",
    "    elif(score >= -0.5 and score < 0):\n",
    "        return 'Negative'\n",
    "    elif(score == 0):\n",
    "        return 'Neutral'\n",
    "    elif(score > 0 and score < 0.5):\n",
    "        return 'Positive'\n",
    "    else:\n",
    "        return 'Very Positive'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## User defined function to calculate the subjectivity score of the text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def subjectivity(positive_score, negative_score, num_words):\n",
    "    return (positive_score+negative_score)/(num_words+ 0.000001)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## User defined function that return complex words with more than 2 syllables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def syllable_morethan2(word):\n",
    "    if(len(word) > 2 and (word[-2:] == 'es' or word[-2:] == 'ed')):\n",
    "        return False\n",
    "    \n",
    "    count =0\n",
    "    vowels = ['a','e','i','o','u']\n",
    "    for i in word:\n",
    "        if(i.lower() in vowels):\n",
    "            count = count +1\n",
    "        \n",
    "    if(count > 2):\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## User defined function to calculate the fog index\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "    \n",
    "def fog_index_cal(average_sentence_length, percentage_complexwords):\n",
    "    return 0.4*(average_sentence_length + percentage_complexwords)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating a column with all the 15 output variables and assigning default value as 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "col = ['positive_score',\n",
    "      'negative_score',\n",
    "      'polarity_score',\n",
    "      'average_sentence_length',\n",
    "      'percentage_of_complex_words',\n",
    "      'fog_index',\n",
    "      'complex_word_count',\n",
    "      'word_count',\n",
    "      'uncertainity_score',\n",
    "      'constraining_score',\n",
    "      'positive_word_proportion',\n",
    "      'negative_word_proportion',\n",
    "      'uncertainity_word_proportion',\n",
    "      'constraining_word_proportion',\n",
    "      'constraining_words_whole_report']\n",
    "\n",
    "for c in col[:]:\n",
    "        file[c] = 0.0\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking how the structure of output file we will get before appending the values of the NLP model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CIK</th>\n",
       "      <th>CONAME</th>\n",
       "      <th>FYRMO</th>\n",
       "      <th>FDATE</th>\n",
       "      <th>FORM</th>\n",
       "      <th>SECFNAME</th>\n",
       "      <th>positive_score</th>\n",
       "      <th>negative_score</th>\n",
       "      <th>polarity_score</th>\n",
       "      <th>average_sentence_length</th>\n",
       "      <th>...</th>\n",
       "      <th>fog_index</th>\n",
       "      <th>complex_word_count</th>\n",
       "      <th>word_count</th>\n",
       "      <th>uncertainity_score</th>\n",
       "      <th>constraining_score</th>\n",
       "      <th>positive_word_proportion</th>\n",
       "      <th>negative_word_proportion</th>\n",
       "      <th>uncertainity_word_proportion</th>\n",
       "      <th>constraining_word_proportion</th>\n",
       "      <th>constraining_words_whole_report</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199803</td>\n",
       "      <td>1998-03-06</td>\n",
       "      <td>10-K405</td>\n",
       "      <td>edgar/data/3662/0000950170-98-000413.txt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199805</td>\n",
       "      <td>1998-05-15</td>\n",
       "      <td>10-Q</td>\n",
       "      <td>edgar/data/3662/0000950170-98-001001.txt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199808</td>\n",
       "      <td>1998-08-13</td>\n",
       "      <td>NT 10-Q</td>\n",
       "      <td>edgar/data/3662/0000950172-98-000783.txt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199811</td>\n",
       "      <td>1998-11-12</td>\n",
       "      <td>10-K/A</td>\n",
       "      <td>edgar/data/3662/0000950170-98-002145.txt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199811</td>\n",
       "      <td>1998-11-16</td>\n",
       "      <td>NT 10-Q</td>\n",
       "      <td>edgar/data/3662/0000950172-98-001203.txt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>12239</td>\n",
       "      <td>SPHERIX INC</td>\n",
       "      <td>200704</td>\n",
       "      <td>2007-04-02</td>\n",
       "      <td>10-K</td>\n",
       "      <td>edgar/data/12239/0001104659-07-024804.txt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>12239</td>\n",
       "      <td>SPHERIX INC</td>\n",
       "      <td>200705</td>\n",
       "      <td>2007-05-16</td>\n",
       "      <td>NT 10-Q</td>\n",
       "      <td>edgar/data/12239/0001104659-07-040463.txt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>12239</td>\n",
       "      <td>SPHERIX INC</td>\n",
       "      <td>200705</td>\n",
       "      <td>2007-05-18</td>\n",
       "      <td>10-Q</td>\n",
       "      <td>edgar/data/12239/0001104659-07-041441.txt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>12239</td>\n",
       "      <td>SPHERIX INC</td>\n",
       "      <td>200705</td>\n",
       "      <td>2007-05-23</td>\n",
       "      <td>10-K/A</td>\n",
       "      <td>edgar/data/12239/0001104659-07-042333.txt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>12239</td>\n",
       "      <td>SPHERIX INC</td>\n",
       "      <td>200708</td>\n",
       "      <td>2007-08-14</td>\n",
       "      <td>10-Q</td>\n",
       "      <td>edgar/data/12239/0001104659-07-062470.txt</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>152 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       CIK            CONAME   FYRMO      FDATE     FORM  \\\n",
       "0     3662  SUNBEAM CORP/FL/  199803 1998-03-06  10-K405   \n",
       "1     3662  SUNBEAM CORP/FL/  199805 1998-05-15     10-Q   \n",
       "2     3662  SUNBEAM CORP/FL/  199808 1998-08-13  NT 10-Q   \n",
       "3     3662  SUNBEAM CORP/FL/  199811 1998-11-12   10-K/A   \n",
       "4     3662  SUNBEAM CORP/FL/  199811 1998-11-16  NT 10-Q   \n",
       "..     ...               ...     ...        ...      ...   \n",
       "147  12239       SPHERIX INC  200704 2007-04-02     10-K   \n",
       "148  12239       SPHERIX INC  200705 2007-05-16  NT 10-Q   \n",
       "149  12239       SPHERIX INC  200705 2007-05-18     10-Q   \n",
       "150  12239       SPHERIX INC  200705 2007-05-23   10-K/A   \n",
       "151  12239       SPHERIX INC  200708 2007-08-14     10-Q   \n",
       "\n",
       "                                      SECFNAME  positive_score  \\\n",
       "0     edgar/data/3662/0000950170-98-000413.txt             0.0   \n",
       "1     edgar/data/3662/0000950170-98-001001.txt             0.0   \n",
       "2     edgar/data/3662/0000950172-98-000783.txt             0.0   \n",
       "3     edgar/data/3662/0000950170-98-002145.txt             0.0   \n",
       "4     edgar/data/3662/0000950172-98-001203.txt             0.0   \n",
       "..                                         ...             ...   \n",
       "147  edgar/data/12239/0001104659-07-024804.txt             0.0   \n",
       "148  edgar/data/12239/0001104659-07-040463.txt             0.0   \n",
       "149  edgar/data/12239/0001104659-07-041441.txt             0.0   \n",
       "150  edgar/data/12239/0001104659-07-042333.txt             0.0   \n",
       "151  edgar/data/12239/0001104659-07-062470.txt             0.0   \n",
       "\n",
       "     negative_score  polarity_score  average_sentence_length  ...  fog_index  \\\n",
       "0               0.0             0.0                      0.0  ...        0.0   \n",
       "1               0.0             0.0                      0.0  ...        0.0   \n",
       "2               0.0             0.0                      0.0  ...        0.0   \n",
       "3               0.0             0.0                      0.0  ...        0.0   \n",
       "4               0.0             0.0                      0.0  ...        0.0   \n",
       "..              ...             ...                      ...  ...        ...   \n",
       "147             0.0             0.0                      0.0  ...        0.0   \n",
       "148             0.0             0.0                      0.0  ...        0.0   \n",
       "149             0.0             0.0                      0.0  ...        0.0   \n",
       "150             0.0             0.0                      0.0  ...        0.0   \n",
       "151             0.0             0.0                      0.0  ...        0.0   \n",
       "\n",
       "     complex_word_count  word_count  uncertainity_score  constraining_score  \\\n",
       "0                   0.0         0.0                 0.0                 0.0   \n",
       "1                   0.0         0.0                 0.0                 0.0   \n",
       "2                   0.0         0.0                 0.0                 0.0   \n",
       "3                   0.0         0.0                 0.0                 0.0   \n",
       "4                   0.0         0.0                 0.0                 0.0   \n",
       "..                  ...         ...                 ...                 ...   \n",
       "147                 0.0         0.0                 0.0                 0.0   \n",
       "148                 0.0         0.0                 0.0                 0.0   \n",
       "149                 0.0         0.0                 0.0                 0.0   \n",
       "150                 0.0         0.0                 0.0                 0.0   \n",
       "151                 0.0         0.0                 0.0                 0.0   \n",
       "\n",
       "     positive_word_proportion  negative_word_proportion  \\\n",
       "0                         0.0                       0.0   \n",
       "1                         0.0                       0.0   \n",
       "2                         0.0                       0.0   \n",
       "3                         0.0                       0.0   \n",
       "4                         0.0                       0.0   \n",
       "..                        ...                       ...   \n",
       "147                       0.0                       0.0   \n",
       "148                       0.0                       0.0   \n",
       "149                       0.0                       0.0   \n",
       "150                       0.0                       0.0   \n",
       "151                       0.0                       0.0   \n",
       "\n",
       "     uncertainity_word_proportion  constraining_word_proportion  \\\n",
       "0                             0.0                           0.0   \n",
       "1                             0.0                           0.0   \n",
       "2                             0.0                           0.0   \n",
       "3                             0.0                           0.0   \n",
       "4                             0.0                           0.0   \n",
       "..                            ...                           ...   \n",
       "147                           0.0                           0.0   \n",
       "148                           0.0                           0.0   \n",
       "149                           0.0                           0.0   \n",
       "150                           0.0                           0.0   \n",
       "151                           0.0                           0.0   \n",
       "\n",
       "     constraining_words_whole_report  \n",
       "0                                0.0  \n",
       "1                                0.0  \n",
       "2                                0.0  \n",
       "3                                0.0  \n",
       "4                                0.0  \n",
       "..                               ...  \n",
       "147                              0.0  \n",
       "148                              0.0  \n",
       "149                              0.0  \n",
       "150                              0.0  \n",
       "151                              0.0  \n",
       "\n",
       "[152 rows x 21 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(file.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from nltk import sent_tokenize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running a loop to access each elemet of the 152 links and calling all the user defined functions by passing the data\n",
    "\n",
    "## Also appending the values returned from the user defined functions to the file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(report)):\n",
    "    content = report[i]\n",
    "    tokenized_words = tokenize(content)\n",
    "    words = remove_stopwords(tokenized_words, stop_words)\n",
    "    num_words = len(words)\n",
    "    sentences = sent_tokenize(content)\n",
    "    num_sentences = len(sentences)\n",
    "    average_sentence_length = num_words/num_sentences\n",
    "    \n",
    "    positive_score = countfunc(pos_dict, words)\n",
    "    negative_score = countfunc(neg_dict, words)\n",
    "    \n",
    "    polarity_score = polarity(positive_score, negative_score)\n",
    "               \n",
    "    subjectivity_score = subjectivity(positive_score, negative_score, num_words)\n",
    "    \n",
    "    num_complexword =0\n",
    "    uncertainity_score = 0\n",
    "    constraining_score = 0\n",
    "    \n",
    "    for word in words:\n",
    "        if(syllable_morethan2(word)):\n",
    "            num_complexword = num_complexword+1\n",
    "                        \n",
    "        if(word in uncertainity_words):\n",
    "            uncertainity_score = uncertainity_score+1\n",
    "                        \n",
    "        if(word in constraining_words):\n",
    "            constraining_score = constraining_score+1\n",
    "            \n",
    "    percentage_complexwords = num_complexword/num_words\n",
    "                \n",
    "    fog_index = fog_index_cal(average_sentence_length, percentage_complexwords)\n",
    "                \n",
    "                \n",
    "    positive_word_proportion = positive_score/num_words\n",
    "    negative_word_proportion = negative_score/num_words\n",
    "    uncertainity_word_proportion = uncertainity_score/num_words\n",
    "    constraining_word_proportion = constraining_score/num_words\n",
    "    \n",
    "\n",
    "    \n",
    "    file.at[i,'positive_score'] = positive_score\n",
    "    file.at[i,'negative_score'] = negative_score\n",
    "    file.at[i,'polarity_score'] = polarity_score\n",
    "    file.at[i,'average_sentence_length'] = average_sentence_length\n",
    "    file.at[i,'percentage_of_complex_words'] = percentage_complexwords\n",
    "    file.at[i,'fog_index'] = fog_index\n",
    "    file.at[i,'complex_word_count'] = num_complexword\n",
    "    file.at[i,'word_count'] = num_words\n",
    "    file.at[i,'uncertainity_score'] = uncertainity_score\n",
    "    file.at[i,'constraining_score'] = constraining_score\n",
    "    file.at[i,'positive_word_proportion'] = positive_word_proportion\n",
    "    file.at[i,'negative_word_proportion'] = negative_word_proportion\n",
    "    file.at[i,'uncertainity_word_proportion'] = uncertainity_word_proportion\n",
    "    file.at[i,'constraining_word_proportion'] = constraining_word_proportion\n",
    "                \n",
    "      \n",
    "                \n",
    "                \n",
    "    constraining_words_whole_report = 0\n",
    "    tokenized_report_words = tokenize(report[i])\n",
    "    report_words = remove_stopwords(tokenized_report_words, stop_words)\n",
    "    for word in report_words:\n",
    "        if word in constraining_words:\n",
    "            constraining_words_whole_report = 1+ constraining_words_whole_report\n",
    "    \n",
    "    file.at[i,'constraining_words_whole_report'] = constraining_words_whole_report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking for the output for first five rows of the file with the score from the NLP model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CIK</th>\n",
       "      <th>CONAME</th>\n",
       "      <th>FYRMO</th>\n",
       "      <th>FDATE</th>\n",
       "      <th>FORM</th>\n",
       "      <th>SECFNAME</th>\n",
       "      <th>positive_score</th>\n",
       "      <th>negative_score</th>\n",
       "      <th>polarity_score</th>\n",
       "      <th>average_sentence_length</th>\n",
       "      <th>...</th>\n",
       "      <th>fog_index</th>\n",
       "      <th>complex_word_count</th>\n",
       "      <th>word_count</th>\n",
       "      <th>uncertainity_score</th>\n",
       "      <th>constraining_score</th>\n",
       "      <th>positive_word_proportion</th>\n",
       "      <th>negative_word_proportion</th>\n",
       "      <th>uncertainity_word_proportion</th>\n",
       "      <th>constraining_word_proportion</th>\n",
       "      <th>constraining_words_whole_report</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199803</td>\n",
       "      <td>1998-03-06</td>\n",
       "      <td>10-K405</td>\n",
       "      <td>edgar/data/3662/0000950170-98-000413.txt</td>\n",
       "      <td>6.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>-0.250000</td>\n",
       "      <td>23.40000</td>\n",
       "      <td>...</td>\n",
       "      <td>9.478803</td>\n",
       "      <td>139.0</td>\n",
       "      <td>468.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.012821</td>\n",
       "      <td>0.021368</td>\n",
       "      <td>0.008547</td>\n",
       "      <td>0.010684</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199805</td>\n",
       "      <td>1998-05-15</td>\n",
       "      <td>10-Q</td>\n",
       "      <td>edgar/data/3662/0000950170-98-001001.txt</td>\n",
       "      <td>585.0</td>\n",
       "      <td>1479.0</td>\n",
       "      <td>-0.433140</td>\n",
       "      <td>43.43149</td>\n",
       "      <td>...</td>\n",
       "      <td>17.493439</td>\n",
       "      <td>32750.0</td>\n",
       "      <td>108405.0</td>\n",
       "      <td>859.0</td>\n",
       "      <td>1046.0</td>\n",
       "      <td>0.005396</td>\n",
       "      <td>0.013643</td>\n",
       "      <td>0.007924</td>\n",
       "      <td>0.009649</td>\n",
       "      <td>1046.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199808</td>\n",
       "      <td>1998-08-13</td>\n",
       "      <td>NT 10-Q</td>\n",
       "      <td>edgar/data/3662/0000950172-98-000783.txt</td>\n",
       "      <td>2.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>-0.600000</td>\n",
       "      <td>44.30000</td>\n",
       "      <td>...</td>\n",
       "      <td>17.829707</td>\n",
       "      <td>243.0</td>\n",
       "      <td>886.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.002257</td>\n",
       "      <td>0.009029</td>\n",
       "      <td>0.010158</td>\n",
       "      <td>0.005643</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199811</td>\n",
       "      <td>1998-11-12</td>\n",
       "      <td>10-K/A</td>\n",
       "      <td>edgar/data/3662/0000950170-98-002145.txt</td>\n",
       "      <td>6.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>-0.250000</td>\n",
       "      <td>23.40000</td>\n",
       "      <td>...</td>\n",
       "      <td>9.478803</td>\n",
       "      <td>139.0</td>\n",
       "      <td>468.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.012821</td>\n",
       "      <td>0.021368</td>\n",
       "      <td>0.008547</td>\n",
       "      <td>0.010684</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3662</td>\n",
       "      <td>SUNBEAM CORP/FL/</td>\n",
       "      <td>199811</td>\n",
       "      <td>1998-11-16</td>\n",
       "      <td>NT 10-Q</td>\n",
       "      <td>edgar/data/3662/0000950172-98-001203.txt</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>-0.454545</td>\n",
       "      <td>44.08000</td>\n",
       "      <td>...</td>\n",
       "      <td>17.734359</td>\n",
       "      <td>282.0</td>\n",
       "      <td>1102.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.002722</td>\n",
       "      <td>0.007260</td>\n",
       "      <td>0.009074</td>\n",
       "      <td>0.003630</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    CIK            CONAME   FYRMO      FDATE     FORM  \\\n",
       "0  3662  SUNBEAM CORP/FL/  199803 1998-03-06  10-K405   \n",
       "1  3662  SUNBEAM CORP/FL/  199805 1998-05-15     10-Q   \n",
       "2  3662  SUNBEAM CORP/FL/  199808 1998-08-13  NT 10-Q   \n",
       "3  3662  SUNBEAM CORP/FL/  199811 1998-11-12   10-K/A   \n",
       "4  3662  SUNBEAM CORP/FL/  199811 1998-11-16  NT 10-Q   \n",
       "\n",
       "                                   SECFNAME  positive_score  negative_score  \\\n",
       "0  edgar/data/3662/0000950170-98-000413.txt             6.0            10.0   \n",
       "1  edgar/data/3662/0000950170-98-001001.txt           585.0          1479.0   \n",
       "2  edgar/data/3662/0000950172-98-000783.txt             2.0             8.0   \n",
       "3  edgar/data/3662/0000950170-98-002145.txt             6.0            10.0   \n",
       "4  edgar/data/3662/0000950172-98-001203.txt             3.0             8.0   \n",
       "\n",
       "   polarity_score  average_sentence_length  ...  fog_index  \\\n",
       "0       -0.250000                 23.40000  ...   9.478803   \n",
       "1       -0.433140                 43.43149  ...  17.493439   \n",
       "2       -0.600000                 44.30000  ...  17.829707   \n",
       "3       -0.250000                 23.40000  ...   9.478803   \n",
       "4       -0.454545                 44.08000  ...  17.734359   \n",
       "\n",
       "   complex_word_count  word_count  uncertainity_score  constraining_score  \\\n",
       "0               139.0       468.0                 4.0                 5.0   \n",
       "1             32750.0    108405.0               859.0              1046.0   \n",
       "2               243.0       886.0                 9.0                 5.0   \n",
       "3               139.0       468.0                 4.0                 5.0   \n",
       "4               282.0      1102.0                10.0                 4.0   \n",
       "\n",
       "   positive_word_proportion  negative_word_proportion  \\\n",
       "0                  0.012821                  0.021368   \n",
       "1                  0.005396                  0.013643   \n",
       "2                  0.002257                  0.009029   \n",
       "3                  0.012821                  0.021368   \n",
       "4                  0.002722                  0.007260   \n",
       "\n",
       "   uncertainity_word_proportion  constraining_word_proportion  \\\n",
       "0                      0.008547                      0.010684   \n",
       "1                      0.007924                      0.009649   \n",
       "2                      0.010158                      0.005643   \n",
       "3                      0.008547                      0.010684   \n",
       "4                      0.009074                      0.003630   \n",
       "\n",
       "   constraining_words_whole_report  \n",
       "0                              5.0  \n",
       "1                           1046.0  \n",
       "2                              5.0  \n",
       "3                              5.0  \n",
       "4                              4.0  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sending the file to an excel sheet with name Output1 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "file.to_excel('Output1.xlsx')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}