recurrent_words_connotation_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Recurrent Words Connotation Analysis\n",
    "\n",
    "We want to establish which recurring words in the corpus are responsible for the sentiment score over time. More specifically, we are interested in the words that impact the positive/negative sentiment score over each one of the phases determined by the topic modeling analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import warnings\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_tokens(texts):\n",
    "    docs = list(nlp.pipe(texts))\n",
    "    tokens = []\n",
    "    for doc in docs:\n",
    "        for token in doc:\n",
    "                tokens.append(token.lemma_)\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ignore warnings in libraries\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "# Import Italo Svevo corpus dataset\n",
    "data = pd.read_csv('datasets/carteggio.svevo3.csv', sep=';', parse_dates=['date'])\n",
    "\n",
    "# Import positive and negative italian lexicons\n",
    "pos = list(pd.read_csv('datasets/nrc_positive_it.txt', sep='\\n', header=None)[0])\n",
    "neg = list(pd.read_csv('datasets/nrc_negative_it.txt', sep='\\n', header=None)[0])\n",
    "\n",
    "# Tokenization\n",
    "nlp = spacy.load(\"it\", disable=['parser', 'ner'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter italian language letters only\n",
    "data_it = data[data['mainLanguage'] == 'ITA']\n",
    "\n",
    "# Split in groups based on years and topics\n",
    "d8501 = list(data_it.query('year > 1884 and year < 1902')['text'])\n",
    "d0208 = list(data_it.query('year > 1901 and year < 1909')['text'])\n",
    "d0922 = list(data_it.query('year > 1908 and year < 1923')['text'])\n",
    "d2328 = list(data_it.query('year > 1922 and year < 1929')['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "t8501 = extract_tokens(d8501)\n",
    "t0208 = extract_tokens(d0208)\n",
    "t0922 = extract_tokens(d0922)\n",
    "t2328 = extract_tokens(d2328)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_8501 = [x for x in t8501 if x in pos]\n",
    "pos_0208 = [x for x in t0208 if x in pos]\n",
    "pos_0922 = [x for x in t0922 if x in pos]\n",
    "pos_2328 = [x for x in t2328 if x in pos]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "neg_8501 = [x for x in t8501 if x in neg]\n",
    "neg_0208 = [x for x in t0208 if x in neg]\n",
    "neg_0922 = [x for x in t0922 if x in neg]\n",
    "neg_2328 = [x for x in t2328 if x in neg]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 most common positive words for 1885-1901:\n",
      "[('primo', 227), ('caro', 208), ('parlare', 171), ('bene', 169), ('parola', 122), ('vero', 77), ('bello', 64), ('affare', 58), ('trattare', 53), ('solito', 50)]\n",
      "10 most common positive words for 1902-1908:\n",
      "[('caro', 194), ('bene', 155), ('primo', 124), ('parlare', 78), ('pregare', 52), ('vero', 51), ('solito', 46), ('affare', 46), ('bello', 42), ('parola', 41)]\n",
      "10 most common positive words for 1909-1922:\n",
      "[('bene', 143), ('caro', 139), ('primo', 105), ('parlare', 70), ('abbastanza', 53), ('solito', 46), ('parola', 37), ('bello', 36), ('pregare', 35), ('vero', 35)]\n",
      "10 most common positive words for 1923-1928:\n",
      "[('primo', 120), ('caro', 105), ('parlare', 85), ('signore', 78), ('bene', 64), ('articolare', 58), ('vero', 51), ('parola', 48), ('saluto', 37), ('amico', 35)]\n",
      "\n",
      "10 most common negative words for 1885-1901:\n",
      "[('caro', 208), ('partire', 159), ('piccolo', 87), ('dispiacere', 79), ('trattare', 53), ('fabbricare', 53), ('madre', 47), ('dolore', 45), ('dubbio', 39), ('soffrire', 37)]\n",
      "10 most common negative words for 1902-1908:\n",
      "[('caro', 194), ('partire', 135), ('fabbricare', 112), ('dispiacere', 56), ('piccolo', 35), ('trattare', 27), ('aspettare', 26), ('peggio', 26), ('dimenticare', 24), ('servire', 22)]\n",
      "10 most common negative words for 1909-1922:\n",
      "[('caro', 139), ('partire', 116), ('dispiacere', 67), ('bagnare', 41), ('fabbricare', 40), ('piccolo', 35), ('colera', 31), ('trattare', 30), ('aspettare', 26), ('appena', 24)]\n",
      "10 most common negative words for 1923-1928:\n",
      "[('caro', 105), ('signore', 78), ('partire', 61), ('copia', 32), ('critico', 30), ('aspettare', 26), ('criticare', 25), ('perdere', 24), ('piccolo', 24), ('trattare', 21)]\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "print(\"10 most common positive words for 1885-1901:\")\n",
    "print(Counter(pos_8501).most_common(10))\n",
    "print(\"10 most common positive words for 1902-1908:\")\n",
    "print(Counter(pos_0208).most_common(10))\n",
    "print(\"10 most common positive words for 1909-1922:\")\n",
    "print(Counter(pos_0922).most_common(10))\n",
    "print(\"10 most common positive words for 1923-1928:\")\n",
    "print(Counter(pos_2328).most_common(10))\n",
    "print()\n",
    "print(\"10 most common negative words for 1885-1901:\")\n",
    "print(Counter(neg_8501).most_common(10))\n",
    "print(\"10 most common negative words for 1902-1908:\")\n",
    "print(Counter(neg_0208).most_common(10))\n",
    "print(\"10 most common negative words for 1909-1922:\")\n",
    "print(Counter(neg_0922).most_common(10))\n",
    "print(\"10 most common negative words for 1923-1928:\")\n",
    "print(Counter(neg_2328).most_common(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now filtering out too common/insignificant words\n",
    "bad_pos = ['primo', 'caro', 'parlare', 'bene', 'parola', 'vero', 'bello', 'solito', 'saluto', 'signore', \n",
    "       'abbastanza', 'salutare', 'affare', 'trattare']\n",
    "\n",
    "gpos_8501 = [x for x in t8501 if x in pos and x not in bad_pos]\n",
    "gpos_0208 = [x for x in t0208 if x in pos and x not in bad_pos]\n",
    "gpos_0922 = [x for x in t0922 if x in pos and x not in bad_pos]\n",
    "gpos_2328 = [x for x in t2328 if x in pos and x not in bad_pos]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 most common positive words for 1885-1901:\n",
      "[('vicino', 49), ('madre', 47), ('pregare', 45), ('sicuro', 41), ('dolce', 37), ('lieto', 37), ('pieno', 33), ('evidente', 32), ('porto', 32), ('mangiare', 30)]\n",
      "10 most common positive words for 1902-1908:\n",
      "[('pregare', 52), ('mangiare', 39), ('insieme', 32), ('vicino', 22), ('dolce', 19), ('bambino', 17), ('pieno', 16), ('sterlina', 16), ('invitare', 15), ('sicuro', 15)]\n",
      "10 most common positive words for 1909-1922:\n",
      "[('pregare', 35), ('mangiare', 30), ('insieme', 23), ('stato', 21), ('sicuro', 19), ('lira', 19), ('pieno', 17), ('madre', 17), ('pagare', 17), ('contare', 15)]\n",
      "10 most common positive words for 1923-1928:\n",
      "[('articolare', 58), ('amico', 35), ('grazia', 33), ('ricordo', 28), ('pregare', 26), ('cordiale', 24), ('ringraziamento', 24), ('gentile', 22), ('sicuro', 22), ('grato', 19)]\n"
     ]
    }
   ],
   "source": [
    "print(\"10 most common positive words for 1885-1901:\")\n",
    "print(Counter(gpos_8501).most_common(10))\n",
    "print(\"10 most common positive words for 1902-1908:\")\n",
    "print(Counter(gpos_0208).most_common(10))\n",
    "print(\"10 most common positive words for 1909-1922:\")\n",
    "print(Counter(gpos_0922).most_common(10))\n",
    "print(\"10 most common positive words for 1923-1928:\")\n",
    "print(Counter(gpos_2328).most_common(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "bad_neg = ['caro', 'partire', 'signore', 'appena', 'trattare', 'piccolo', 'fabbricare', 'dispiacere', 'aspettare']\n",
    "\n",
    "gneg_8501 = [x for x in t8501 if x in neg and x not in bad_neg]\n",
    "gneg_0208 = [x for x in t0208 if x in neg and x not in bad_neg]\n",
    "gneg_0922 = [x for x in t0922 if x in neg and x not in bad_neg]\n",
    "gneg_2328 = [x for x in t2328 if x in neg and x not in bad_neg]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 most common negative words for 1885-1901:\n",
      "[('madre', 47), ('dolore', 45), ('dubbio', 39), ('soffrire', 37), ('gelosia', 35), ('partenza', 31), ('paura', 30), ('finora', 30), ('dimenticare', 30), ('bagnare', 29)]\n",
      "10 most common negative words for 1902-1908:\n",
      "[('peggio', 26), ('dimenticare', 24), ('servire', 22), ('bagnare', 20), ('partenza', 20), ('paura', 19), ('soffrire', 19), ('perdere', 18), ('sterlina', 16), ('cuocere', 15)]\n",
      "10 most common negative words for 1909-1922:\n",
      "[('bagnare', 41), ('colera', 31), ('malattia', 23), ('paura', 22), ('madre', 17), ('perdere', 17), ('soffrire', 16), ('finora', 15), ('servire', 14), ('dimenticare', 14)]\n",
      "10 most common negative words for 1923-1928:\n",
      "[('copia', 32), ('critico', 30), ('criticare', 25), ('perdere', 24), ('difficile', 21), ('grato', 19), ('dimenticare', 18), ('spesa', 18), ('taglio', 14), ('ristampare', 13)]\n"
     ]
    }
   ],
   "source": [
    "print(\"10 most common negative words for 1885-1901:\")\n",
    "print(Counter(gneg_8501).most_common(10))\n",
    "print(\"10 most common negative words for 1902-1908:\")\n",
    "print(Counter(gneg_0208).most_common(10))\n",
    "print(\"10 most common negative words for 1909-1922:\")\n",
    "print(Counter(gneg_0922).most_common(10))\n",
    "print(\"10 most common negative words for 1923-1928:\")\n",
    "print(Counter(gneg_2328).most_common(10))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}