SCOWL_wordlist.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SCOWL wordlist\n",
    "\n",
    "Create a definitive list of 'real' words that we can use across all our projects for deciding what to consider a word/non-word.\n",
    "\n",
    "Wordlist is based on the SCOWL set of word lists at http://wordlist.aspell.net/."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing necessary modules\n",
    "\n",
    "import pandas as pd\n",
    "import pickle as pkl\n",
    "import glob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import necessary files\n",
    "All the dictionary files are in the `SCOWL/final` folder from the SCOWL download (link above). Here we check them to see which ones to include, then use glob to create a final .txt file, i.e. a single long list of words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "347\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['../../../Documents/SCOWL/final/english-words.20',\n",
       " '../../../Documents/SCOWL/final/british-words.80',\n",
       " '../../../Documents/SCOWL/final/canadian-words.10',\n",
       " '../../../Documents/SCOWL/final/english-abbreviations.20',\n",
       " '../../../Documents/SCOWL/final/canadian_variant_2-words.10',\n",
       " '../../../Documents/SCOWL/final/british_variant_1-words.20',\n",
       " '../../../Documents/SCOWL/final/australian-words.40',\n",
       " '../../../Documents/SCOWL/final/variant_1-words.80',\n",
       " '../../../Documents/SCOWL/final/australian_variant_1-contractions.50',\n",
       " '../../../Documents/SCOWL/final/variant_2-abbreviations.70',\n",
       " '../../../Documents/SCOWL/final/variant_1-proper-names.80',\n",
       " '../../../Documents/SCOWL/final/canadian-abbreviations.95',\n",
       " '../../../Documents/SCOWL/final/canadian_variant_1-proper-names.95',\n",
       " '../../../Documents/SCOWL/final/variant_1-words.20',\n",
       " '../../../Documents/SCOWL/final/australian-upper.50',\n",
       " '../../../Documents/SCOWL/final/british_variant_1-words.80',\n",
       " '../../../Documents/SCOWL/final/english-abbreviations.80',\n",
       " '../../../Documents/SCOWL/final/australian_variant_1-contractions.35',\n",
       " '../../../Documents/SCOWL/final/english-proper-names.40',\n",
       " '../../../Documents/SCOWL/final/australian-abbreviations.80']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Checking the files\n",
    "path = \"../../../Documents/SCOWL/final/\" # Set path to wherever you have downloaded the SCOWL files\n",
    "all_file_names = glob.glob(path+\"*\")\n",
    "print(len(all_file_names))\n",
    "all_file_names[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'american-abbreviations',\n",
       " 'american-proper-names',\n",
       " 'american-upper',\n",
       " 'american-words',\n",
       " 'australian-abbreviations',\n",
       " 'australian-contractions',\n",
       " 'australian-proper-names',\n",
       " 'australian-upper',\n",
       " 'australian-words',\n",
       " 'australian_variant_1-abbreviations',\n",
       " 'australian_variant_1-contractions',\n",
       " 'australian_variant_1-proper-names',\n",
       " 'australian_variant_1-upper',\n",
       " 'australian_variant_1-words',\n",
       " 'australian_variant_2-abbreviations',\n",
       " 'australian_variant_2-contractions',\n",
       " 'australian_variant_2-proper-names',\n",
       " 'australian_variant_2-upper',\n",
       " 'australian_variant_2-words',\n",
       " 'british-abbreviations',\n",
       " 'british-proper-names',\n",
       " 'british-upper',\n",
       " 'british-words',\n",
       " 'british_variant_1-abbreviations',\n",
       " 'british_variant_1-contractions',\n",
       " 'british_variant_1-upper',\n",
       " 'british_variant_1-words',\n",
       " 'british_variant_2-abbreviations',\n",
       " 'british_variant_2-contractions',\n",
       " 'british_variant_2-upper',\n",
       " 'british_variant_2-words',\n",
       " 'british_z-abbreviations',\n",
       " 'british_z-proper-names',\n",
       " 'british_z-upper',\n",
       " 'british_z-words',\n",
       " 'canadian-abbreviations',\n",
       " 'canadian-proper-names',\n",
       " 'canadian-upper',\n",
       " 'canadian-words',\n",
       " 'canadian_variant_1-abbreviations',\n",
       " 'canadian_variant_1-contractions',\n",
       " 'canadian_variant_1-proper-names',\n",
       " 'canadian_variant_1-upper',\n",
       " 'canadian_variant_1-words',\n",
       " 'canadian_variant_2-abbreviations',\n",
       " 'canadian_variant_2-contractions',\n",
       " 'canadian_variant_2-upper',\n",
       " 'canadian_variant_2-words',\n",
       " 'english-abbreviations',\n",
       " 'english-contractions',\n",
       " 'english-proper-names',\n",
       " 'english-upper',\n",
       " 'english-words',\n",
       " 'special-hacker',\n",
       " 'special-roman-numerals',\n",
       " 'variant_1-abbreviations',\n",
       " 'variant_1-contractions',\n",
       " 'variant_1-proper-names',\n",
       " 'variant_1-upper',\n",
       " 'variant_1-words',\n",
       " 'variant_2-abbreviations',\n",
       " 'variant_2-contractions',\n",
       " 'variant_2-upper',\n",
       " 'variant_2-words',\n",
       " 'variant_3-abbreviations',\n",
       " 'variant_3-words'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_types = set([x[31:-3] for x in all_file_names])\n",
    "print(len(file_types))\n",
    "file_types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'words', 'hacker', 'abbreviations', 'contractions', 'names', 'numerals', 'upper'}\n"
     ]
    }
   ],
   "source": [
    "file_categories = [x.split('-') for x in file_types]\n",
    "file_categories = set([x[-1] for x in file_categories])\n",
    "print(file_categories)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are 7 different categories so we will manually look into each, checking different regional varieties, to decide whether to include in the final SCOWL list."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Abbreviations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AAAA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AAAAAA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AAAL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AAAS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AAE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5108</th>\n",
       "      <td>wrnt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5109</th>\n",
       "      <td>xd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5110</th>\n",
       "      <td>xdiv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5111</th>\n",
       "      <td>yrbk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5112</th>\n",
       "      <td>zod</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5113 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        word\n",
       "0       AAAA\n",
       "1     AAAAAA\n",
       "2       AAAL\n",
       "3       AAAS\n",
       "4        AAE\n",
       "...      ...\n",
       "5108    wrnt\n",
       "5109      xd\n",
       "5110    xdiv\n",
       "5111    yrbk\n",
       "5112     zod\n",
       "\n",
       "[5113 rows x 1 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(path+'american-abbreviations.70',names=['word'])\n",
    "pd.read_csv(path+'australian-abbreviations.70',names=['word'])\n",
    "pd.read_csv(path+'british-abbreviations.95',names=['word'])\n",
    "pd.read_csv(path+'canadian-abbreviations.80',names=['word'])\n",
    "pd.read_csv(path+'english-abbreviations.95',names=['word']) # The 'english' lists are the big ones"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "My vote is to remove all of the abbreviations from our list - these are more likely to be misspellings than real words when used by the students."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### contractions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a'body</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a'thing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cowslip'd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d'Indy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d'accord</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>dog'sbane</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>entr'actes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>fa'ard</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>fatwa'd</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>freez'd</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         word\n",
       "0      a'body\n",
       "1     a'thing\n",
       "2   cowslip'd\n",
       "3      d'Indy\n",
       "4    d'accord\n",
       "5   dog'sbane\n",
       "6  entr'actes\n",
       "7      fa'ard\n",
       "8     fatwa'd\n",
       "9     freez'd"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Weirdly, no list for US English - maybe just part of the big 'English' list\n",
    "pd.read_csv(path+'australian-contractions.35',names=['word']) #hahaha love it\n",
    "pd.read_csv(path+'british_variant_2-contractions.70',names=['word'])\n",
    "pd.read_csv(path+'canadian_variant_1-contractions.35',names=['word']) #Interesting\n",
    "pd.read_csv(path+'english-contractions.80',names=['word']).head(10) # The 'english' lists are the big ones"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Keep contractions. Probably not many occurrences, but shouldn't be corrected and are unlikely to be errors."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### hacker (special hacker words!)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1629</th>\n",
       "      <td>zap</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1630</th>\n",
       "      <td>zapped</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1631</th>\n",
       "      <td>zapping</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1632</th>\n",
       "      <td>zaps</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1633</th>\n",
       "      <td>zen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1634</th>\n",
       "      <td>zenned</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1635</th>\n",
       "      <td>zens</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1636</th>\n",
       "      <td>zero</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1637</th>\n",
       "      <td>zeroed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1638</th>\n",
       "      <td>zeroes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1639</th>\n",
       "      <td>zeroing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1640</th>\n",
       "      <td>zeros</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1641</th>\n",
       "      <td>zeroth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1642</th>\n",
       "      <td>zip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1643</th>\n",
       "      <td>zipped</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1644</th>\n",
       "      <td>zipping</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1645</th>\n",
       "      <td>zips</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1646</th>\n",
       "      <td>zombie</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1647</th>\n",
       "      <td>zombies</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1648</th>\n",
       "      <td>zorch</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         word\n",
       "1629      zap\n",
       "1630   zapped\n",
       "1631  zapping\n",
       "1632     zaps\n",
       "1633      zen\n",
       "1634   zenned\n",
       "1635     zens\n",
       "1636     zero\n",
       "1637   zeroed\n",
       "1638   zeroes\n",
       "1639  zeroing\n",
       "1640    zeros\n",
       "1641   zeroth\n",
       "1642      zip\n",
       "1643   zipped\n",
       "1644  zipping\n",
       "1645     zips\n",
       "1646   zombie\n",
       "1647  zombies\n",
       "1648    zorch"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Just one file\n",
    "pd.read_csv(path+'special-hacker.50',names=['word']).head(20)\n",
    "pd.read_csv(path+'special-hacker.50',names=['word']).tail(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I'm in favour of keeping these but am ambivalent - there are a mix of abbreviations and real words. Even though we didn't include other abbreivations, these are all pretty common, and there may be words that will not appear in other lists related to technology."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(path+'american-proper-names.80',names=['word']).sample(5)\n",
    "len(pd.read_csv(path+'american-proper-names.80',names=['word']))\n",
    "\n",
    "pd.read_csv(path+'australian-proper-names.50',names=['word']).sample(5)\n",
    "len(pd.read_csv(path+'australian-proper-names.50',names=['word']))\n",
    "\n",
    "pd.read_csv(path+'british-proper-names.95',names=['word']).sample(5)\n",
    "len(pd.read_csv(path+'british-proper-names.95',names=['word']))\n",
    "\n",
    "pd.read_csv(path+'canadian-proper-names.80',names=['word']).sample(5)\n",
    "len(pd.read_csv(path+'canadian-proper-names.80',names=['word']))\n",
    "\n",
    "pd.read_csv(path+'english-proper-names.40',names=['word'])\n",
    "len(pd.read_csv(path+'english-proper-names.40',names=['word']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Definitely keep - proper names is one important element lacking in some other word lists, and our POS tagging definitely hasn't caugh all the proper nouns in our texts. We will need to remove the possessives (reducing the lists significantly). There are some words here which are not names, e.g. verbs, but that's fine."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### numerals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "63"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Just one file\n",
    "\n",
    "pd.read_csv(path+'special-roman-numerals.35',names=['word']).sample(5)\n",
    "len(pd.read_csv(path+'special-roman-numerals.35',names=['word']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Keep roman numerals - we may end up with a few false negatives where it's actually a spelling mistake we miss, but better that than accidentally correcting an intended roman numeral."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### upper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(path+'american-upper.95',names=['word']).sample(5)\n",
    "len(pd.read_csv(path+'american-upper.95',names=['word']))\n",
    "\n",
    "pd.read_csv(path+'australian-upper.80',names=['word']).sample(5)\n",
    "len(pd.read_csv(path+'australian-upper.80',names=['word']))\n",
    "\n",
    "pd.read_csv(path+'british_z-upper.50',names=['word']).sample(5)\n",
    "len(pd.read_csv(path+'british_z-upper.50',names=['word']))\n",
    "\n",
    "pd.read_csv(path+'canadian_variant_2-upper.40',names=['word']).sample(5)\n",
    "len(pd.read_csv(path+'canadian_variant_2-upper.40',names=['word']))\n",
    "\n",
    "pd.read_csv(path+'english-upper.10',names=['word']).sample(5)\n",
    "len(pd.read_csv(path+'english-upper.10',names=['word']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not sure how these are different than the 'name' category, but we can definitely keep."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(path+'american-words.95',names=['word']).sample(5) # All the 'z' spellings - good\n",
    "len(pd.read_csv(path+'american-words.95',names=['word']))\n",
    "\n",
    "pd.read_csv(path+'australian_variant_2-words.40',names=['word']).sample(5)\n",
    "len(pd.read_csv(path+'australian_variant_2-words.40',names=['word']))\n",
    "\n",
    "pd.read_csv(path+'british-words.55',names=['word']).sample(5)\n",
    "len(pd.read_csv(path+'british-words.55',names=['word']))\n",
    "\n",
    "pd.read_csv(path+'canadian-words.70',names=['word']).sample(5)\n",
    "len(pd.read_csv(path+'canadian-words.70',names=['word']))\n",
    "\n",
    "pd.read_csv(path+'english-words.80',names=['word'], encoding = 'latin-1').sample(5) # Big files\n",
    "len(pd.read_csv(path+'english-words.80',names=['word'], encoding = 'latin-1'))\n",
    "\n",
    "pd.read_csv(path+'variant_3-words.60',names=['word']).sample(5) # Generic 'variant' whatever that is!\n",
    "len(pd.read_csv(path+'variant_3-words.60',names=['word']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These are the main lists that need to be included."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Summary of checking:\n",
    "Keep all except the abbreviation files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "exclude = [x for x in all_file_names if 'abbreviations' in x]\n",
    "include = [x for x in all_file_names if 'abbreviations' not in x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "296"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_file_names)\n",
    "len(exclude)\n",
    "len(include)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compiling files\n",
    "#### Reading in SCOWL files\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "\n",
    "# Read in and combine all of our text files\n",
    "\n",
    "with open(\"SCOWL_wordlist.txt\", \"wb\") as outfile:\n",
    "    for f in include:\n",
    "        with open(f, \"rb\") as infile:\n",
    "            outfile.write(infile.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "726845"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read in the new SCOWL_wordlist to manipulate\n",
    "text_file = open(\"SCOWL_wordlist.txt\", \"r\", encoding = 'latin-1')\n",
    "SCOWL = text_file.read().split('\\n')\n",
    "SCOWL[:10]\n",
    "len(SCOWL)\n",
    "# Output = a single tokenized list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "668130"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Turn into set to save space and remove duplicates\n",
    "SCOWL = set(SCOWL)\n",
    "len(SCOWL)\n",
    "\n",
    "# Removed 60k items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'poutine' in SCOWL\n",
    "\n",
    "# Good - the Canadian dict worked!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remove any blanks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "668129"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SCOWL = [x for x in SCOWL if len(x) > 0]\n",
    "len(SCOWL)\n",
    "\n",
    "#1 blank removed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 's\n",
    "From earlier, we know that there are a lot of possessives of nouns. With our tokenization splitting the 's, we don't need these in this list as well."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"Ben's\" in SCOWL\n",
    "\"Ben\" in SCOWL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "147872"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[x for x in SCOWL if x[-2:] == \"'s\"][:10]\n",
    "len([x for x in SCOWL if x[-2:] == \"'s\"])\n",
    "\n",
    "# Interesting proper names!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "520257"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove these items\n",
    "SCOWL = set([x for x in SCOWL if x[-2:] != \"'s\"])\n",
    "len(SCOWL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Final list is almost exactly approximately 520k words."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write out SCOWL.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Change SCOWL back to a sorted list\n",
    "SCOWL = sorted(list(SCOWL))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pickle it for convenience\n",
    "with open('SCOWL_condensed.pkl', 'wb') as f:\n",
    "    pkl.dump(SCOWL, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "\n",
    "# Create a text file - takes a while\n",
    "with open('SCOWL_condensed.txt', 'w') as f:\n",
    "    for item in SCOWL:\n",
    "        f.write(\"%s\\n\" % item)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Names experiment\n",
    "Creating a list of names to add to our 'safe' list when spell checking\n",
    "\n",
    "Names from 1990 US census data, collected by https://pypi.org/project/names/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../../../Documents/names_dataset/names/\" # Set path to wherever you have downloaded the names files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1219\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>male_first</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>JAMES          3.318  3.318      1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>JOHN           3.271  6.589      2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ROBERT         3.143  9.732      3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MICHAEL        2.629 12.361      4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>WILLIAM        2.451 14.812      5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           male_first\n",
       "0  JAMES          3.318  3.318      1\n",
       "1  JOHN           3.271  6.589      2\n",
       "2  ROBERT         3.143  9.732      3\n",
       "3  MICHAEL        2.629 12.361      4\n",
       "4  WILLIAM        2.451 14.812      5"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "male_first = pd.read_csv(path+'dist.male.first',names=['male_first'])\n",
    "print(len(male_first))\n",
    "male_first.head()\n",
    "\n",
    "# 1200 Male names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4275\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>female_first</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>MARY           2.629  2.629      1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>PATRICIA       1.073  3.702      2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LINDA          1.035  4.736      3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BARBARA        0.980  5.716      4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ELIZABETH      0.937  6.653      5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         female_first\n",
       "0  MARY           2.629  2.629      1\n",
       "1  PATRICIA       1.073  3.702      2\n",
       "2  LINDA          1.035  4.736      3\n",
       "3  BARBARA        0.980  5.716      4\n",
       "4  ELIZABETH      0.937  6.653      5"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "female_first = pd.read_csv(path+'dist.female.first',names=['female_first'])\n",
    "print(len(female_first))\n",
    "female_first.head()\n",
    "\n",
    "# 4275 female first names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "88799\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>last_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SMITH          1.006  1.006      1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>JOHNSON        0.810  1.816      2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>WILLIAMS       0.699  2.515      3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>JONES          0.621  3.136      4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>BROWN          0.621  3.757      5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            last_name\n",
       "0  SMITH          1.006  1.006      1\n",
       "1  JOHNSON        0.810  1.816      2\n",
       "2  WILLIAMS       0.699  2.515      3\n",
       "3  JONES          0.621  3.136      4\n",
       "4  BROWN          0.621  3.757      5"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_name = pd.read_csv(path+'dist.all.last',names=['last_name'])\n",
    "print(len(last_name))\n",
    "last_name.head()\n",
    "\n",
    "# 88799 last names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combining all these names into one list\n",
    "male_first_list = [x[0] for x in male_first.male_first.apply(lambda x: x.split(' '))]\n",
    "female_first_list = [x[0] for x in female_first.female_first.apply(lambda x: x.split(' '))]\n",
    "last_name_list = [x[0] for x in last_name.last_name.apply(lambda x: x.split(' '))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_names = set(male_first_list + female_first_list + last_name_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "91910"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "\n",
    "# Create a text file\n",
    "with open('all_names.txt', 'w') as f:\n",
    "    for item in all_names:\n",
    "        f.write(\"%s\\n\" % item)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Back to top](#SCOWL-wordlist)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}