tf_kdd99.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# T81-558: Applications of Deep Neural Networks\n",
    "### TensorFlow (SKFLOW) Meets KDD-99\n",
    "* Instructor: [Jeff Heaton](https://sites.wustl.edu/jeffheaton/), School of Engineering and Applied Science, [Washington University in St. Louis](https://engineering.wustl.edu/Programs/Pages/default.aspx)\n",
    "* For more information visit the [class website](https://sites.wustl.edu/jeffheaton/t81-558/).\n",
    "\n",
    "This simple example shows how to load a non-trivial dataset from CSV and train a neural network.  The dataset is the\n",
    "[KDD99 dataset](http://kdd.ics.uci.edu/databases/kddcup99/kddcup99.html).  This dataset is used to detect between normal and malicious network activity.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Imports for this Notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Imports\n",
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "from sklearn.cross_validation import train_test_split\n",
    "import tensorflow.contrib.learn as skflow\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Several Useful Functions\n",
    "\n",
    "These are functions that I reuse often to encode the feature vector (FV)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# These are several handy functions that I use in my class:\n",
    "\n",
    "# Encode a text field to dummy variables\n",
    "def encode_text_dummy(df,name):\n",
    "    dummies = pd.get_dummies(df[name])\n",
    "    for x in dummies.columns:\n",
    "        dummy_name = \"{}-{}\".format(name,x)\n",
    "        df[dummy_name] = dummies[x]\n",
    "    df.drop(name, axis=1, inplace=True)\n",
    "    \n",
    "# Encode a text field to a single index value\n",
    "def encode_text_index(df,name): \n",
    "    le = preprocessing.LabelEncoder()\n",
    "    df[name] = le.fit_transform(df[name])\n",
    "    return le.classes_\n",
    "                \n",
    "# Encode a numeric field to Z-Scores\n",
    "def encode_numeric_zscore(df,name,mean=None,sd=None):\n",
    "    if mean is None:\n",
    "        mean = df[name].mean()\n",
    "        \n",
    "    if sd is None:\n",
    "        sd = df[name].std()\n",
    "        \n",
    "    df[name] = (df[name]-mean)/sd\n",
    "    \n",
    "# Encode a numeric field to fill missing values with the median.\n",
    "def missing_median(df, name):\n",
    "    med = df[name].median()\n",
    "    df[name] = df[name].fillna(med)\n",
    "\n",
    "# Convert a dataframe to x/y suitable for training.\n",
    "def to_xy(df,target):\n",
    "    result = []\n",
    "    for x in df.columns:\n",
    "        if x != target:\n",
    "            result.append(x)\n",
    "    return df.as_matrix(result),df[target]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read in Raw KDD-99 Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read 494021 rows.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>protocol_type</th>\n",
       "      <th>service</th>\n",
       "      <th>flag</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>...</th>\n",
       "      <th>dst_host_srv_count</th>\n",
       "      <th>dst_host_same_srv_rate</th>\n",
       "      <th>dst_host_diff_srv_rate</th>\n",
       "      <th>dst_host_same_src_port_rate</th>\n",
       "      <th>dst_host_srv_diff_host_rate</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>dst_host_srv_serror_rate</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "      <th>dst_host_srv_rerror_rate</th>\n",
       "      <th>outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>309974</th>\n",
       "      <td>0</td>\n",
       "      <td>icmp</td>\n",
       "      <td>ecr_i</td>\n",
       "      <td>SF</td>\n",
       "      <td>1032</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>255</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>smurf.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293952</th>\n",
       "      <td>0</td>\n",
       "      <td>icmp</td>\n",
       "      <td>ecr_i</td>\n",
       "      <td>SF</td>\n",
       "      <td>1032</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>255</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>smurf.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211354</th>\n",
       "      <td>0</td>\n",
       "      <td>icmp</td>\n",
       "      <td>ecr_i</td>\n",
       "      <td>SF</td>\n",
       "      <td>1032</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>255</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>smurf.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302812</th>\n",
       "      <td>0</td>\n",
       "      <td>icmp</td>\n",
       "      <td>ecr_i</td>\n",
       "      <td>SF</td>\n",
       "      <td>1032</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>255</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>smurf.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>438732</th>\n",
       "      <td>0</td>\n",
       "      <td>icmp</td>\n",
       "      <td>ecr_i</td>\n",
       "      <td>SF</td>\n",
       "      <td>520</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>255</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>smurf.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        duration protocol_type service flag  src_bytes  dst_bytes  land  \\\n",
       "309974         0          icmp   ecr_i   SF       1032          0     0   \n",
       "293952         0          icmp   ecr_i   SF       1032          0     0   \n",
       "211354         0          icmp   ecr_i   SF       1032          0     0   \n",
       "302812         0          icmp   ecr_i   SF       1032          0     0   \n",
       "438732         0          icmp   ecr_i   SF        520          0     0   \n",
       "\n",
       "        wrong_fragment  urgent  hot   ...     dst_host_srv_count  \\\n",
       "309974               0       0    0   ...                    255   \n",
       "293952               0       0    0   ...                    255   \n",
       "211354               0       0    0   ...                    255   \n",
       "302812               0       0    0   ...                    255   \n",
       "438732               0       0    0   ...                    255   \n",
       "\n",
       "        dst_host_same_srv_rate  dst_host_diff_srv_rate  \\\n",
       "309974                     1.0                     0.0   \n",
       "293952                     1.0                     0.0   \n",
       "211354                     1.0                     0.0   \n",
       "302812                     1.0                     0.0   \n",
       "438732                     1.0                     0.0   \n",
       "\n",
       "        dst_host_same_src_port_rate  dst_host_srv_diff_host_rate  \\\n",
       "309974                          1.0                          0.0   \n",
       "293952                          1.0                          0.0   \n",
       "211354                          1.0                          0.0   \n",
       "302812                          1.0                          0.0   \n",
       "438732                          1.0                          0.0   \n",
       "\n",
       "        dst_host_serror_rate  dst_host_srv_serror_rate  dst_host_rerror_rate  \\\n",
       "309974                   0.0                       0.0                   0.0   \n",
       "293952                   0.0                       0.0                   0.0   \n",
       "211354                   0.0                       0.0                   0.0   \n",
       "302812                   0.0                       0.0                   0.0   \n",
       "438732                   0.0                       0.0                   0.0   \n",
       "\n",
       "        dst_host_srv_rerror_rate  outcome  \n",
       "309974                       0.0   smurf.  \n",
       "293952                       0.0   smurf.  \n",
       "211354                       0.0   smurf.  \n",
       "302812                       0.0   smurf.  \n",
       "438732                       0.0   smurf.  \n",
       "\n",
       "[5 rows x 42 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# This file is a CSV, just no CSV extension or headers\n",
    "# Download from: http://kdd.ics.uci.edu/databases/kddcup99/kddcup99.html\n",
    "df = pd.read_csv(\"/Users/jeff/Downloads/data/kddcup.data_10_percent\", header=None)\n",
    "\n",
    "print(\"Read {} rows.\".format(len(df)))\n",
    "# df = df.sample(frac=0.1, replace=False) # Uncomment this line to sample only 10% of the dataset\n",
    "df.dropna(inplace=True,axis=1) # For now, just drop NA's (rows with missing values)\n",
    "\n",
    "# The CSV file has no column heads, so add them\n",
    "df.columns = [\n",
    "    'duration',\n",
    "    'protocol_type',\n",
    "    'service',\n",
    "    'flag',\n",
    "    'src_bytes',\n",
    "    'dst_bytes',\n",
    "    'land',\n",
    "    'wrong_fragment',\n",
    "    'urgent',\n",
    "    'hot',\n",
    "    'num_failed_logins',\n",
    "    'logged_in',\n",
    "    'num_compromised',\n",
    "    'root_shell',\n",
    "    'su_attempted',\n",
    "    'num_root',\n",
    "    'num_file_creations',\n",
    "    'num_shells',\n",
    "    'num_access_files',\n",
    "    'num_outbound_cmds',\n",
    "    'is_host_login',\n",
    "    'is_guest_login',\n",
    "    'count',\n",
    "    'srv_count',\n",
    "    'serror_rate',\n",
    "    'srv_serror_rate',\n",
    "    'rerror_rate',\n",
    "    'srv_rerror_rate',\n",
    "    'same_srv_rate',\n",
    "    'diff_srv_rate',\n",
    "    'srv_diff_host_rate',\n",
    "    'dst_host_count',\n",
    "    'dst_host_srv_count',\n",
    "    'dst_host_same_srv_rate',\n",
    "    'dst_host_diff_srv_rate',\n",
    "    'dst_host_same_src_port_rate',\n",
    "    'dst_host_srv_diff_host_rate',\n",
    "    'dst_host_serror_rate',\n",
    "    'dst_host_srv_serror_rate',\n",
    "    'dst_host_rerror_rate',\n",
    "    'dst_host_srv_rerror_rate',\n",
    "    'outcome'\n",
    "]\n",
    "\n",
    "# display 5 rows\n",
    "df[0:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Encode the feature vector\n",
    "Encode every row in the database.  This is not instant!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>num_compromised</th>\n",
       "      <th>root_shell</th>\n",
       "      <th>su_attempted</th>\n",
       "      <th>num_root</th>\n",
       "      <th>...</th>\n",
       "      <th>flag-S2</th>\n",
       "      <th>flag-SF</th>\n",
       "      <th>flag-SH</th>\n",
       "      <th>land-0</th>\n",
       "      <th>land-1</th>\n",
       "      <th>logged_in-0</th>\n",
       "      <th>logged_in-1</th>\n",
       "      <th>is_host_login-0</th>\n",
       "      <th>is_guest_login-0</th>\n",
       "      <th>is_guest_login-1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>309974</th>\n",
       "      <td>-0.065862</td>\n",
       "      <td>-0.013753</td>\n",
       "      <td>-0.029016</td>\n",
       "      <td>-0.049704</td>\n",
       "      <td>-0.043171</td>\n",
       "      <td>-0.007793</td>\n",
       "      <td>-0.008656</td>\n",
       "      <td>-0.008999</td>\n",
       "      <td>-0.004499</td>\n",
       "      <td>-0.009776</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293952</th>\n",
       "      <td>-0.065862</td>\n",
       "      <td>-0.013753</td>\n",
       "      <td>-0.029016</td>\n",
       "      <td>-0.049704</td>\n",
       "      <td>-0.043171</td>\n",
       "      <td>-0.007793</td>\n",
       "      <td>-0.008656</td>\n",
       "      <td>-0.008999</td>\n",
       "      <td>-0.004499</td>\n",
       "      <td>-0.009776</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211354</th>\n",
       "      <td>-0.065862</td>\n",
       "      <td>-0.013753</td>\n",
       "      <td>-0.029016</td>\n",
       "      <td>-0.049704</td>\n",
       "      <td>-0.043171</td>\n",
       "      <td>-0.007793</td>\n",
       "      <td>-0.008656</td>\n",
       "      <td>-0.008999</td>\n",
       "      <td>-0.004499</td>\n",
       "      <td>-0.009776</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302812</th>\n",
       "      <td>-0.065862</td>\n",
       "      <td>-0.013753</td>\n",
       "      <td>-0.029016</td>\n",
       "      <td>-0.049704</td>\n",
       "      <td>-0.043171</td>\n",
       "      <td>-0.007793</td>\n",
       "      <td>-0.008656</td>\n",
       "      <td>-0.008999</td>\n",
       "      <td>-0.004499</td>\n",
       "      <td>-0.009776</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>438732</th>\n",
       "      <td>-0.065862</td>\n",
       "      <td>-0.020571</td>\n",
       "      <td>-0.029016</td>\n",
       "      <td>-0.049704</td>\n",
       "      <td>-0.043171</td>\n",
       "      <td>-0.007793</td>\n",
       "      <td>-0.008656</td>\n",
       "      <td>-0.008999</td>\n",
       "      <td>-0.004499</td>\n",
       "      <td>-0.009776</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 115 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        duration  src_bytes  dst_bytes  wrong_fragment       hot  \\\n",
       "309974 -0.065862  -0.013753  -0.029016       -0.049704 -0.043171   \n",
       "293952 -0.065862  -0.013753  -0.029016       -0.049704 -0.043171   \n",
       "211354 -0.065862  -0.013753  -0.029016       -0.049704 -0.043171   \n",
       "302812 -0.065862  -0.013753  -0.029016       -0.049704 -0.043171   \n",
       "438732 -0.065862  -0.020571  -0.029016       -0.049704 -0.043171   \n",
       "\n",
       "        num_failed_logins  num_compromised  root_shell  su_attempted  \\\n",
       "309974          -0.007793        -0.008656   -0.008999     -0.004499   \n",
       "293952          -0.007793        -0.008656   -0.008999     -0.004499   \n",
       "211354          -0.007793        -0.008656   -0.008999     -0.004499   \n",
       "302812          -0.007793        -0.008656   -0.008999     -0.004499   \n",
       "438732          -0.007793        -0.008656   -0.008999     -0.004499   \n",
       "\n",
       "        num_root        ...         flag-S2  flag-SF  flag-SH  land-0  land-1  \\\n",
       "309974 -0.009776        ...             0.0      1.0      0.0     1.0     0.0   \n",
       "293952 -0.009776        ...             0.0      1.0      0.0     1.0     0.0   \n",
       "211354 -0.009776        ...             0.0      1.0      0.0     1.0     0.0   \n",
       "302812 -0.009776        ...             0.0      1.0      0.0     1.0     0.0   \n",
       "438732 -0.009776        ...             0.0      1.0      0.0     1.0     0.0   \n",
       "\n",
       "        logged_in-0  logged_in-1  is_host_login-0  is_guest_login-0  \\\n",
       "309974          1.0          0.0              1.0               1.0   \n",
       "293952          1.0          0.0              1.0               1.0   \n",
       "211354          1.0          0.0              1.0               1.0   \n",
       "302812          1.0          0.0              1.0               1.0   \n",
       "438732          1.0          0.0              1.0               1.0   \n",
       "\n",
       "        is_guest_login-1  \n",
       "309974               0.0  \n",
       "293952               0.0  \n",
       "211354               0.0  \n",
       "302812               0.0  \n",
       "438732               0.0  \n",
       "\n",
       "[5 rows x 115 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now encode the feature vector\n",
    "\n",
    "encode_numeric_zscore(df, 'duration')\n",
    "encode_text_dummy(df, 'protocol_type')\n",
    "encode_text_dummy(df, 'service')\n",
    "encode_text_dummy(df, 'flag')\n",
    "encode_numeric_zscore(df, 'src_bytes')\n",
    "encode_numeric_zscore(df, 'dst_bytes')\n",
    "encode_text_dummy(df, 'land')\n",
    "encode_numeric_zscore(df, 'wrong_fragment')\n",
    "encode_numeric_zscore(df, 'urgent')\n",
    "encode_numeric_zscore(df, 'hot')\n",
    "encode_numeric_zscore(df, 'num_failed_logins')\n",
    "encode_text_dummy(df, 'logged_in')\n",
    "encode_numeric_zscore(df, 'num_compromised')\n",
    "encode_numeric_zscore(df, 'root_shell')\n",
    "encode_numeric_zscore(df, 'su_attempted')\n",
    "encode_numeric_zscore(df, 'num_root')\n",
    "encode_numeric_zscore(df, 'num_file_creations')\n",
    "encode_numeric_zscore(df, 'num_shells')\n",
    "encode_numeric_zscore(df, 'num_access_files')\n",
    "encode_numeric_zscore(df, 'num_outbound_cmds')\n",
    "encode_text_dummy(df, 'is_host_login')\n",
    "encode_text_dummy(df, 'is_guest_login')\n",
    "encode_numeric_zscore(df, 'count')\n",
    "encode_numeric_zscore(df, 'srv_count')\n",
    "encode_numeric_zscore(df, 'serror_rate')\n",
    "encode_numeric_zscore(df, 'srv_serror_rate')\n",
    "encode_numeric_zscore(df, 'rerror_rate')\n",
    "encode_numeric_zscore(df, 'srv_rerror_rate')\n",
    "encode_numeric_zscore(df, 'same_srv_rate')\n",
    "encode_numeric_zscore(df, 'diff_srv_rate')\n",
    "encode_numeric_zscore(df, 'srv_diff_host_rate')\n",
    "encode_numeric_zscore(df, 'dst_host_count')\n",
    "encode_numeric_zscore(df, 'dst_host_srv_count')\n",
    "encode_numeric_zscore(df, 'dst_host_same_srv_rate')\n",
    "encode_numeric_zscore(df, 'dst_host_diff_srv_rate')\n",
    "encode_numeric_zscore(df, 'dst_host_same_src_port_rate')\n",
    "encode_numeric_zscore(df, 'dst_host_srv_diff_host_rate')\n",
    "encode_numeric_zscore(df, 'dst_host_serror_rate')\n",
    "encode_numeric_zscore(df, 'dst_host_srv_serror_rate')\n",
    "encode_numeric_zscore(df, 'dst_host_rerror_rate')\n",
    "encode_numeric_zscore(df, 'dst_host_srv_rerror_rate')\n",
    "outcomes = encode_text_index(df, 'outcome')\n",
    "num_classes = len(outcomes)\n",
    "\n",
    "# display 5 rows\n",
    "\n",
    "df.dropna(inplace=True,axis=1)\n",
    "df[0:5]\n",
    "# This is the numeric feature vector, as it goes to the neural net\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step #49, avg. train loss: 0.42411, avg. val loss: 0.38433\n",
      "Step #99, avg. train loss: 0.09951, avg. val loss: 0.10089\n",
      "Step #149, avg. train loss: 0.09113, avg. val loss: 0.08418\n",
      "Step #199, avg. train loss: 0.08270, avg. val loss: 0.07179\n",
      "Step #249, avg. train loss: 0.05196, avg. val loss: 0.06019\n",
      "Step #299, avg. train loss: 0.06040, avg. val loss: 0.05594\n",
      "Step #349, avg. train loss: 0.06639, avg. val loss: 0.05264\n",
      "Step #399, avg. train loss: 0.06915, avg. val loss: 0.05109\n",
      "Step #449, avg. train loss: 0.03100, avg. val loss: 0.04427\n",
      "Step #499, avg. train loss: 0.04303, avg. val loss: 0.04306\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TensorFlowDNNClassifier(batch_size=32, class_weight=None, clip_gradients=5.0,\n",
       "            config=None, continue_training=False, dropout=None,\n",
       "            hidden_units=[10, 20, 10], learning_rate=0.1, n_classes=19,\n",
       "            optimizer='Adagrad', steps=500, verbose=1)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Break into X (predictors) & y (prediction)\n",
    "x, y = to_xy(df,'outcome')\n",
    "\n",
    "# Create a test/train split.  25% test\n",
    "# Split into train/test\n",
    "x_train, x_test, y_train, y_test = train_test_split(\n",
    "    x, y, test_size=0.25, random_state=42)\n",
    "\n",
    "# Create a deep neural network with 3 hidden layers of 10, 20, 10\n",
    "classifier = skflow.TensorFlowDNNClassifier(hidden_units=[10, 20, 10], \n",
    "    n_classes=num_classes, steps=500)\n",
    "\n",
    "# Early stopping\n",
    "early_stop = skflow.monitors.ValidationMonitor(x_test, y_test,\n",
    "                                                early_stopping_rounds=200,\n",
    "                                                n_classes=num_classes,\n",
    "                                                print_steps=50)\n",
    "        \n",
    "# Fit/train neural network\n",
    "classifier.fit(x, y, early_stop)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation score: 0.9872075135616549\n"
     ]
    }
   ],
   "source": [
    "# Measure accuracy\n",
    "pred = classifier.predict(x_test)\n",
    "score = metrics.accuracy_score(y_test, pred)\n",
    "print(\"Validation score: {}\".format(score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}