Day6_3_Self_Attention.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Day6_3_Self_Attention.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"code","metadata":{"id":"Vg97P6ouu20p","colab_type":"code","outputId":"3e22cb80-7dc9-40b8-fbe9-5798ed06a65a","executionInfo":{"status":"ok","timestamp":1562364011515,"user_tz":-540,"elapsed":80189,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":642}},"source":["!pip install --upgrade tensorflow-gpu==1.13.0rc1"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Collecting tensorflow-gpu==1.13.0rc1\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ac/2d/6b9b115b0d507b885a0766717c324fb71fce6cb6380e67fe24db1ab699e7/tensorflow_gpu-1.13.0rc1-cp36-cp36m-manylinux1_x86_64.whl (345.2MB)\n","\u001b[K     |████████████████████████████████| 345.2MB 60kB/s \n","\u001b[?25hRequirement already satisfied, skipping upgrade: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (1.1.0)\n","Requirement already satisfied, skipping upgrade: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (1.16.4)\n","Requirement already satisfied, skipping upgrade: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (1.15.0)\n","Requirement already satisfied, skipping upgrade: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (3.7.1)\n","Collecting tensorflow-estimator<1.14.0rc0,>=1.13.0rc0 (from tensorflow-gpu==1.13.0rc1)\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bb/48/13f49fc3fa0fdf916aa1419013bb8f2ad09674c275b4046d5ee669a46873/tensorflow_estimator-1.13.0-py2.py3-none-any.whl (367kB)\n","\u001b[K     |████████████████████████████████| 368kB 38.7MB/s \n","\u001b[?25hRequirement already satisfied, skipping upgrade: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (0.2.2)\n","Requirement already satisfied, skipping upgrade: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (1.0.8)\n","Requirement already satisfied, skipping upgrade: absl-py>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (0.7.1)\n","Collecting tensorboard<1.13.0,>=1.12.0 (from tensorflow-gpu==1.13.0rc1)\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/07/53/8d32ce9471c18f8d99028b7cef2e5b39ea8765bd7ef250ca05b490880971/tensorboard-1.12.2-py3-none-any.whl (3.0MB)\n","\u001b[K     |████████████████████████████████| 3.1MB 20.4MB/s \n","\u001b[?25hRequirement already satisfied, skipping upgrade: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (0.33.4)\n","Requirement already satisfied, skipping upgrade: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (1.1.0)\n","Requirement already satisfied, skipping upgrade: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (1.12.0)\n","Requirement already satisfied, skipping upgrade: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (0.8.0)\n","Requirement already satisfied, skipping upgrade: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow-gpu==1.13.0rc1) (41.0.1)\n","Collecting mock>=2.0.0 (from tensorflow-estimator<1.14.0rc0,>=1.13.0rc0->tensorflow-gpu==1.13.0rc1)\n","  Downloading https://files.pythonhosted.org/packages/05/d2/f94e68be6b17f46d2c353564da56e6fb89ef09faeeff3313a046cb810ca9/mock-3.0.5-py2.py3-none-any.whl\n","Requirement already satisfied, skipping upgrade: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow-gpu==1.13.0rc1) (2.8.0)\n","Requirement already satisfied, skipping upgrade: werkzeug>=0.11.10 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.13.0,>=1.12.0->tensorflow-gpu==1.13.0rc1) (0.15.4)\n","Requirement already satisfied, skipping upgrade: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.13.0,>=1.12.0->tensorflow-gpu==1.13.0rc1) (3.1.1)\n","\u001b[31mERROR: tensorflow 1.14.0 has requirement tensorboard<1.15.0,>=1.14.0, but you'll have tensorboard 1.12.2 which is incompatible.\u001b[0m\n","\u001b[31mERROR: tensorflow 1.14.0 has requirement tensorflow-estimator<1.15.0rc0,>=1.14.0rc0, but you'll have tensorflow-estimator 1.13.0 which is incompatible.\u001b[0m\n","Installing collected packages: mock, tensorflow-estimator, tensorboard, tensorflow-gpu\n","  Found existing installation: tensorflow-estimator 1.14.0\n","    Uninstalling tensorflow-estimator-1.14.0:\n","      Successfully uninstalled tensorflow-estimator-1.14.0\n","  Found existing installation: tensorboard 1.14.0\n","    Uninstalling tensorboard-1.14.0:\n","      Successfully uninstalled tensorboard-1.14.0\n","Successfully installed mock-3.0.5 tensorboard-1.12.2 tensorflow-estimator-1.13.0 tensorflow-gpu-1.13.0rc1\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"ssFmgQCeu-Qm","colab_type":"code","outputId":"1ea61554-8da9-4648-a59d-dc22974e7623","executionInfo":{"status":"ok","timestamp":1562364015094,"user_tz":-540,"elapsed":83760,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["import tensorflow as tf\n","\n","tf.__version__"],"execution_count":2,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'1.13.0-rc1'"]},"metadata":{"tags":[]},"execution_count":2}]},{"cell_type":"code","metadata":{"id":"ckPpw7Hjg7Kd","colab_type":"code","colab":{}},"source":["import os\n","from datetime import datetime\n","import tensorflow as tf\n","import numpy as np\n","import json\n","from sklearn.model_selection import train_test_split"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"bsn1XjI0hDsr","colab_type":"code","outputId":"4b773c85-9450-4d4c-ee20-692a2fdcd592","executionInfo":{"status":"ok","timestamp":1562364038425,"user_tz":-540,"elapsed":107074,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":54}},"source":["from google.colab import auth\n","auth.authenticate_user()\n","\n","from google.colab import drive\n","drive.mount('/content/gdrive')"],"execution_count":4,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"VONKijgCtzXM","colab_type":"code","colab":{}},"source":["DATA_IN_PATH = './data_in/'\n","DATA_OUT_PATH = './data_out/'"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"M2WFPIawtz8x","colab_type":"code","outputId":"5c5d52af-d5b8-4ba5-9079-42fc661f8ac1","executionInfo":{"status":"ok","timestamp":1562364040579,"user_tz":-540,"elapsed":109215,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["!rm -rf data_in;rm -rf data_out;ls;"],"execution_count":6,"outputs":[{"output_type":"stream","text":["adc.json  gdrive  sample_data\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"fqk8DRqVtE7-","colab_type":"code","colab":{}},"source":["if not os.path.exists(DATA_IN_PATH):\n","    os.makedirs(DATA_IN_PATH)\n","    \n","if not os.path.exists(DATA_OUT_PATH):\n","    os.makedirs(DATA_OUT_PATH)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"vehpB8bF61-u","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":591},"outputId":"f5803516-694d-463c-de11-83e919a3f79c","executionInfo":{"status":"ok","timestamp":1562364045815,"user_tz":-540,"elapsed":114436,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}}},"source":["!wget -P '/content/gdrive/My Drive/Colab Notebooks/Data/' https://raw.githubusercontent.com/changwookjun/learningspoons/master/Data/nsmc_train_input.npy;\n","!wget -P '/content/gdrive/My Drive/Colab Notebooks/Data/' https://raw.githubusercontent.com/changwookjun/learningspoons/master/Data/nsmc_train_label.npy;\n","!wget -P '/content/gdrive/My Drive/Colab Notebooks/Data/' https://raw.githubusercontent.com/changwookjun/learningspoons/master/Data/data_configs.json;"],"execution_count":8,"outputs":[{"output_type":"stream","text":["--2019-07-05 22:00:41--  https://raw.githubusercontent.com/changwookjun/learningspoons/master/Data/nsmc_train_input.npy\n","Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n","Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 4800128 (4.6M) [application/octet-stream]\n","Saving to: ‘/content/gdrive/My Drive/Colab Notebooks/Data/nsmc_train_input.npy.3’\n","\n","\rnsmc_train_input.np   0%[                    ]       0  --.-KB/s               \rnsmc_train_input.np 100%[===================>]   4.58M  --.-KB/s    in 0.06s   \n","\n","2019-07-05 22:00:41 (72.5 MB/s) - ‘/content/gdrive/My Drive/Colab Notebooks/Data/nsmc_train_input.npy.3’ saved [4800128/4800128]\n","\n","--2019-07-05 22:00:42--  https://raw.githubusercontent.com/changwookjun/learningspoons/master/Data/nsmc_train_label.npy\n","Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n","Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 1200128 (1.1M) [application/octet-stream]\n","Saving to: ‘/content/gdrive/My Drive/Colab Notebooks/Data/nsmc_train_label.npy.3’\n","\n","nsmc_train_label.np 100%[===================>]   1.14M  --.-KB/s    in 0.04s   \n","\n","2019-07-05 22:00:43 (25.8 MB/s) - ‘/content/gdrive/My Drive/Colab Notebooks/Data/nsmc_train_label.npy.3’ saved [1200128/1200128]\n","\n","--2019-07-05 22:00:44--  https://raw.githubusercontent.com/changwookjun/learningspoons/master/Data/data_configs.json\n","Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n","Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 830764 (811K) [text/plain]\n","Saving to: ‘/content/gdrive/My Drive/Colab Notebooks/Data/data_configs.json.3’\n","\n","data_configs.json.3 100%[===================>] 811.29K  --.-KB/s    in 0.04s   \n","\n","2019-07-05 22:00:44 (17.9 MB/s) - ‘/content/gdrive/My Drive/Colab Notebooks/Data/data_configs.json.3’ saved [830764/830764]\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"GQbbrJwisaPP","colab_type":"code","colab":{}},"source":["!cp '/content/gdrive/My Drive/Colab Notebooks/Data/nsmc_train_input.npy' ./data_in/nsmc_train_input.npy;\n","!cp '/content/gdrive/My Drive/Colab Notebooks/Data/nsmc_train_label.npy' ./data_in/nsmc_train_label.npy;\n","!cp '/content/gdrive/My Drive/Colab Notebooks/Data/data_configs.json' ./data_in/data_configs.json;"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"2_niNzPVs6pB","colab_type":"code","outputId":"6a3ad7cd-41f4-450c-c727-1bc7cddc16ad","executionInfo":{"status":"ok","timestamp":1562364053167,"user_tz":-540,"elapsed":121776,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":118}},"source":["!ls -al ./data_in/"],"execution_count":10,"outputs":[{"output_type":"stream","text":["total 6684\n","drwxr-xr-x 2 root root    4096 Jul  5 22:00 .\n","drwxr-xr-x 1 root root    4096 Jul  5 22:00 ..\n","-rw-r--r-- 1 root root  830764 Jul  5 22:00 data_configs.json\n","-rw-r--r-- 1 root root 4800128 Jul  5 22:00 nsmc_train_input.npy\n","-rw-r--r-- 1 root root 1200128 Jul  5 22:00 nsmc_train_label.npy\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"0xHXgGwHhDvc","colab_type":"code","colab":{}},"source":["DATA_IN_PATH = './data_in/'\n","DATA_OUT_PATH = './data_out/'\n","INPUT_TRAIN_DATA = 'nsmc_train_input.npy'\n","LABEL_TRAIN_DATA = 'nsmc_train_label.npy'\n","DATA_CONFIGS = 'data_configs.json'\n","\n","input_data = np.load(open(DATA_IN_PATH + INPUT_TRAIN_DATA, 'rb'))\n","label_data = np.load(open(DATA_IN_PATH + LABEL_TRAIN_DATA, 'rb'))\n","prepro_configs = json.load(open(DATA_IN_PATH + DATA_CONFIGS, 'r'))"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"agYxGd_whDx7","colab_type":"code","colab":{}},"source":["TEST_SPLIT = 0.1\n","RNG_SEED = 13371447\n","VOCAB_SIZE = prepro_configs['vocab_size']\n","EMB_SIZE = 128\n","BATCH_SIZE = 64 \n","NUM_EPOCHS = 1\n","HIDDEN_SIZE = 128\n","MAX_SEQUENCE_LENGTH = 8\n","SCALE_OUT = 2\n","ATTENITON_SIZE = 128 * SCALE_OUT\n","ATTENITON_HOP_SIZE = ATTENITON_SIZE \n","\n","input_train, input_eval, label_train, label_eval = train_test_split(input_data, label_data, test_size=TEST_SPLIT, random_state=RNG_SEED)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"M-TwQ29ehD0e","colab_type":"code","colab":{}},"source":["def mapping_fn(X, Y):\n","    input, label = {'x': X}, Y\n","    return input, label\n","\n","def train_input_fn():\n","    dataset = tf.data.Dataset.from_tensor_slices((input_train, label_train))\n","    dataset = dataset.shuffle(buffer_size=len(input_train))\n","    dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)\n","    dataset = dataset.map(mapping_fn)\n","    dataset = dataset.repeat(count=NUM_EPOCHS)\n","    iterator = dataset.make_one_shot_iterator()\n","    \n","    return iterator.get_next()\n","\n","def eval_input_fn():\n","    dataset = tf.data.Dataset.from_tensor_slices((input_eval, label_eval))\n","    dataset = dataset.shuffle(buffer_size=len(input_eval))\n","    dataset = dataset.batch(BATCH_SIZE, drop_remainder=True) \n","    dataset = dataset.map(mapping_fn)\n","    iterator = dataset.make_one_shot_iterator()\n","    \n","    return iterator.get_next()"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"sAw6fWnHhD2_","colab_type":"code","colab":{}},"source":["def model_fn(features, labels, mode, params):\n","    TRAIN = mode == tf.estimator.ModeKeys.TRAIN\n","    EVAL = mode == tf.estimator.ModeKeys.EVAL\n","    PREDICT = mode == tf.estimator.ModeKeys.PREDICT\n","\n","    initializer = tf.contrib.layers.xavier_initializer()\n","    \n","    \n","    embedding_layer = tf.keras.layers.Embedding(\n","                            VOCAB_SIZE,\n","                            EMB_SIZE)(features['x'])\n","    \n","    dropout_emb = tf.keras.layers.Dropout(rate = 0.5)(embedding_layer) # ? , 8, 128\n","\n","    ### Model start\n","    lstm_cell_fw = tf.nn.rnn_cell.BasicLSTMCell(HIDDEN_SIZE) \n","    \n","    lstm_cell_bw = tf.nn.rnn_cell.BasicLSTMCell(HIDDEN_SIZE) \n","    \n","    ((fw_outputs, bw_outputs), states) = tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_cell_fw, # ?, 8, 128\n","                                                                             cell_bw=lstm_cell_bw, # ?, 8, 128\n","                                                                             inputs=dropout_emb,\n","                                                                             dtype=tf.float32)\n","    \n","    hidden = tf.concat([fw_outputs, bw_outputs], axis=2) #?, 8, 256\n","    \n","    a_w1 = tf.get_variable(\"a_w1\", shape=[BATCH_SIZE, HIDDEN_SIZE * 2, ATTENITON_HOP_SIZE], initializer=initializer) # ?,  256, 256\n","    \n","    result1 = tf.nn.tanh(tf.matmul(hidden, a_w1)) # ? 8, 256\n","    \n","    a_w2 = tf.get_variable(\"a_w2\", shape=[BATCH_SIZE, ATTENITON_HOP_SIZE, ATTENITON_SIZE], initializer=initializer) # ?, 256, 256\n","    \n","    result2 = tf.matmul(result1, a_w2) # ? 8, 256\n","    \n","    result2_reshape = tf.transpose(result2, [0, 2, 1]) # ?, 256, 8\n","      \n","    score = tf.nn.softmax(result2_reshape) # ?, 256, 8\n","    \n","    score = tf.transpose(score, [0, 2, 1]) # ?, 8, 256\n","    \n","    last_metrix = hidden * score # ?, 8, 256\n","    \n","    last_metrix = tf.reshape(last_metrix, [-1, MAX_SEQUENCE_LENGTH * ATTENITON_HOP_SIZE]) # ?, 2048\n","\n","    logits = tf.keras.layers.Dense(units=1)(last_metrix) # ?, 1\n","    \n","    if labels is not None:\n","        labels_ = tf.reshape(labels, [-1, 1]) # ?, 1\n","    \n","    if TRAIN:\n","        global_step = tf.train.get_global_step()\n","        loss = tf.losses.sigmoid_cross_entropy(labels_, logits)\n","        train_op = tf.train.AdamOptimizer(0.001).minimize(loss, global_step)\n","\n","        return tf.estimator.EstimatorSpec(mode=mode, train_op=train_op, loss = loss)\n","    \n","    elif EVAL:\n","        loss = tf.losses.sigmoid_cross_entropy(labels_, logits)\n","        pred = tf.nn.sigmoid(logits)\n","        accuracy = tf.metrics.accuracy(labels, tf.round(pred))\n","        return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops={'acc': accuracy})\n","        \n","    elif PREDICT:\n","        return tf.estimator.EstimatorSpec(\n","            mode=mode,\n","            predictions={\n","                'prob': tf.nn.sigmoid(logits),\n","            }\n","        )"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"5yg6tPXNhD5X","colab_type":"code","outputId":"6dd038f3-bfec-4fef-f1f1-81afa10953e5","executionInfo":{"status":"ok","timestamp":1562364055482,"user_tz":-540,"elapsed":124082,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":134}},"source":["!rm -rf data_out;ls;ls -al ./data_in"],"execution_count":15,"outputs":[{"output_type":"stream","text":["adc.json  data_in  gdrive  sample_data\n","total 6684\n","drwxr-xr-x 2 root root    4096 Jul  5 22:00 .\n","drwxr-xr-x 1 root root    4096 Jul  5 22:00 ..\n","-rw-r--r-- 1 root root  830764 Jul  5 22:00 data_configs.json\n","-rw-r--r-- 1 root root 4800128 Jul  5 22:00 nsmc_train_input.npy\n","-rw-r--r-- 1 root root 1200128 Jul  5 22:00 nsmc_train_label.npy\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"pi5-EBdxhD7m","colab_type":"code","outputId":"482b8d1d-19c5-4061-9a3b-f4d975b14c76","executionInfo":{"status":"ok","timestamp":1562364055483,"user_tz":-540,"elapsed":124079,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":188}},"source":["est = tf.estimator.Estimator(model_fn, model_dir=\"data_out/checkpoint/self_attention\")"],"execution_count":16,"outputs":[{"output_type":"stream","text":["INFO:tensorflow:Using default config.\n","INFO:tensorflow:Using config: {'_model_dir': 'data_out/checkpoint/self_attention', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true\n","graph_options {\n","  rewrite_options {\n","    meta_optimizer_iterations: ONE\n","  }\n","}\n",", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7faf83cb1c50>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n","WARNING:tensorflow:Estimator's model_fn (<function model_fn at 0x7faf8c161a60>) includes params argument, but params are not passed to Estimator.\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"k4r4-M4ChD94","colab_type":"code","outputId":"d6b4c6d5-c829-4e20-9bf4-4bf0ee138e26","executionInfo":{"status":"ok","timestamp":1562364408835,"user_tz":-540,"elapsed":477426,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":1000}},"source":["est.train(train_input_fn)"],"execution_count":17,"outputs":[{"output_type":"stream","text":["WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Colocations handled automatically by placer.\n","WARNING:tensorflow:From <ipython-input-13-af712a77ea51>:11: DatasetV1.make_one_shot_iterator (from tensorflow.python.data.ops.dataset_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use `for ... in dataset:` to iterate over a dataset. If using `tf.estimator`, return the `Dataset` object directly from your input function. As a last resort, you can use `tf.compat.v1.data.make_one_shot_iterator(dataset)`.\n","INFO:tensorflow:Calling model_fn.\n","\n","WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n","For more information, please see:\n","  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n","  * https://github.com/tensorflow/addons\n","If you depend on functionality not listed there, please file an issue.\n","\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n","WARNING:tensorflow:From <ipython-input-14-9b5eb4f61e4f>:16: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n","WARNING:tensorflow:From <ipython-input-14-9b5eb4f61e4f>:23: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:443: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/losses/losses_impl.py:209: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.cast instead.\n","INFO:tensorflow:Done calling model_fn.\n","INFO:tensorflow:Create CheckpointSaverHook.\n","INFO:tensorflow:Graph was finalized.\n","INFO:tensorflow:Running local_init_op.\n","INFO:tensorflow:Done running local_init_op.\n","INFO:tensorflow:Saving checkpoints for 0 into data_out/checkpoint/self_attention/model.ckpt.\n","INFO:tensorflow:loss = 0.693162, step = 1\n","INFO:tensorflow:global_step/sec: 5.94496\n","INFO:tensorflow:loss = 0.5473889, step = 101 (16.824 sec)\n","INFO:tensorflow:global_step/sec: 6.10735\n","INFO:tensorflow:loss = 0.3550779, step = 201 (16.373 sec)\n","INFO:tensorflow:global_step/sec: 6.15734\n","INFO:tensorflow:loss = 0.47260237, step = 301 (16.241 sec)\n","INFO:tensorflow:global_step/sec: 6.17636\n","INFO:tensorflow:loss = 0.36666715, step = 401 (16.191 sec)\n","INFO:tensorflow:global_step/sec: 6.1632\n","INFO:tensorflow:loss = 0.40059298, step = 501 (16.225 sec)\n","INFO:tensorflow:global_step/sec: 6.14872\n","INFO:tensorflow:loss = 0.4394063, step = 601 (16.266 sec)\n","INFO:tensorflow:global_step/sec: 6.07337\n","INFO:tensorflow:loss = 0.38419595, step = 701 (16.467 sec)\n","INFO:tensorflow:global_step/sec: 6.10785\n","INFO:tensorflow:loss = 0.4192267, step = 801 (16.367 sec)\n","INFO:tensorflow:global_step/sec: 6.12014\n","INFO:tensorflow:loss = 0.47303677, step = 901 (16.343 sec)\n","INFO:tensorflow:global_step/sec: 6.11644\n","INFO:tensorflow:loss = 0.4283538, step = 1001 (16.347 sec)\n","INFO:tensorflow:global_step/sec: 6.11131\n","INFO:tensorflow:loss = 0.43828177, step = 1101 (16.363 sec)\n","INFO:tensorflow:global_step/sec: 6.17608\n","INFO:tensorflow:loss = 0.48144674, step = 1201 (16.193 sec)\n","INFO:tensorflow:global_step/sec: 6.14535\n","INFO:tensorflow:loss = 0.6216367, step = 1301 (16.271 sec)\n","INFO:tensorflow:global_step/sec: 6.15509\n","INFO:tensorflow:loss = 0.3459075, step = 1401 (16.247 sec)\n","INFO:tensorflow:global_step/sec: 6.17879\n","INFO:tensorflow:loss = 0.37386233, step = 1501 (16.184 sec)\n","INFO:tensorflow:global_step/sec: 6.15462\n","INFO:tensorflow:loss = 0.4364525, step = 1601 (16.248 sec)\n","INFO:tensorflow:global_step/sec: 6.15171\n","INFO:tensorflow:loss = 0.32824683, step = 1701 (16.256 sec)\n","INFO:tensorflow:global_step/sec: 6.13693\n","INFO:tensorflow:loss = 0.33905262, step = 1801 (16.294 sec)\n","INFO:tensorflow:global_step/sec: 6.12627\n","INFO:tensorflow:loss = 0.3423342, step = 1901 (16.323 sec)\n","INFO:tensorflow:global_step/sec: 6.0994\n","INFO:tensorflow:loss = 0.44363248, step = 2001 (16.397 sec)\n","INFO:tensorflow:global_step/sec: 6.12736\n","INFO:tensorflow:loss = 0.32284153, step = 2101 (16.319 sec)\n","INFO:tensorflow:Saving checkpoints for 2109 into data_out/checkpoint/self_attention/model.ckpt.\n","INFO:tensorflow:Loss for final step: 0.3905813.\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["<tensorflow_estimator.python.estimator.estimator.Estimator at 0x7faf83cb1748>"]},"metadata":{"tags":[]},"execution_count":17}]},{"cell_type":"code","metadata":{"id":"fI-6SSWQhEAO","colab_type":"code","outputId":"817c2496-611d-4099-8b89-00ad59c31703","executionInfo":{"status":"ok","timestamp":1562364415797,"user_tz":-540,"elapsed":484382,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":255}},"source":["valid = est.evaluate(eval_input_fn)"],"execution_count":18,"outputs":[{"output_type":"stream","text":["INFO:tensorflow:Calling model_fn.\n","INFO:tensorflow:Done calling model_fn.\n","INFO:tensorflow:Starting evaluation at 2019-07-05T22:06:49Z\n","INFO:tensorflow:Graph was finalized.\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use standard file APIs to check for files with this prefix.\n","INFO:tensorflow:Restoring parameters from data_out/checkpoint/self_attention/model.ckpt-2109\n","INFO:tensorflow:Running local_init_op.\n","INFO:tensorflow:Done running local_init_op.\n","INFO:tensorflow:Finished evaluation at 2019-07-05-22:06:55\n","INFO:tensorflow:Saving dict for global step 2109: acc = 0.82558763, global_step = 2109, loss = 0.3829688\n","INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2109: data_out/checkpoint/self_attention/model.ckpt-2109\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"9MBaqr_WhEMc","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]}]}