Day6_2_Attention_visualization.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Day6_2_Attention_visualization.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","metadata":{"id":"JFFAEboEwS69","colab_type":"text"},"source":["# Attention Visualization"]},{"cell_type":"code","metadata":{"id":"UnlKkYZgjQHv","colab_type":"code","outputId":"99d22f82-8cfe-4656-bb2d-c0ed62c3961f","executionInfo":{"status":"ok","timestamp":1562364564039,"user_tz":-540,"elapsed":4770,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":356}},"source":["!pip install --upgrade tensorflow-gpu==1.13.0rc1"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Requirement already up-to-date: tensorflow-gpu==1.13.0rc1 in /usr/local/lib/python3.6/dist-packages (1.13.0rc1)\n","Requirement already satisfied, skipping upgrade: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (1.0.8)\n","Requirement already satisfied, skipping upgrade: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (1.1.0)\n","Requirement already satisfied, skipping upgrade: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (1.12.0)\n","Requirement already satisfied, skipping upgrade: tensorflow-estimator<1.14.0rc0,>=1.13.0rc0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (1.13.0)\n","Requirement already satisfied, skipping upgrade: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (1.16.4)\n","Requirement already satisfied, skipping upgrade: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (1.15.0)\n","Requirement already satisfied, skipping upgrade: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (1.1.0)\n","Requirement already satisfied, skipping upgrade: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (0.8.0)\n","Requirement already satisfied, skipping upgrade: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (0.33.4)\n","Requirement already satisfied, skipping upgrade: absl-py>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (0.7.1)\n","Requirement already satisfied, skipping upgrade: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (0.2.2)\n","Requirement already satisfied, skipping upgrade: tensorboard<1.13.0,>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (1.12.2)\n","Requirement already satisfied, skipping upgrade: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.13.0rc1) (3.7.1)\n","Requirement already satisfied, skipping upgrade: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow-gpu==1.13.0rc1) (2.8.0)\n","Requirement already satisfied, skipping upgrade: mock>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-estimator<1.14.0rc0,>=1.13.0rc0->tensorflow-gpu==1.13.0rc1) (3.0.5)\n","Requirement already satisfied, skipping upgrade: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.13.0,>=1.12.0->tensorflow-gpu==1.13.0rc1) (3.1.1)\n","Requirement already satisfied, skipping upgrade: werkzeug>=0.11.10 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.13.0,>=1.12.0->tensorflow-gpu==1.13.0rc1) (0.15.4)\n","Requirement already satisfied, skipping upgrade: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow-gpu==1.13.0rc1) (41.0.1)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"pK0Ntj1JjSrb","colab_type":"code","outputId":"8bbb25e8-7f3c-4c81-87db-e03f9a7756fd","executionInfo":{"status":"ok","timestamp":1562364565545,"user_tz":-540,"elapsed":6261,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["import tensorflow as tf\n","\n","tf.__version__"],"execution_count":2,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'1.13.0-rc1'"]},"metadata":{"tags":[]},"execution_count":2}]},{"cell_type":"code","metadata":{"id":"rbpvGdtnxdkb","colab_type":"code","outputId":"8309e6dd-d937-4eed-de7a-aa87ec8d295c","executionInfo":{"status":"ok","timestamp":1562364568763,"user_tz":-540,"elapsed":9469,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":54}},"source":["from google.colab import auth\n","auth.authenticate_user()\n","\n","from google.colab import drive\n","drive.mount('/content/gdrive')"],"execution_count":3,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"cRdV6_8BxwG-","colab_type":"code","outputId":"2c7dfd05-0a1e-4ea9-8a07-d1a9c6a3aa15","executionInfo":{"status":"ok","timestamp":1562364570836,"user_tz":-540,"elapsed":11533,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":168}},"source":["!ls -al"],"execution_count":4,"outputs":[{"output_type":"stream","text":["total 40\n","drwxr-xr-x 1 root root 4096 Jul  5 20:53 .\n","drwxr-xr-x 1 root root 4096 Jul  5 20:51 ..\n","-rw-r--r-- 1 root root 2529 Jul  5 20:53 adc.json\n","drwxr-xr-x 1 root root 4096 Jul  5 20:53 .config\n","drwxr-xr-x 2 root root 4096 Jul  5 20:53 data_in\n","drwxr-xr-x 3 root root 4096 Jul  5 20:55 data_out\n","drwx------ 3 root root 4096 Jul  5 20:53 gdrive\n","drwxr-xr-x 1 root root 4096 Jun 18 16:14 sample_data\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"NNmzxxEEydpr","colab_type":"code","colab":{}},"source":["DATA_IN_PATH = './data_in/'\n","DATA_OUT_PATH = './data_out/'"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"JH9MtriSyeY6","colab_type":"code","outputId":"4364ec9b-4d05-42b8-9b01-3c9d484269da","executionInfo":{"status":"ok","timestamp":1562364573181,"user_tz":-540,"elapsed":13862,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["!rm -rf data_in;rm -rf data_out;ls;"],"execution_count":6,"outputs":[{"output_type":"stream","text":["adc.json  gdrive  sample_data\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"7L7k7kdRylj2","colab_type":"code","colab":{}},"source":["import os\n","\n","if not os.path.exists(DATA_IN_PATH):\n","    os.makedirs(DATA_IN_PATH)\n","    \n","if not os.path.exists(DATA_OUT_PATH):\n","    os.makedirs(DATA_OUT_PATH)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"qhufNIP8GZBT","colab_type":"code","outputId":"43c7a5b2-cf72-4afe-b054-f33ce9a032c1","executionInfo":{"status":"ok","timestamp":1562364574921,"user_tz":-540,"elapsed":15588,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":134}},"source":["!ls -al '/content/gdrive/My Drive/Colab Notebooks/Data/'"],"execution_count":8,"outputs":[{"output_type":"stream","text":["total 9495\n","-rw------- 1 root root  889842 Jun 29 03:10 ChatBotData.csv\n","-rw------- 1 root root  830764 Jul  3 06:37 data_configs.json\n","-rw------- 1 root root 1600128 Jul  3 06:37 nsmc_test_input.npy\n","-rw------- 1 root root  400128 Jul  3 06:37 nsmc_test_label.npy\n","-rw------- 1 root root 4800128 Jul  3 06:37 nsmc_train_input.npy\n","-rw------- 1 root root 1200128 Jul  3 06:37 nsmc_train_label.npy\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"RspuTlNoGeBs","colab_type":"code","outputId":"24d18685-322b-4e3b-c249-b203a518bdc9","executionInfo":{"status":"ok","timestamp":1562364577561,"user_tz":-540,"elapsed":18221,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":84}},"source":["!ls -al '/content/gdrive/My Drive/Colab Notebooks/Seq2seq/'"],"execution_count":9,"outputs":[{"output_type":"stream","text":["total 20\n","-rw------- 1 root root 10819 Jun 29 03:42 data.py\n","-rw------- 1 root root  4657 Jun 29 03:42 model.py\n","drwx------ 2 root root  4096 Jul  5 20:54 __pycache__\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"-P_EGDdBGuQ5","colab_type":"code","outputId":"31169f8f-d701-4df4-e827-c984da2d61f3","executionInfo":{"status":"ok","timestamp":1562364580836,"user_tz":-540,"elapsed":21489,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":134}},"source":["!ls -al '/content/gdrive/My Drive/Colab Notebooks/Data/'"],"execution_count":10,"outputs":[{"output_type":"stream","text":["total 9495\n","-rw------- 1 root root  889842 Jun 29 03:10 ChatBotData.csv\n","-rw------- 1 root root  830764 Jul  3 06:37 data_configs.json\n","-rw------- 1 root root 1600128 Jul  3 06:37 nsmc_test_input.npy\n","-rw------- 1 root root  400128 Jul  3 06:37 nsmc_test_label.npy\n","-rw------- 1 root root 4800128 Jul  3 06:37 nsmc_train_input.npy\n","-rw------- 1 root root 1200128 Jul  3 06:37 nsmc_train_label.npy\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"hc-fN_ViLmXG","colab_type":"code","outputId":"360fe033-2970-4de7-f010-1ce1da61759c","executionInfo":{"status":"ok","timestamp":1562364583332,"user_tz":-540,"elapsed":23979,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":202}},"source":["!wget -P '/content/gdrive/My Drive/Colab Notebooks/Seq2seq/' https://raw.githubusercontent.com/changwookjun/learningspoons/master/Seq2seq/model_att.py"],"execution_count":11,"outputs":[{"output_type":"stream","text":["--2019-07-05 22:09:41--  https://raw.githubusercontent.com/changwookjun/learningspoons/master/Seq2seq/model_att.py\n","Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n","Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 6820 (6.7K) [text/plain]\n","Saving to: ‘/content/gdrive/My Drive/Colab Notebooks/Seq2seq/model_att.py’\n","\n","\rmodel_att.py          0%[                    ]       0  --.-KB/s               \rmodel_att.py        100%[===================>]   6.66K  --.-KB/s    in 0.001s  \n","\n","2019-07-05 22:09:42 (5.58 MB/s) - ‘/content/gdrive/My Drive/Colab Notebooks/Seq2seq/model_att.py’ saved [6820/6820]\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"VOznj_D2x-Kq","colab_type":"code","outputId":"68026c76-c990-4dfd-f2b2-a40650a3d866","executionInfo":{"status":"ok","timestamp":1562364585592,"user_tz":-540,"elapsed":26233,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":202}},"source":["!wget -P '/content/gdrive/My Drive/Colab Notebooks/Seq2seq/' https://raw.githubusercontent.com/changwookjun/learningspoons/master/Seq2seq/configs.py"],"execution_count":12,"outputs":[{"output_type":"stream","text":["--2019-07-05 22:09:44--  https://raw.githubusercontent.com/changwookjun/learningspoons/master/Seq2seq/configs.py\n","Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n","Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 1450 (1.4K) [text/plain]\n","Saving to: ‘/content/gdrive/My Drive/Colab Notebooks/Seq2seq/configs.py’\n","\n","\rconfigs.py            0%[                    ]       0  --.-KB/s               \rconfigs.py          100%[===================>]   1.42K  --.-KB/s    in 0s      \n","\n","2019-07-05 22:09:44 (98.8 MB/s) - ‘/content/gdrive/My Drive/Colab Notebooks/Seq2seq/configs.py’ saved [1450/1450]\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"JlkHqQABGyGj","colab_type":"code","outputId":"d1838083-b4f7-4751-a1f6-f1ee0a16ed60","executionInfo":{"status":"ok","timestamp":1562364588225,"user_tz":-540,"elapsed":28861,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":118}},"source":["!ls -al '/content/gdrive/My Drive/Colab Notebooks/Seq2seq/'"],"execution_count":13,"outputs":[{"output_type":"stream","text":["total 29\n","-rw------- 1 root root  1450 Jul  5 22:09 configs.py\n","-rw------- 1 root root 10819 Jun 29 03:42 data.py\n","-rw------- 1 root root  6820 Jul  5 22:09 model_att.py\n","-rw------- 1 root root  4657 Jun 29 03:42 model.py\n","drwx------ 2 root root  4096 Jul  5 20:54 __pycache__\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"iHOBh68Ny7zA","colab_type":"code","outputId":"d5cd2f0b-d993-4c6c-f83d-3aaf53129162","executionInfo":{"status":"ok","timestamp":1562364590326,"user_tz":-540,"elapsed":30954,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":67}},"source":["!ls -al ./data_in/"],"execution_count":14,"outputs":[{"output_type":"stream","text":["total 12\n","drwxr-xr-x 2 root root 4096 Jul  5 22:09 .\n","drwxr-xr-x 1 root root 4096 Jul  5 22:09 ..\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"51SnMwcbzQyj","colab_type":"code","outputId":"a7ffae3e-9c2b-4a9e-df88-84313c968e0c","executionInfo":{"status":"ok","timestamp":1562364592021,"user_tz":-540,"elapsed":32643,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":84}},"source":["!cp '/content/gdrive/My Drive/Colab Notebooks/Data/ChatBotData.csv' ./data_in/ChatBotData.csv;ls -al ./data_in/"],"execution_count":15,"outputs":[{"output_type":"stream","text":["total 884\n","drwxr-xr-x 2 root root   4096 Jul  5 22:09 .\n","drwxr-xr-x 1 root root   4096 Jul  5 22:09 ..\n","-rw------- 1 root root 889842 Jul  5 22:09 ChatBotData.csv\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"INhIGHZbzh7L","colab_type":"code","outputId":"b560e6b1-e1d9-48e6-e3b9-1044cf786438","executionInfo":{"status":"ok","timestamp":1562364593814,"user_tz":-540,"elapsed":34427,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":84}},"source":["!ls -al ./data_in/"],"execution_count":16,"outputs":[{"output_type":"stream","text":["total 884\n","drwxr-xr-x 2 root root   4096 Jul  5 22:09 .\n","drwxr-xr-x 1 root root   4096 Jul  5 22:09 ..\n","-rw------- 1 root root 889842 Jul  5 22:09 ChatBotData.csv\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"t2BH-cTXzA1u","colab_type":"code","outputId":"ffcfe869-cf0c-430f-f06b-b1d46fd949dd","executionInfo":{"status":"ok","timestamp":1562364593815,"user_tz":-540,"elapsed":34422,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":195}},"source":["import pandas as pd\n","\n","train_data = pd.read_csv(DATA_IN_PATH + 'ChatBotData.csv')\n","train_data.head()"],"execution_count":17,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Q</th>\n","      <th>A</th>\n","      <th>label</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>12시 땡!</td>\n","      <td>하루가 또 가네요.</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1지망 학교 떨어졌어</td>\n","      <td>위로해 드립니다.</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>3박4일 놀러가고 싶다</td>\n","      <td>여행은 언제나 좋죠.</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>3박4일 정도 놀러가고 싶다</td>\n","      <td>여행은 언제나 좋죠.</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>PPL 심하네</td>\n","      <td>눈살이 찌푸려지죠.</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["                 Q            A  label\n","0           12시 땡!   하루가 또 가네요.      0\n","1      1지망 학교 떨어졌어    위로해 드립니다.      0\n","2     3박4일 놀러가고 싶다  여행은 언제나 좋죠.      0\n","3  3박4일 정도 놀러가고 싶다  여행은 언제나 좋죠.      0\n","4          PPL 심하네   눈살이 찌푸려지죠.      0"]},"metadata":{"tags":[]},"execution_count":17}]},{"cell_type":"markdown","metadata":{"id":"yRWS8DR2e971","colab_type":"text"},"source":["### Install Pakage"]},{"cell_type":"code","metadata":{"id":"hi8mqGZe0_Uo","colab_type":"code","outputId":"b0345d50-1994-4856-fa5c-ad0c41aba9f5","executionInfo":{"status":"ok","timestamp":1562364597457,"user_tz":-540,"elapsed":38058,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":50}},"source":["!pip install konlpy"],"execution_count":18,"outputs":[{"output_type":"stream","text":["Requirement already satisfied: konlpy in /usr/local/lib/python3.6/dist-packages (0.5.1)\n","Requirement already satisfied: JPype1>=0.5.7 in /usr/local/lib/python3.6/dist-packages (from konlpy) (0.7.0)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"-p77IRnd8t42","colab_type":"code","outputId":"aafcacdf-1c19-488b-970c-5e7db0b2891f","executionInfo":{"status":"ok","timestamp":1562364600926,"user_tz":-540,"elapsed":41521,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["!pip install tqdm"],"execution_count":19,"outputs":[{"output_type":"stream","text":["Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (4.28.1)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"4Cv6Tp4rfB32","colab_type":"text"},"source":["### Import Package"]},{"cell_type":"code","metadata":{"id":"zkzSp9-a-YM1","colab_type":"code","colab":{}},"source":["import tensorflow as tf\n","import numpy as np\n","from tqdm import tqdm\n","from konlpy.tag import Okt\n","import re\n","import os\n","import sys\n","\n","import matplotlib\n","import matplotlib.pyplot as plt\n","import matplotlib.font_manager as fm"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"yyIrIZelfFIq","colab_type":"text"},"source":["### Mount Gdrive Source"]},{"cell_type":"code","metadata":{"id":"fCYXo2R9Jj2Y","colab_type":"code","colab":{}},"source":["sys.path.insert(0, '/content/gdrive/My Drive/Colab Notebooks/Seq2seq/')\n","\n","import data\n","import configs as conf\n","import model_att as ml"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"-HVSPtcFwWI5","colab_type":"code","outputId":"895e970a-2ec0-4545-edd1-352e3f796bb3","executionInfo":{"status":"ok","timestamp":1562364630953,"user_tz":-540,"elapsed":71531,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":218}},"source":["char2idx,  idx2char, vocabulary_length = data.load_vocabulary()"],"execution_count":22,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/jpype/_core.py:210: UserWarning: \n","-------------------------------------------------------------------------------\n","Deprecated: convertStrings was not specified when starting the JVM. The default\n","behavior in JPype will be False starting in JPype 0.8. The recommended setting\n","for new code is convertStrings=False.  The legacy value of True was assumed for\n","this session. If you are a user of an application that reported this warning,\n","please file a ticket with the developer.\n","-------------------------------------------------------------------------------\n","\n","  \"\"\")\n","100%|██████████| 11823/11823 [00:17<00:00, 657.76it/s]\n","100%|██████████| 11823/11823 [00:11<00:00, 1072.30it/s]\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"vIpAKWD12vnI","colab_type":"code","outputId":"80dfb0b8-2356-42c1-d6ac-8b1acd74ed6f","executionInfo":{"status":"ok","timestamp":1562364630954,"user_tz":-540,"elapsed":71526,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["print(vocabulary_length)"],"execution_count":23,"outputs":[{"output_type":"stream","text":["12657\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"l2oK2uyBKPPj","colab_type":"code","colab":{}},"source":["train_input, train_label, eval_input, eval_label = data.load_data()"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Ry-UEQOiCmXn","colab_type":"code","outputId":"8de56d0e-d76d-4f40-9c43-a4a2e5f16886","executionInfo":{"status":"ok","timestamp":1562364651819,"user_tz":-540,"elapsed":92380,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":67}},"source":["train_input_enc, train_input_enc_length = data.enc_processing(train_input, char2idx)\n","train_output_dec, train_output_dec_length = data.dec_input_processing(train_label, char2idx)\n","train_target_dec = data.dec_target_processing(train_label, char2idx)"],"execution_count":25,"outputs":[{"output_type":"stream","text":["100%|██████████| 7921/7921 [00:05<00:00, 1368.21it/s]\n","100%|██████████| 7921/7921 [00:07<00:00, 1081.14it/s]\n","100%|██████████| 7921/7921 [00:07<00:00, 1079.93it/s]\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"p_ZEpxw4KJZv","colab_type":"code","outputId":"a0abf3b7-8944-426d-d1ef-2e89efe10db2","executionInfo":{"status":"ok","timestamp":1562364662277,"user_tz":-540,"elapsed":102832,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":67}},"source":["eval_input_enc, eval_input_enc_length = data.enc_processing(eval_input,char2idx)\n","eval_output_dec, eval_output_dec_length = data.dec_input_processing(eval_label, char2idx)\n","eval_target_dec = data.dec_target_processing(eval_label, char2idx)"],"execution_count":26,"outputs":[{"output_type":"stream","text":["100%|██████████| 3902/3902 [00:02<00:00, 1343.78it/s]\n","100%|██████████| 3902/3902 [00:03<00:00, 1078.64it/s]\n","100%|██████████| 3902/3902 [00:03<00:00, 1080.58it/s]\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"1n49nuuuKaAg","colab_type":"code","colab":{}},"source":["check_point_path = os.path.join(os.getcwd(), conf.DEFINES.check_point_path)\n","os.makedirs(check_point_path, exist_ok=True)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"W8km5p3iKs7E","colab_type":"code","outputId":"c58f9b9a-16db-4130-b85b-0c03c3184efd","executionInfo":{"status":"ok","timestamp":1562364662280,"user_tz":-540,"elapsed":102826,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":171}},"source":["seq2seq = tf.estimator.Estimator(\n","        model_fn=ml.model, # 모델 등록한다.\n","        model_dir=conf.DEFINES.check_point_path, # 체크포인트 위치 등록한다.\n","        params={ # 모델 쪽으로 파라메터 전달한다.\n","            'hidden_size': conf.DEFINES.hidden_size, # 가중치 크기 설정한다.\n","            'layer_size': conf.DEFINES.layer_size, # 멀티 레이어 층 개수를 설정한다.\n","            'learning_rate': conf.DEFINES.learning_rate, # 학습율 설정한다. \n","            'vocabulary_length': vocabulary_length, # 딕셔너리 크기를 설정한다.\n","            'embedding_size': conf.DEFINES.embedding_size, # 임베딩 크기를 설정한다.\n","            'teacher_forcing_rate': 0.7\n","        })"],"execution_count":28,"outputs":[{"output_type":"stream","text":["INFO:tensorflow:Using default config.\n","INFO:tensorflow:Using config: {'_model_dir': './data_out/check_point', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true\n","graph_options {\n","  rewrite_options {\n","    meta_optimizer_iterations: ONE\n","  }\n","}\n",", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f5c2d052da0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"nI7oIZGwS5aK","colab_type":"code","colab":{}},"source":["!rm -rf ./data_out/check_point"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"DGYbg9v_NwBh","colab_type":"text"},"source":["### 04:58분 시작"]},{"cell_type":"code","metadata":{"id":"8MQ8rwkaL186","colab_type":"code","outputId":"edd18afc-4c46-445f-c980-29b72d4b25dc","executionInfo":{"status":"ok","timestamp":1562368008680,"user_tz":-540,"elapsed":3449214,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":1000}},"source":["seq2seq.train(input_fn=lambda:data.train_input_fn(\n","    train_input_enc, train_output_dec, train_target_dec,  conf.DEFINES.batch_size), steps=conf.DEFINES.train_steps)"],"execution_count":30,"outputs":[{"output_type":"stream","text":["WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Colocations handled automatically by placer.\n","WARNING:tensorflow:From /content/gdrive/My Drive/Colab Notebooks/Seq2seq/data.py:173: DatasetV1.make_one_shot_iterator (from tensorflow.python.data.ops.dataset_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use `for ... in dataset:` to iterate over a dataset. If using `tf.estimator`, return the `Dataset` object directly from your input function. As a last resort, you can use `tf.compat.v1.data.make_one_shot_iterator(dataset)`.\n","INFO:tensorflow:Calling model_fn.\n","\n","WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n","For more information, please see:\n","  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n","  * https://github.com/tensorflow/addons\n","If you depend on functionality not listed there, please file an issue.\n","\n","WARNING:tensorflow:From /content/gdrive/My Drive/Colab Notebooks/Seq2seq/model_att.py:9: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n","WARNING:tensorflow:<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f5c054ae470>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","WARNING:tensorflow:<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f5c054ae5c0>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","WARNING:tensorflow:<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f5c054bfb70>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","WARNING:tensorflow:From /content/gdrive/My Drive/Colab Notebooks/Seq2seq/model_att.py:38: MultiRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","This class is equivalent as tf.keras.layers.StackedRNNCells, and will be replaced by that in Tensorflow 2.0.\n","WARNING:tensorflow:From /content/gdrive/My Drive/Colab Notebooks/Seq2seq/model_att.py:42: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn_cell_impl.py:1259: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n","WARNING:tensorflow:<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f5c0c2db710>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","WARNING:tensorflow:<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f5c04f1b7f0>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","WARNING:tensorflow:<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f5c050db4e0>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","WARNING:tensorflow:From /content/gdrive/My Drive/Colab Notebooks/Seq2seq/model_att.py:108: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use keras.layers.dense instead.\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/metrics_impl.py:455: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.cast instead.\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use tf.cast instead.\n","INFO:tensorflow:Done calling model_fn.\n","INFO:tensorflow:Create CheckpointSaverHook.\n","INFO:tensorflow:Graph was finalized.\n","INFO:tensorflow:Running local_init_op.\n","INFO:tensorflow:Done running local_init_op.\n","INFO:tensorflow:Saving checkpoints for 0 into ./data_out/check_point/model.ckpt.\n","INFO:tensorflow:loss = 9.44591, step = 0\n","INFO:tensorflow:global_step/sec: 3.49978\n","INFO:tensorflow:loss = 1.878986, step = 100 (28.575 sec)\n","INFO:tensorflow:global_step/sec: 6.2187\n","INFO:tensorflow:loss = 1.8499106, step = 200 (16.082 sec)\n","INFO:tensorflow:global_step/sec: 6.21819\n","INFO:tensorflow:loss = 1.7317423, step = 300 (16.082 sec)\n","INFO:tensorflow:global_step/sec: 6.15982\n","INFO:tensorflow:loss = 1.7275037, step = 400 (16.234 sec)\n","INFO:tensorflow:global_step/sec: 6.1401\n","INFO:tensorflow:loss = 1.435574, step = 500 (16.286 sec)\n","INFO:tensorflow:global_step/sec: 6.19026\n","INFO:tensorflow:loss = 1.4406939, step = 600 (16.154 sec)\n","INFO:tensorflow:global_step/sec: 6.16818\n","INFO:tensorflow:loss = 1.4646063, step = 700 (16.213 sec)\n","INFO:tensorflow:global_step/sec: 6.15902\n","INFO:tensorflow:loss = 1.220103, step = 800 (16.236 sec)\n","INFO:tensorflow:global_step/sec: 6.16848\n","INFO:tensorflow:loss = 1.43798, step = 900 (16.209 sec)\n","INFO:tensorflow:global_step/sec: 6.17411\n","INFO:tensorflow:loss = 1.4002222, step = 1000 (16.199 sec)\n","INFO:tensorflow:global_step/sec: 6.16209\n","INFO:tensorflow:loss = 1.397702, step = 1100 (16.226 sec)\n","INFO:tensorflow:global_step/sec: 6.14491\n","INFO:tensorflow:loss = 1.3407332, step = 1200 (16.274 sec)\n","INFO:tensorflow:global_step/sec: 6.16627\n","INFO:tensorflow:loss = 1.2048097, step = 1300 (16.217 sec)\n","INFO:tensorflow:global_step/sec: 6.17436\n","INFO:tensorflow:loss = 1.3174424, step = 1400 (16.198 sec)\n","INFO:tensorflow:global_step/sec: 6.13128\n","INFO:tensorflow:loss = 1.3016216, step = 1500 (16.310 sec)\n","INFO:tensorflow:global_step/sec: 6.14194\n","INFO:tensorflow:loss = 1.3853749, step = 1600 (16.281 sec)\n","INFO:tensorflow:global_step/sec: 6.20025\n","INFO:tensorflow:loss = 1.2058386, step = 1700 (16.130 sec)\n","INFO:tensorflow:global_step/sec: 6.16524\n","INFO:tensorflow:loss = 1.2347078, step = 1800 (16.220 sec)\n","INFO:tensorflow:global_step/sec: 6.0538\n","INFO:tensorflow:loss = 1.1815163, step = 1900 (16.518 sec)\n","INFO:tensorflow:global_step/sec: 6.15097\n","INFO:tensorflow:loss = 1.2964321, step = 2000 (16.258 sec)\n","INFO:tensorflow:global_step/sec: 6.19334\n","INFO:tensorflow:loss = 1.2645055, step = 2100 (16.147 sec)\n","INFO:tensorflow:global_step/sec: 6.16561\n","INFO:tensorflow:loss = 1.1567436, step = 2200 (16.219 sec)\n","INFO:tensorflow:global_step/sec: 6.13996\n","INFO:tensorflow:loss = 1.213918, step = 2300 (16.285 sec)\n","INFO:tensorflow:global_step/sec: 6.1684\n","INFO:tensorflow:loss = 1.1406472, step = 2400 (16.212 sec)\n","INFO:tensorflow:global_step/sec: 6.14953\n","INFO:tensorflow:loss = 1.3112167, step = 2500 (16.261 sec)\n","INFO:tensorflow:global_step/sec: 6.18579\n","INFO:tensorflow:loss = 1.2310902, step = 2600 (16.167 sec)\n","INFO:tensorflow:global_step/sec: 6.16436\n","INFO:tensorflow:loss = 1.2917695, step = 2700 (16.223 sec)\n","INFO:tensorflow:global_step/sec: 6.17728\n","INFO:tensorflow:loss = 1.1939225, step = 2800 (16.188 sec)\n","INFO:tensorflow:global_step/sec: 6.15108\n","INFO:tensorflow:loss = 1.2121692, step = 2900 (16.256 sec)\n","INFO:tensorflow:global_step/sec: 6.13515\n","INFO:tensorflow:loss = 0.9914197, step = 3000 (16.299 sec)\n","INFO:tensorflow:global_step/sec: 6.1497\n","INFO:tensorflow:loss = 1.2585225, step = 3100 (16.263 sec)\n","INFO:tensorflow:global_step/sec: 6.16689\n","INFO:tensorflow:loss = 1.1789075, step = 3200 (16.216 sec)\n","INFO:tensorflow:global_step/sec: 6.16756\n","INFO:tensorflow:loss = 1.0921613, step = 3300 (16.213 sec)\n","INFO:tensorflow:global_step/sec: 6.09696\n","INFO:tensorflow:loss = 1.2299471, step = 3400 (16.402 sec)\n","INFO:tensorflow:global_step/sec: 6.12629\n","INFO:tensorflow:loss = 0.92811865, step = 3500 (16.323 sec)\n","INFO:tensorflow:Saving checkpoints for 3545 into ./data_out/check_point/model.ckpt.\n","INFO:tensorflow:global_step/sec: 4.8481\n","INFO:tensorflow:loss = 1.1814351, step = 3600 (20.628 sec)\n","INFO:tensorflow:global_step/sec: 6.16709\n","INFO:tensorflow:loss = 1.1255263, step = 3700 (16.212 sec)\n","INFO:tensorflow:global_step/sec: 6.05541\n","INFO:tensorflow:loss = 1.1680417, step = 3800 (16.516 sec)\n","INFO:tensorflow:global_step/sec: 6.18123\n","INFO:tensorflow:loss = 1.0905596, step = 3900 (16.178 sec)\n","INFO:tensorflow:global_step/sec: 6.15217\n","INFO:tensorflow:loss = 0.9397093, step = 4000 (16.255 sec)\n","INFO:tensorflow:global_step/sec: 6.12894\n","INFO:tensorflow:loss = 0.9362621, step = 4100 (16.314 sec)\n","INFO:tensorflow:global_step/sec: 6.17444\n","INFO:tensorflow:loss = 1.0092889, step = 4200 (16.198 sec)\n","INFO:tensorflow:global_step/sec: 6.15507\n","INFO:tensorflow:loss = 0.85276514, step = 4300 (16.247 sec)\n","INFO:tensorflow:global_step/sec: 6.1686\n","INFO:tensorflow:loss = 0.984903, step = 4400 (16.212 sec)\n","INFO:tensorflow:global_step/sec: 6.11693\n","INFO:tensorflow:loss = 1.0449747, step = 4500 (16.348 sec)\n","INFO:tensorflow:global_step/sec: 6.17824\n","INFO:tensorflow:loss = 0.92098755, step = 4600 (16.184 sec)\n","INFO:tensorflow:global_step/sec: 6.19494\n","INFO:tensorflow:loss = 1.1073141, step = 4700 (16.142 sec)\n","INFO:tensorflow:global_step/sec: 6.15393\n","INFO:tensorflow:loss = 0.98024124, step = 4800 (16.247 sec)\n","INFO:tensorflow:global_step/sec: 6.16467\n","INFO:tensorflow:loss = 1.1845824, step = 4900 (16.226 sec)\n","INFO:tensorflow:global_step/sec: 6.14093\n","INFO:tensorflow:loss = 0.9778266, step = 5000 (16.283 sec)\n","INFO:tensorflow:global_step/sec: 6.18649\n","INFO:tensorflow:loss = 1.0296376, step = 5100 (16.164 sec)\n","INFO:tensorflow:global_step/sec: 6.15286\n","INFO:tensorflow:loss = 1.2878827, step = 5200 (16.252 sec)\n","INFO:tensorflow:global_step/sec: 6.13212\n","INFO:tensorflow:loss = 0.956422, step = 5300 (16.308 sec)\n","INFO:tensorflow:global_step/sec: 6.13895\n","INFO:tensorflow:loss = 0.989044, step = 5400 (16.294 sec)\n","INFO:tensorflow:global_step/sec: 6.14221\n","INFO:tensorflow:loss = 0.8809074, step = 5500 (16.276 sec)\n","INFO:tensorflow:global_step/sec: 6.17463\n","INFO:tensorflow:loss = 0.9294156, step = 5600 (16.193 sec)\n","INFO:tensorflow:global_step/sec: 6.08968\n","INFO:tensorflow:loss = 0.9585843, step = 5700 (16.422 sec)\n","INFO:tensorflow:global_step/sec: 6.16387\n","INFO:tensorflow:loss = 0.99988294, step = 5800 (16.226 sec)\n","INFO:tensorflow:global_step/sec: 6.18453\n","INFO:tensorflow:loss = 0.9137225, step = 5900 (16.171 sec)\n","INFO:tensorflow:global_step/sec: 6.15291\n","INFO:tensorflow:loss = 0.8134388, step = 6000 (16.249 sec)\n","INFO:tensorflow:global_step/sec: 6.21352\n","INFO:tensorflow:loss = 0.89391875, step = 6100 (16.100 sec)\n","INFO:tensorflow:global_step/sec: 6.1797\n","INFO:tensorflow:loss = 0.7970875, step = 6200 (16.179 sec)\n","INFO:tensorflow:global_step/sec: 6.18422\n","INFO:tensorflow:loss = 0.88335633, step = 6300 (16.169 sec)\n","INFO:tensorflow:global_step/sec: 6.16235\n","INFO:tensorflow:loss = 0.82702065, step = 6400 (16.227 sec)\n","INFO:tensorflow:global_step/sec: 6.1918\n","INFO:tensorflow:loss = 0.85268706, step = 6500 (16.151 sec)\n","INFO:tensorflow:global_step/sec: 6.18071\n","INFO:tensorflow:loss = 0.7282122, step = 6600 (16.177 sec)\n","INFO:tensorflow:global_step/sec: 6.19425\n","INFO:tensorflow:loss = 0.80339205, step = 6700 (16.144 sec)\n","INFO:tensorflow:global_step/sec: 6.18739\n","INFO:tensorflow:loss = 1.001495, step = 6800 (16.166 sec)\n","INFO:tensorflow:global_step/sec: 6.19092\n","INFO:tensorflow:loss = 0.6930074, step = 6900 (16.152 sec)\n","INFO:tensorflow:global_step/sec: 6.19485\n","INFO:tensorflow:loss = 0.8418407, step = 7000 (16.140 sec)\n","INFO:tensorflow:global_step/sec: 6.18564\n","INFO:tensorflow:loss = 0.71630234, step = 7100 (16.172 sec)\n","INFO:tensorflow:global_step/sec: 6.1199\n","INFO:tensorflow:loss = 0.9410371, step = 7200 (16.339 sec)\n","INFO:tensorflow:Saving checkpoints for 7216 into ./data_out/check_point/model.ckpt.\n","INFO:tensorflow:global_step/sec: 4.84139\n","INFO:tensorflow:loss = 0.709776, step = 7300 (20.651 sec)\n","INFO:tensorflow:global_step/sec: 6.1511\n","INFO:tensorflow:loss = 0.8548536, step = 7400 (16.261 sec)\n","INFO:tensorflow:global_step/sec: 6.17321\n","INFO:tensorflow:loss = 0.6874913, step = 7500 (16.198 sec)\n","INFO:tensorflow:global_step/sec: 6.07701\n","INFO:tensorflow:loss = 0.6393522, step = 7600 (16.456 sec)\n","INFO:tensorflow:global_step/sec: 6.18779\n","INFO:tensorflow:loss = 0.68695307, step = 7700 (16.158 sec)\n","INFO:tensorflow:global_step/sec: 6.12492\n","INFO:tensorflow:loss = 0.73908484, step = 7800 (16.329 sec)\n","INFO:tensorflow:global_step/sec: 6.13725\n","INFO:tensorflow:loss = 0.7593465, step = 7900 (16.295 sec)\n","INFO:tensorflow:global_step/sec: 6.18707\n","INFO:tensorflow:loss = 0.6806864, step = 8000 (16.162 sec)\n","INFO:tensorflow:global_step/sec: 6.1772\n","INFO:tensorflow:loss = 0.73670024, step = 8100 (16.186 sec)\n","INFO:tensorflow:global_step/sec: 6.19504\n","INFO:tensorflow:loss = 0.6622138, step = 8200 (16.143 sec)\n","INFO:tensorflow:global_step/sec: 6.20749\n","INFO:tensorflow:loss = 0.82478803, step = 8300 (16.110 sec)\n","INFO:tensorflow:global_step/sec: 6.13772\n","INFO:tensorflow:loss = 0.74472135, step = 8400 (16.293 sec)\n","INFO:tensorflow:global_step/sec: 6.1549\n","INFO:tensorflow:loss = 0.8327496, step = 8500 (16.248 sec)\n","INFO:tensorflow:global_step/sec: 6.17112\n","INFO:tensorflow:loss = 0.7379046, step = 8600 (16.204 sec)\n","INFO:tensorflow:global_step/sec: 6.18361\n","INFO:tensorflow:loss = 0.67458016, step = 8700 (16.172 sec)\n","INFO:tensorflow:global_step/sec: 6.17785\n","INFO:tensorflow:loss = 0.64928305, step = 8800 (16.186 sec)\n","INFO:tensorflow:global_step/sec: 6.15827\n","INFO:tensorflow:loss = 0.68931544, step = 8900 (16.238 sec)\n","INFO:tensorflow:global_step/sec: 6.14138\n","INFO:tensorflow:loss = 0.75737137, step = 9000 (16.280 sec)\n","INFO:tensorflow:global_step/sec: 6.15588\n","INFO:tensorflow:loss = 0.73358864, step = 9100 (16.247 sec)\n","INFO:tensorflow:global_step/sec: 6.16593\n","INFO:tensorflow:loss = 0.8588429, step = 9200 (16.217 sec)\n","INFO:tensorflow:global_step/sec: 6.15487\n","INFO:tensorflow:loss = 0.6223374, step = 9300 (16.251 sec)\n","INFO:tensorflow:global_step/sec: 6.1629\n","INFO:tensorflow:loss = 0.63295746, step = 9400 (16.225 sec)\n","INFO:tensorflow:global_step/sec: 6.08102\n","INFO:tensorflow:loss = 0.68776864, step = 9500 (16.444 sec)\n","INFO:tensorflow:global_step/sec: 6.16325\n","INFO:tensorflow:loss = 0.6438167, step = 9600 (16.225 sec)\n","INFO:tensorflow:global_step/sec: 6.15532\n","INFO:tensorflow:loss = 0.68408906, step = 9700 (16.246 sec)\n","INFO:tensorflow:global_step/sec: 6.18208\n","INFO:tensorflow:loss = 0.60805833, step = 9800 (16.177 sec)\n","INFO:tensorflow:global_step/sec: 6.17923\n","INFO:tensorflow:loss = 0.63798, step = 9900 (16.182 sec)\n","INFO:tensorflow:global_step/sec: 6.16164\n","INFO:tensorflow:loss = 0.58921546, step = 10000 (16.231 sec)\n","INFO:tensorflow:global_step/sec: 6.17374\n","INFO:tensorflow:loss = 0.75372815, step = 10100 (16.198 sec)\n","INFO:tensorflow:global_step/sec: 6.15981\n","INFO:tensorflow:loss = 0.8230141, step = 10200 (16.235 sec)\n","INFO:tensorflow:global_step/sec: 6.186\n","INFO:tensorflow:loss = 0.61795723, step = 10300 (16.165 sec)\n","INFO:tensorflow:global_step/sec: 6.13279\n","INFO:tensorflow:loss = 0.5799612, step = 10400 (16.303 sec)\n","INFO:tensorflow:global_step/sec: 6.18175\n","INFO:tensorflow:loss = 0.6140557, step = 10500 (16.180 sec)\n","INFO:tensorflow:global_step/sec: 6.15248\n","INFO:tensorflow:loss = 0.75801086, step = 10600 (16.254 sec)\n","INFO:tensorflow:global_step/sec: 6.17931\n","INFO:tensorflow:loss = 0.61770487, step = 10700 (16.182 sec)\n","INFO:tensorflow:global_step/sec: 6.14395\n","INFO:tensorflow:loss = 0.663692, step = 10800 (16.274 sec)\n","INFO:tensorflow:Saving checkpoints for 10885 into ./data_out/check_point/model.ckpt.\n","INFO:tensorflow:global_step/sec: 4.82405\n","INFO:tensorflow:loss = 0.49592, step = 10900 (20.733 sec)\n","INFO:tensorflow:global_step/sec: 6.15884\n","INFO:tensorflow:loss = 0.54151785, step = 11000 (16.236 sec)\n","INFO:tensorflow:global_step/sec: 6.15026\n","INFO:tensorflow:loss = 0.5591236, step = 11100 (16.260 sec)\n","INFO:tensorflow:global_step/sec: 6.17786\n","INFO:tensorflow:loss = 0.49740872, step = 11200 (16.186 sec)\n","INFO:tensorflow:global_step/sec: 6.12967\n","INFO:tensorflow:loss = 0.51651293, step = 11300 (16.315 sec)\n","INFO:tensorflow:global_step/sec: 6.0465\n","INFO:tensorflow:loss = 0.55551285, step = 11400 (16.535 sec)\n","INFO:tensorflow:global_step/sec: 6.12908\n","INFO:tensorflow:loss = 0.50126165, step = 11500 (16.319 sec)\n","INFO:tensorflow:global_step/sec: 6.16342\n","INFO:tensorflow:loss = 0.5801778, step = 11600 (16.224 sec)\n","INFO:tensorflow:global_step/sec: 6.13091\n","INFO:tensorflow:loss = 0.569376, step = 11700 (16.309 sec)\n","INFO:tensorflow:global_step/sec: 6.14716\n","INFO:tensorflow:loss = 0.64542544, step = 11800 (16.268 sec)\n","INFO:tensorflow:global_step/sec: 6.12087\n","INFO:tensorflow:loss = 0.52929765, step = 11900 (16.339 sec)\n","INFO:tensorflow:global_step/sec: 6.14889\n","INFO:tensorflow:loss = 0.6927161, step = 12000 (16.263 sec)\n","INFO:tensorflow:global_step/sec: 6.16315\n","INFO:tensorflow:loss = 0.63523364, step = 12100 (16.226 sec)\n","INFO:tensorflow:global_step/sec: 6.15871\n","INFO:tensorflow:loss = 0.5919337, step = 12200 (16.237 sec)\n","INFO:tensorflow:global_step/sec: 6.1481\n","INFO:tensorflow:loss = 0.43728587, step = 12300 (16.263 sec)\n","INFO:tensorflow:global_step/sec: 6.12754\n","INFO:tensorflow:loss = 0.4690597, step = 12400 (16.322 sec)\n","INFO:tensorflow:global_step/sec: 6.17587\n","INFO:tensorflow:loss = 0.49375212, step = 12500 (16.192 sec)\n","INFO:tensorflow:global_step/sec: 6.14019\n","INFO:tensorflow:loss = 0.56287557, step = 12600 (16.286 sec)\n","INFO:tensorflow:global_step/sec: 6.149\n","INFO:tensorflow:loss = 0.5083548, step = 12700 (16.262 sec)\n","INFO:tensorflow:global_step/sec: 6.12781\n","INFO:tensorflow:loss = 0.52693415, step = 12800 (16.319 sec)\n","INFO:tensorflow:global_step/sec: 6.16286\n","INFO:tensorflow:loss = 0.5712679, step = 12900 (16.227 sec)\n","INFO:tensorflow:global_step/sec: 6.13251\n","INFO:tensorflow:loss = 0.4810971, step = 13000 (16.306 sec)\n","INFO:tensorflow:global_step/sec: 6.17981\n","INFO:tensorflow:loss = 0.46640128, step = 13100 (16.179 sec)\n","INFO:tensorflow:global_step/sec: 6.16479\n","INFO:tensorflow:loss = 0.5035194, step = 13200 (16.225 sec)\n","INFO:tensorflow:global_step/sec: 6.05228\n","INFO:tensorflow:loss = 0.522413, step = 13300 (16.522 sec)\n","INFO:tensorflow:global_step/sec: 6.18479\n","INFO:tensorflow:loss = 0.45756927, step = 13400 (16.167 sec)\n","INFO:tensorflow:global_step/sec: 6.2019\n","INFO:tensorflow:loss = 0.51967657, step = 13500 (16.125 sec)\n","INFO:tensorflow:global_step/sec: 6.18734\n","INFO:tensorflow:loss = 0.4820292, step = 13600 (16.159 sec)\n","INFO:tensorflow:global_step/sec: 6.14798\n","INFO:tensorflow:loss = 0.6405527, step = 13700 (16.269 sec)\n","INFO:tensorflow:global_step/sec: 6.16456\n","INFO:tensorflow:loss = 0.5701801, step = 13800 (16.221 sec)\n","INFO:tensorflow:global_step/sec: 6.18367\n","INFO:tensorflow:loss = 0.4628784, step = 13900 (16.169 sec)\n","INFO:tensorflow:global_step/sec: 6.2087\n","INFO:tensorflow:loss = 0.51293963, step = 14000 (16.108 sec)\n","INFO:tensorflow:global_step/sec: 6.20276\n","INFO:tensorflow:loss = 0.44448486, step = 14100 (16.122 sec)\n","INFO:tensorflow:global_step/sec: 6.1913\n","INFO:tensorflow:loss = 0.46000195, step = 14200 (16.152 sec)\n","INFO:tensorflow:global_step/sec: 6.1842\n","INFO:tensorflow:loss = 0.40156373, step = 14300 (16.171 sec)\n","INFO:tensorflow:global_step/sec: 6.22475\n","INFO:tensorflow:loss = 0.5229053, step = 14400 (16.066 sec)\n","INFO:tensorflow:global_step/sec: 6.17399\n","INFO:tensorflow:loss = 0.47781402, step = 14500 (16.194 sec)\n","INFO:tensorflow:Saving checkpoints for 14553 into ./data_out/check_point/model.ckpt.\n","INFO:tensorflow:global_step/sec: 4.85962\n","INFO:tensorflow:loss = 0.40919805, step = 14600 (20.577 sec)\n","INFO:tensorflow:global_step/sec: 6.20769\n","INFO:tensorflow:loss = 0.46191996, step = 14700 (16.109 sec)\n","INFO:tensorflow:global_step/sec: 6.15962\n","INFO:tensorflow:loss = 0.434636, step = 14800 (16.236 sec)\n","INFO:tensorflow:global_step/sec: 6.21799\n","INFO:tensorflow:loss = 0.42059302, step = 14900 (16.085 sec)\n","INFO:tensorflow:global_step/sec: 6.23371\n","INFO:tensorflow:loss = 0.5164181, step = 15000 (16.040 sec)\n","INFO:tensorflow:global_step/sec: 6.15883\n","INFO:tensorflow:loss = 0.44024813, step = 15100 (16.236 sec)\n","INFO:tensorflow:global_step/sec: 6.07167\n","INFO:tensorflow:loss = 0.470094, step = 15200 (16.472 sec)\n","INFO:tensorflow:global_step/sec: 6.15986\n","INFO:tensorflow:loss = 0.44439286, step = 15300 (16.231 sec)\n","INFO:tensorflow:global_step/sec: 6.21038\n","INFO:tensorflow:loss = 0.3709181, step = 15400 (16.104 sec)\n","INFO:tensorflow:global_step/sec: 6.19382\n","INFO:tensorflow:loss = 0.45836318, step = 15500 (16.143 sec)\n","INFO:tensorflow:global_step/sec: 6.20116\n","INFO:tensorflow:loss = 0.4181847, step = 15600 (16.127 sec)\n","INFO:tensorflow:global_step/sec: 6.18904\n","INFO:tensorflow:loss = 0.48019925, step = 15700 (16.157 sec)\n","INFO:tensorflow:global_step/sec: 6.1776\n","INFO:tensorflow:loss = 0.43553525, step = 15800 (16.190 sec)\n","INFO:tensorflow:global_step/sec: 6.15624\n","INFO:tensorflow:loss = 0.41828713, step = 15900 (16.243 sec)\n","INFO:tensorflow:global_step/sec: 6.19363\n","INFO:tensorflow:loss = 0.42771232, step = 16000 (16.146 sec)\n","INFO:tensorflow:global_step/sec: 6.20952\n","INFO:tensorflow:loss = 0.43320054, step = 16100 (16.104 sec)\n","INFO:tensorflow:global_step/sec: 6.21402\n","INFO:tensorflow:loss = 0.5240876, step = 16200 (16.092 sec)\n","INFO:tensorflow:global_step/sec: 6.20811\n","INFO:tensorflow:loss = 0.42010653, step = 16300 (16.106 sec)\n","INFO:tensorflow:global_step/sec: 6.20314\n","INFO:tensorflow:loss = 0.44454342, step = 16400 (16.125 sec)\n","INFO:tensorflow:global_step/sec: 6.1325\n","INFO:tensorflow:loss = 0.37908536, step = 16500 (16.304 sec)\n","INFO:tensorflow:global_step/sec: 6.13885\n","INFO:tensorflow:loss = 0.3504328, step = 16600 (16.291 sec)\n","INFO:tensorflow:global_step/sec: 6.19806\n","INFO:tensorflow:loss = 0.41324112, step = 16700 (16.134 sec)\n","INFO:tensorflow:global_step/sec: 6.16044\n","INFO:tensorflow:loss = 0.46740326, step = 16800 (16.230 sec)\n","INFO:tensorflow:global_step/sec: 6.23627\n","INFO:tensorflow:loss = 0.36948106, step = 16900 (16.040 sec)\n","INFO:tensorflow:global_step/sec: 6.14348\n","INFO:tensorflow:loss = 0.37801537, step = 17000 (16.276 sec)\n","INFO:tensorflow:global_step/sec: 6.11502\n","INFO:tensorflow:loss = 0.42332995, step = 17100 (16.354 sec)\n","INFO:tensorflow:global_step/sec: 6.16811\n","INFO:tensorflow:loss = 0.5146849, step = 17200 (16.211 sec)\n","INFO:tensorflow:global_step/sec: 6.22499\n","INFO:tensorflow:loss = 0.40865996, step = 17300 (16.064 sec)\n","INFO:tensorflow:global_step/sec: 6.2297\n","INFO:tensorflow:loss = 0.5121224, step = 17400 (16.054 sec)\n","INFO:tensorflow:global_step/sec: 6.20824\n","INFO:tensorflow:loss = 0.355673, step = 17500 (16.106 sec)\n","INFO:tensorflow:global_step/sec: 6.23319\n","INFO:tensorflow:loss = 0.29980212, step = 17600 (16.043 sec)\n","INFO:tensorflow:global_step/sec: 6.20993\n","INFO:tensorflow:loss = 0.3714814, step = 17700 (16.103 sec)\n","INFO:tensorflow:global_step/sec: 6.21354\n","INFO:tensorflow:loss = 0.4897036, step = 17800 (16.093 sec)\n","INFO:tensorflow:global_step/sec: 6.23716\n","INFO:tensorflow:loss = 0.49091163, step = 17900 (16.033 sec)\n","INFO:tensorflow:global_step/sec: 6.1974\n","INFO:tensorflow:loss = 0.3782512, step = 18000 (16.136 sec)\n","INFO:tensorflow:global_step/sec: 6.20387\n","INFO:tensorflow:loss = 0.4306773, step = 18100 (16.120 sec)\n","INFO:tensorflow:global_step/sec: 6.1795\n","INFO:tensorflow:loss = 0.45464233, step = 18200 (16.180 sec)\n","INFO:tensorflow:Saving checkpoints for 18239 into ./data_out/check_point/model.ckpt.\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:966: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use standard file APIs to delete files with this prefix.\n","INFO:tensorflow:global_step/sec: 4.82214\n","INFO:tensorflow:loss = 0.52282923, step = 18300 (20.739 sec)\n","INFO:tensorflow:global_step/sec: 6.20178\n","INFO:tensorflow:loss = 0.3796684, step = 18400 (16.124 sec)\n","INFO:tensorflow:global_step/sec: 6.2\n","INFO:tensorflow:loss = 0.34998667, step = 18500 (16.126 sec)\n","INFO:tensorflow:global_step/sec: 6.25589\n","INFO:tensorflow:loss = 0.36947873, step = 18600 (15.987 sec)\n","INFO:tensorflow:global_step/sec: 6.21956\n","INFO:tensorflow:loss = 0.3623648, step = 18700 (16.078 sec)\n","INFO:tensorflow:global_step/sec: 6.17719\n","INFO:tensorflow:loss = 0.33826983, step = 18800 (16.190 sec)\n","INFO:tensorflow:global_step/sec: 6.22395\n","INFO:tensorflow:loss = 0.39126652, step = 18900 (16.065 sec)\n","INFO:tensorflow:global_step/sec: 6.12001\n","INFO:tensorflow:loss = 0.4172858, step = 19000 (16.341 sec)\n","INFO:tensorflow:global_step/sec: 6.24409\n","INFO:tensorflow:loss = 0.31487796, step = 19100 (16.015 sec)\n","INFO:tensorflow:global_step/sec: 6.22661\n","INFO:tensorflow:loss = 0.30724815, step = 19200 (16.058 sec)\n","INFO:tensorflow:global_step/sec: 6.19235\n","INFO:tensorflow:loss = 0.41598418, step = 19300 (16.148 sec)\n","INFO:tensorflow:global_step/sec: 6.18398\n","INFO:tensorflow:loss = 0.3196979, step = 19400 (16.173 sec)\n","INFO:tensorflow:global_step/sec: 6.16488\n","INFO:tensorflow:loss = 0.34731284, step = 19500 (16.223 sec)\n","INFO:tensorflow:global_step/sec: 6.18554\n","INFO:tensorflow:loss = 0.33786082, step = 19600 (16.166 sec)\n","INFO:tensorflow:global_step/sec: 6.22596\n","INFO:tensorflow:loss = 0.48051265, step = 19700 (16.062 sec)\n","INFO:tensorflow:global_step/sec: 6.22056\n","INFO:tensorflow:loss = 0.33147535, step = 19800 (16.076 sec)\n","INFO:tensorflow:global_step/sec: 6.2047\n","INFO:tensorflow:loss = 0.3958847, step = 19900 (16.117 sec)\n","INFO:tensorflow:Saving checkpoints for 20000 into ./data_out/check_point/model.ckpt.\n","INFO:tensorflow:Loss for final step: 0.30921665.\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["<tensorflow_estimator.python.estimator.estimator.Estimator at 0x7f5c2d052d30>"]},"metadata":{"tags":[]},"execution_count":30}]},{"cell_type":"code","metadata":{"id":"Z_oduPwcNFMR","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":406},"outputId":"19aac588-c1ad-4e32-ac81-89c9bf97f50d","executionInfo":{"status":"ok","timestamp":1562368022489,"user_tz":-540,"elapsed":3463008,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}}},"source":["eval_result = seq2seq.evaluate(input_fn=lambda:data.eval_input_fn(\n","    eval_input_enc, eval_output_dec, eval_target_dec,  conf.DEFINES.batch_size))\n","print('\\nEVAL set accuracy: {accuracy:0.3f}\\n'.format(**eval_result))"],"execution_count":31,"outputs":[{"output_type":"stream","text":["INFO:tensorflow:Calling model_fn.\n","WARNING:tensorflow:<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f5bf1c36dd8>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","WARNING:tensorflow:<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f5bf1c36908>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","WARNING:tensorflow:<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f5bf1c365f8>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","WARNING:tensorflow:<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f5bf1c36e48>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","WARNING:tensorflow:<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f5bf1c36550>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","WARNING:tensorflow:<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f5be0227390>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","INFO:tensorflow:Done calling model_fn.\n","INFO:tensorflow:Starting evaluation at 2019-07-05T23:06:53Z\n","INFO:tensorflow:Graph was finalized.\n","WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n","Instructions for updating:\n","Use standard file APIs to check for files with this prefix.\n","INFO:tensorflow:Restoring parameters from ./data_out/check_point/model.ckpt-20000\n","INFO:tensorflow:Running local_init_op.\n","INFO:tensorflow:Done running local_init_op.\n","INFO:tensorflow:Finished evaluation at 2019-07-05-23:07:01\n","INFO:tensorflow:Saving dict for global step 20000: accuracy = 0.7209841, global_step = 20000, loss = 3.166473\n","INFO:tensorflow:Saving 'checkpoint_path' summary for global step 20000: ./data_out/check_point/model.ckpt-20000\n","\n","EVAL set accuracy: 0.721\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Vz6N3UFMfM9S","colab_type":"code","outputId":"83c1e658-187e-4ff0-e39e-7fad06047f4d","executionInfo":{"status":"ok","timestamp":1562368030499,"user_tz":-540,"elapsed":3471012,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":339}},"source":["# 테스트용 데이터 만드는 부분이다.\n","# 인코딩 부분 만든다.\n","predic_input_enc, predic_input_enc_length = data.enc_processing([\"남자친구 승진 선물로 뭐가 좋을까?\"], char2idx)\n","# 학습 과정이 아니므로 디코딩 입력은 \n","# 존재하지 않는다.(구조를 맞추기 위해 넣는다.)\n","predic_output_dec, predic_output_decLength = data.dec_input_processing([\"\"], char2idx)       \n","# 학습 과정이 아니므로 디코딩 출력 부분도 \n","# 존재하지 않는다.(구조를 맞추기 위해 넣는다.)\n","predic_target_dec = data.dec_target_processing([\"\"], char2idx)      \n","\n","# 예측을 하는 부분이다.\n","predictions = seq2seq.predict(\n","    input_fn=lambda:data.eval_input_fn(predic_input_enc, predic_output_dec, predic_target_dec, conf.DEFINES.batch_size))\n","\n","# 예측한 값을 인지 할 수 있도록 \n","# 텍스트로 변경하는 부분이다.\n","answer = data.pred2string(predictions, idx2char)\n","print(answer)"],"execution_count":32,"outputs":[{"output_type":"stream","text":["100%|██████████| 1/1 [00:00<00:00, 135.62it/s]\n","100%|██████████| 1/1 [00:00<00:00, 513.25it/s]\n","100%|██████████| 1/1 [00:00<00:00, 478.80it/s]"],"name":"stderr"},{"output_type":"stream","text":["INFO:tensorflow:Calling model_fn.\n","WARNING:tensorflow:<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f5afc3d02b0>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","WARNING:tensorflow:<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f5afc3d00f0>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","WARNING:tensorflow:<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f5afc3d09b0>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n"],"name":"stdout"},{"output_type":"stream","text":["\n"],"name":"stderr"},{"output_type":"stream","text":["WARNING:tensorflow:<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f5afc3ef3c8>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","WARNING:tensorflow:<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f5afc3ef9e8>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","WARNING:tensorflow:<tensorflow.python.ops.rnn_cell_impl.BasicLSTMCell object at 0x7f5afc72d160>: Using a concatenated state is slower and will soon be deprecated.  Use state_is_tuple=True.\n","INFO:tensorflow:Done calling model_fn.\n","INFO:tensorflow:Graph was finalized.\n","INFO:tensorflow:Restoring parameters from ./data_out/check_point/model.ckpt-20000\n","INFO:tensorflow:Running local_init_op.\n","INFO:tensorflow:Done running local_init_op.\n","['직접', '물어보는', '게', '좋을', '것', '같아요', '<END>', '<PADDING>', '<PADDING>', '<PADDING>', '<PADDING>', '<PADDING>', '<PADDING>', '<PADDING>', '<PADDING>', '<PADDING>', '<PADDING>', '<PADDING>', '<PADDING>', '<PADDING>', '<PADDING>', '<PADDING>', '<PADDING>', '<PADDING>', '<PADDING>']\n","직접 물어보는 게 좋을 것 같아요 \n","직접 물어보는 게 좋을 것 같아요 \n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"N3ucNhJe3bEL","colab_type":"code","colab":{}},"source":["variable_names = seq2seq.get_variable_names()"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"awWW5zu3fcea","colab_type":"code","colab":{}},"source":["emb = seq2seq.get_variable_value(variable_names[2])"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"AkI40AsMfx9X","colab_type":"code","colab":{}},"source":["#emb"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"sfSp-Ppqf3mM","colab_type":"code","colab":{}},"source":["def plot_attention(attention, sentence, predicted_sentence):\n","    fig = plt.figure(figsize=(10,10))\n","    ax = fig.add_subplot(1, 1, 1)\n","    ax.matshow(attention, cmap='viridis')\n","\n","    fontdict = {'fontsize': 14}\n","\n","    ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n","    ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n","\n","    plt.show()"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"V7iZCAODonBZ","colab_type":"code","colab":{}},"source":["def translate(attention, sentence, result):\n","    print('Input: %s' % (sentence))\n","    print('Predicted translation: {}'.format(result))\n","\n","    attention_plot = attention[:len(result.split(' ')), :len(sentence.split(' '))]\n","    plot_attention(attention_plot, sentence.split(' '), result.split(' '))"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"r6sdAkuYvuWn","colab_type":"code","colab":{}},"source":["!apt-get update -qq #나눔고딕 인스톨\n","!apt-get install fonts-nanum* -qq"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"7QcUX3ujv9hm","colab_type":"code","outputId":"3164eb53-8842-4174-cd8f-0f40ae7f50c8","executionInfo":{"status":"ok","timestamp":1562368036795,"user_tz":-540,"elapsed":3477263,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["path = '/usr/share/fonts/truetype/nanum/NanumGothicEco.ttf'  \n","font_name = fm.FontProperties(fname=path, size=10).get_name() \n","print(font_name)\n","plt.rc('font', family=font_name)"],"execution_count":39,"outputs":[{"output_type":"stream","text":["NanumGothic Eco\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"lKGt3SJmwAjf","colab_type":"code","colab":{}},"source":["fm._rebuild()"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"0cayxsW_wDf6","colab_type":"code","colab":{}},"source":["import matplotlib\n","matplotlib.rcParams['axes.unicode_minus'] = False "],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"ivSVfrI7pEt1","colab_type":"code","outputId":"bdff52d6-209b-4a54-e7a2-c988a6119b17","executionInfo":{"status":"ok","timestamp":1562368037543,"user_tz":-540,"elapsed":3477997,"user":{"displayName":"ChangWook Jun","photoUrl":"https://lh6.googleusercontent.com/-NHLAhAbNfPg/AAAAAAAAAAI/AAAAAAAAAEc/NDLQ8z7-qns/s64/photo.jpg","userId":"00685987924881157185"}},"colab":{"base_uri":"https://localhost:8080/","height":671}},"source":["import matplotlib.pyplot as plt\n","\n","translate(emb, \"남자친구 승진 선물로 뭐가 좋을까?\", answer)"],"execution_count":42,"outputs":[{"output_type":"stream","text":["Input: 남자친구 승진 선물로 뭐가 좋을까?\n","Predicted translation: 직접 물어보는 게 좋을 것 같아요 \n"],"name":"stdout"},{"output_type":"display_data","data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAdEAAAJtCAYAAABpI0oIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmYZFV9//H3d6ZnZ19UQAFFcSUa\nQYMaUFEUo4gLRoNi1CQoKm7RiPoTl2hMFDciSsZd1LijghFREAXC4qDIoghEEBCUfRtmn+/vj3OL\nrunpWfpMd99b3e/X8/RTVbdOVX+rpqY/dc4999zITCRJ0tjNaLsASZIGlSEqSVIlQ1SSpEqGqCRJ\nlQxRSZIqGaKSJFUyRCVJqmSISpJUyRCVJKmSIToAIuI+ETGr7TokSWsyRAfD94BHt12EJGlNhmjH\nRMT8iNguIob6Nm8DRFs1SZJGN7ThJppkZwKrgdMi4nHAF4FTgHmtViVJWosh2j1DmfkXABGxLfBI\n4EWA+0QlqWMczu2ee8IyM2/OzNOAXYE7WqtIkjQqe6Lds3SUbUPAwyJil+b2LcBWwJLMPGnSKpOk\nChExD3g1sBh4GfDGzDy71aLGiSHaPaPt+5wDzAUeACTwS+BhzXVDVFLXfQg4HrgNuAjYKyIeDnw1\nM+9utbJNZIh2z5JRtt0FnJ+Zn+zbdsIk1SNJm+oWYFFmrgJ+B5wVEc8Ajo6IU4BLgD8DezW7sAaG\nIdo9c0bZFniIi6TBNa8J0Htk5g8j4lLgsZS/e9tQ5n8MFEO0e1aPsi2bH0kaREsj4l6ZeUP/xsy8\nEriyb9PnJresTWeIds9nR9m2GkNU0uD6OHB720VMBA9x6ZjM/Ogom2dgiEoaUJl5U2au6N2OiK3a\nrGc8GaKD4fnApW0XoekjIh4ZEVPiEAR10sAN266Lw7kDIDMvbLsGTS3N2szzgaWZubzZti9wTbOf\nahawY4slaoqIiDnAzP5NwNyImL+Oh6zOzNGOl+8kQ7RDIuI1lGX+TqHMVrsdeDBwcbN9CXBGZl4Q\nEd8GdsrMvduqVwPtbMqxxz+MiKObCR/vBt5BmegxA/8+aHz8BJgN3Av4X+Cq5vI/gQcBZwFPB/4H\neHtze582Cq3hf5Lu6e3/DMoHr3d4y0zW/PfaBthp0qvTVDEzM/eIiKcAWwA3UD5nvdnhgbt7NA4y\nc2MC8W3N5f+byFomgiHaIZl57HruPnnSCtF00FsZayZrHoPcW7s5Gf1wK2nMIuKJlC/+lwDXDvoq\nRf38pjkAImL7iPh823VoSrmruRy5iMeeEbE/ZVnJVUjj45PAo4EnAadHxOER8fSImLn+h3WfIdpB\nEfGMiHhp34m5DwF+NkrTZZNYlqaWuc3lMtbscS6gfN6S4aCVNtXmwH9l5kJgb+B8YAfgtRFxv1Yr\n20SGaIdExPsjYmtgZ+D+wBER8SLKv9OXRnmIx46q1srmcmRv8/zMfDllkfAtJrckTWHX0HzmMnN1\nZp6XmV8AfgC8MSKeEhEDmUcDWfQUdjiwH2UW20eBc4C3Aidn5mj7p1xPV7XW9dlZ3lwO/DCbOuV8\n4FkRcVhEPC4itgfIzCuANwO7UyZSDhxDtFtWU863NxugOd/euyn7pwCIiD0j4i3AuQzoh06d0Pvs\nzAWeFhFPBv5E+fxBGeVwn6jGy9OBC4ArKOdC/lxE/L+I2L7pmX5qkI4N7WeIdstShg9vASAzvwds\n2wzrQhkSWUIJ0cMnvUJNFV9rLs+m7JvaAfgucFmzfTWGqMbPXOBPmXlaZv4QeDZwGvDtiNiy3dI2\nTWS6W61rIuJo4D2ZeWdz+17AD4EnDOq3NQ2WiHg08J3M3LXtWjT4IuJc4KDM/NOI7Q8E3g8cnpm3\ntFLcJjJEOygits7MW0dsewJw+chTCUkTISK2A56cmd9suxapyxzO7aD+AI2IBc22swxQTZSIOK7Z\nLwrcc9YNA3QDIuITbdcwiCLi6Ij4q7brGA+uWNR9r4+IEzPzorYL6ZqIeA5wM+UYtC36rt8M3Juy\nb3lRZv5fRBwDbJeZh7RVb8c9mOFJRfdoDrmanZl/nvySBoLHatfZk3L0wbn9GyPi3sDczPxDK1VV\nMES7r7d2rtb2GOCnwHaUBfp/ATyQMlnmUZTP92+atntQgmLai4hPAq8EPghcC5zU3LUiIt4IPITy\nZeRjwL8DWwN/2UKpnRIRP6Isnn4u8HhKEJwSEacBP6Ysqv4OygkkPgj8JDOf1k613dGcxeVw4Bbg\n9mayJMDqiDiSsgj9Lyjv38eALSnv7UAwRDskIs6hHOQ+izJbckvKWpNfaO57JnAjcDrwJuDczHxc\nO9W2LzPfsZ67Tx1l25RZr3MTPQf4JfAZ4L8on7GknDnob4BjKGGwM+UL3IJ2yuyclzO8tvBony+A\nf2ouvwmsWEeb6eb1wAeAg4B3RMSZzfbVwFOBDwHvohydEJQvcAPDEO2Wx7P+XucRfdffiisWjSoi\n3gScmpm/HnHXnDbq6aAVwJ3NMPdMhlcmmk1ZZOFmymFU/ceSTnuZeV3bNQyoxwBXZOZJEfERyhq6\nSTlcbyawPSVAewbq/6kTizqkOeh41bp+gAMop7DqbZv2Z9mIiH0jYve+2/tR9uGNDFDw896zcsTt\nGZRe+r0ogblVs70Xnn7ZbkTENhGx1YZbqs/9gJua638GehOKen+/eqd/7PXcB2q1LP9zdEwzkWOz\nzLxmxPZ9KcO5p7RSWAdFxHzgJcBtETEb+CJlyOgD63iIC6qPbjnl1GiPouxPXtJs742KTPsva30O\nBw6IiB9QFqY4f5AmwbRkHtA74mAla59BaGTPc6C+7A5UsdPEM4C3RMSrImIfgIj4e2Av4G2Z6X6W\nYbOAAzLzXyhrcx4IfHfkAd19/LwXI5eLXEX5g3YCZQJRb5buUN/9KhY3J5n+POX8mC+OiEMGdfH0\nSXIXML+5PpfhXukChnugQZnABgM249meaPc8irKT/eHA6yJiT+ApwIsyc61DEFROLp2Zx0fETsDb\nIuLqzPw/gObcmA+gTKS5f3tldsrIVa96i873hm+3bS57PQUnZA2bA9Ac8vOZZjTkTcChEXFeZv62\n1eq66SrKBCKA+1LO6BIMD9+uoIRp73M4UCMffnvqnlXAqsw8E3ghZVr4pQzYjLVJdM8Xi8z8I/Bh\n4J8iYrdm847AfYDfUr6cqPREnxgRB1L2UfU+WzMpf9zuovxh24bSO5g12pNMU2uMBGXm3Zn5Psr7\nNK+dkjrvBGD7iPgw5djQ85vtvXXCeyc72JHhL3ADwxDtnnu+hWXxJcp+0JdExH3aK6uz1hhNycwr\nKUH6tIiIzPxiZr4nMz+TmV9sp8TOeQ/wHcp79xXgDMofs9XAtyhnc3k/ZT/Wt4Ej2ymzk0bdnZKZ\nHwOeHxF/M8n1dF5mfgvYn/L5OiQze/tEhyifr99Sjqu9lfL5e3sbddZyOLd71ho6y8wfR0QCr46I\n92bmyNmV01Jm3h4Rzx9l+40RsQVlSPziya+s2zLzc8Dn+rdFBMDSzDy22XQ58LNJLm0QXLCe+z4K\nHBkRZ/ROHqEiM09l7WNrl2dm/7KJP5nEksaNC9B3TEQ8ETizOaRl5H0HAztm5jGTX9lgiIhnAS/O\nzL9ru5ZB0ny2zm6GxKUJFRFHAT/NzDParmVT2RPtmMz8Gdxz+rOVI04P9B3cPzWq5iw3d1B2UUyJ\nha0nSkTsTdkPNYsymWM28Edg14i4X7P9isy8PiLeBmyfmW9qreCOiIi9GP7/N5Myi3SI8pnrnX91\nNmXId4gyt+GcFkrtrObL2iLgSZTdCETEtsC9BnVSlvtEu22N1YuaxRgGavr3JPpLygSiNUTECyLi\nxIj4m2jGLMXrKKvEHNFcPo+yqszLKDOYX8vwe/k0yrG3gldQ9u3tTpn0twWwG/C3wCOa28dS1h4+\niPJ+TnsR8ZC++RyHUr5gBMOduGcA/91GbePBnmiHRMTXgYOBt1Amd7y02XYo8F7K4S+zKT2tnwF7\nZubft1RuZzTnvtwd+CTwXJreQkTsCnyNsp7piZT38autFNkhfWeyOalZfefS3iFBjZF/0Fz2D8jM\nV/fdvGeSWkScBDwyM38WEbOcwLaWE4ClEfFqSu/8iub7bG9f4mKaCZXN6dFeArxhtF1aXWRPtFve\nBjyMclaNRZQew5mUVVIuBl4E/A9l9trplA+bSu/g5GYZxCGGlw17PHBMM5Hm85SA1ZpW0/wxa4bV\nRjPyuFKtKSlnCQIXphjNfMoqWAcC3+/bvkNEnEB5z1ZHxA6UY7oXMLyQf+fZE+2QzPz9yG0RsRB4\nf2beFBFk5sV9903r4clm8fQXAvfJzNGGg26nDFNCGWKbP0qb6W4lsKz5A3ZcRBwyyqIeHv/YiIjX\nUc6utD9l6HZmZn61OQ8mlJEPrekuSoguoZyqsGcWZTfMVyhffB+amf8dEdtT9pkeN8l1VjFEu28+\n8GzgP0e5b7pPrf4gZZr82/q2LWF42bAdGF5sfRUuWLGWzLw7IhYAbwSOXseqWC41OeyjlPPSfomy\nX++FzfaZAJn5/pbq6rLe7oCfAj9vPm9nN9tvZXiyVm8N4oMZoFWyDNHuu5jh3pQzc9f0VmD/iHht\n3/FmyfCXi7mU4XCAX1FO3i2gWdHpj8DTKSc0/2Zm/modzQ3RYZGZV9xzI6K3sPqyZmRob8rnbghY\n0qw8Nt31vtSeQznJ+zspx4x+nXLGoN7ugl4eXcjw6fk6z32i3XctZQYgDNgpgiZaZq7MzB8Cl0XE\nY5vNMxie1XwtsFlzfR5l+UQVHwNeQ5n08auRARoRBzSHI1yNn7t+I0d/dmgu51BGjWZS/r8egLOa\ne2ZB+f9K+WKxivI+3gmcTDnMaiWwPCLeRTmf7Ykt1Tpm9kS7byYDenaDSXQp5XCC8yizl3tnKTkT\n+GBE/AbYF/i3VqrroMw8ECAizgPuHxHPAb7fd47ah1DWMv0hZehNo7uz7/LeTc/T3uea+r947BUR\nvd0qN2fm4RHxQkoW3UEZ0t0uM7852UXWMkS7bznDIwYnRsQ/A7+jDPFe1FpVHZKZV0fEHyNiR8q3\n3qFm+00R8VLg3sAHMvP4NuvsombFmDOaEH0nZV3d3lqwWltExAuAG4EnA72h3dsouwvWmhwojgdu\naK6fATy0ud4L11XAHZl5M/CFyS1t0xmiHRMRb6cMZzwK+C7wFwyfqeQYyvGQUA6D+dxaTzB9ncXw\npKLeKZVwxZiNk5nfjYhHR8T+mfnjtuvpsMdT9nkuoayju6jZfhfl0AyNkJn/2nfzvxgeKVrZ3P8t\nysLzA8m1czsmIvaj9KaCMkS0OXBxZl7bamEDIiIeTFk796i2axk0ETEXmJ2Zd7Rdy6CJiN2BWzPz\nxrZrGQQR8RrgpMz8wwYbd5whKklSJWfnSpJUyRCVJKmSITogIuKwtmsYNL5ndXzf6vi+jd1UeM8M\n0cEx8B+2Fvie1fF9q+P7NnYD/54ZopIkVXJ2bp9Zsxfk3Plbb7hhC1YsX8ys2d08DC1WdvMztGLF\nYmbN6uh7try7y9EuX72E2TO6eeKWeQ9cueFGLbn7tmXM32pO22WM6s4bu/n/YOWSxQzN62Zty+68\nhZVLF2/wTFkuttBn7vyt+cu/fl3bZQycOTd5usmxGrr6hg030loe9lXftxqnf2LvtksYOJd+76Mb\n1c7hXEmSKhmikiRVMkQlSapkiEqSVMkQlSSpkiEqSVIlQ1SSpEqGqCRJlQxRSZIqGaKSJFUyRCVJ\nqmSISpJUyRCVJKmSISpJUiVDVJKkSoaoJEmVDFFJkioZopIkVTJEJUmqZIhKklTJEJUkqZIhKklS\nJUNUkqRKhqgkSZUMUUmSKhmikiRVMkQlSapkiEqSVMkQlSSpkiEqSVIlQ1SSpEoDFaIRcUxE7D+G\n9v8aEc+byJokSdNXZ0I0IvaKiN9HxOUR8X8RcXVE3BARz+xr9iPgzqb9HhFxcURcEhEXRsRFEXFZ\nRBzQ134mMGsyX4ckafoYaruAnsxcBDygdzsiZgHvBP7U12x/4IdN+4uAR/S1D+BIYFFEbNZsvgi4\ndWIrlyRNV50J0VG8ArguM8/v23YXkCMbRsQQ8GTgx8CDgWcBAcwFTpr4UiVJ01EnQzQi3gXsDfzd\niLsWMCJEI2IG8ChgV0rP9LOZeVZz3xuAzZAkaQJ0Zp8olB5lRHwd2BH4FHB4RLw/IuY2TX4FLO1r\nfy9K0G6XmZ8GVgNviIiXN00uAW6ftBcgSZpWOhOiEbElJSBPzcxXZub3gc8Bj2O4R/osYMum/ZuB\nM4DLMvNkgMz8OPBpyr5QgH2BbTbwew+LiEURsWjF8sXj/KokSVNZl4ZzDwUelZkX9zZk5p+B/ZpJ\nQwDvAe5orv8c+BjwqohY3dt3mpnXA9c3bW7oaz+qzFwILATYfKv7rrW/VZKkdelMiGbmJwAi4mTg\nIZReaW8Yd1lEzKdMLDqoaX9e0/4m1v06bqQc5iJJ0rjrTIj2eR/QC8wFzba7ga2AXYAdgMv62t8J\nbB8Ru1Fm5EbffU8CTp/YciVJ01XnQjQzz1zXfRHxSeD3IzavBFYBixkO0Wwu1zuUK0nSpuhciG7A\natbsaQLsDFyemX8a2TgifoOzcyVJE2TQQnQJsHzEttnAT4fnHq3lyRNakSRp2hqoEM3Mt4yy7Vjg\n2BbKkSRNc505TlSSpEFjiEqSVMkQlSSpkiEqSVIlQ1SSpEqGqCRJlQxRSZIqGaKSJFUyRCVJqmSI\nSpJUyRCVJKmSISpJUiVDVJKkSoaoJEmVDFFJkioZopIkVTJEJUmqZIhKklTJEJUkqZIhKklSJUNU\nkqRKhqgkSZUMUUmSKhmikiRVMkQlSapkiEqSVMkQlSSpkiEqSVIlQ1SSpEqGqCRJlYbaLqBLVm63\nmhv/4e62yxg4K3+zRdslDJy5e6xsu4SBdO3V27RdwkC674+vabuEgXPFHcs3qp09UUmSKhmikiRV\nMkQlSapkiEqSVMkQlSSpkiEqSVIlQ1SSpEqGqCRJlQxRSZIqGaKSJFUyRCVJqmSISpJUyRCVJKmS\nISpJUiVDVJKkSoaoJEmVDFFJkioZopIkVTJEJUmqZIhKklTJEJUkqZIhKklSJUNUkqRKhqgkSZUM\nUUmSKhmikiRVMkQlSapkiEqSVMkQlSSpkiEqSVIlQ1SSpEqGqCRJlTY6RCPi3RFxZURcHhHXR8Rl\nEXFtRFwREddFxFFj+cUR8YGIOHjsJa/1PLtHxDc29XkkSRqrsfREDwUWAH8EfgdcB1wBXAvcBry6\nv3FEPDoisvm5JCL2GfF8BwK7jnjM1yPi8HUVEBEnRMScEZtnA/PH8DokSRoXQ2NoexRwY2aeMvKO\niNgPeEjf7c2AvweeC5wMPAb4h4i4IjOvb5p9ArhqxFN9Abh9PTXcDsQo23PjXoIkSeNnLCG6HHhz\nRLx9lPvuBL7dd/sFwLLM/G5z+4yIeABwb6AXovcFbh3xPK8HdoqIhcDmzc8QcCWwBXDBKL87gVMj\n4tHN9RkMB+3q5vKqzLxlo16lJEkbaSwh+nNgCbCMMnyazeNXAX+mDOv23A2cPuLxFwAvYjgIzwVW\njmhzHGV49mRKGM5uti9pfte/jFLXtcAhwIuBFcC85mcVcEdT5yeBL2/k65QkaaNsMEQj4gjg+aPc\ntYISULP72kIJ27MpAdbvT6VJXN3cXgy8d0SbVcAtmTnysb3nf9zIbZl5O/DYDb0OSZLG28b0RE+i\n9B57Q6TLKMOkhwNfaW4va+6bS+ldLmDtSUv3AR6RmTsDRMTRrL1/cxfgn5qgXQ1cBDyeMnFpDnA/\nyrAyEbEb8NmNeZGNVcDTMnNV/8aIOAw4DGBo+y3H8HSSpOlugyGamVcCV0bE+4C/7rvrYuDdlHBa\nAXwxM78MEBHPBvYAzuhr/0DWDL3/ZURvNTM/QZlwRPM8rweOyswzRyltOXB08/u3pPSKb6UMNc+n\n7KdNYDNKsG+2jte3EFgIMO+BOzpBSZK00cayT/RrwM8owXQnsC1wIqWXuAfwoL62ZwEPa44D/R6w\nD2Wm7jv72jwW+NUGfuee62qTmdcA10TEnsDDMvPDY3gtkiRtsrGE6MOAVwIzR7lvK+BHvRuZeXOz\nAMLHgY9Rjit9S9Or7VlJM5wbEdsDOzfbZzQ/K4CfArtGxApKj3Nm87gVmdmboLQVZfauJEmTaiwh\nujtwGmXINIGllCHVoOwTXSPIMvP3lAUV1uU8yuQigCcDH2qedyYwixKad1IOc5lFCdXZze+9Dvir\n5rGzKMPM3xnDa5EkaZONJUR3Ah5HGb4dzQpg/zE831HA8cCpmfkNoHbpvtXALRFx+nranJCZH698\nfkmSRjWWEL0EOICy6ME8Sq8xKMeE9nqmY/Ej1jy2tNaVlONAe0PAMxjuza5u6lowDr9HkqQ1bHSI\njpw5u6ky8x3j9DyXU2b+SpI0qTwVmiRJlQxRSZIqGaKSJFUyRCVJqmSISpJUyRCVJKmSISpJUiVD\nVJKkSoaoJEmVDFFJkioZopIkVTJEJUmqZIhKklTJEJUkqZIhKklSJUNUkqRKhqgkSZUMUUmSKhmi\nkiRVMkQlSapkiEqSVMkQlSSpkiEqSVIlQ1SSpEqGqCRJlQxRSZIqGaKSJFUyRCVJqmSISpJUyRCV\nJKnSUNsFdMmsPwc7fnRW22UMnJO/fmzbJQycq1be3XYJA+mI3/9t2yUMpKUP2qHtEgbO6ls3Lgvs\niUqSVMkQlSSpkiEqSVIlQ1SSpEqGqCRJlQxRSZIqGaKSJFUyRCVJqmSISpJUyRCVJKmSISpJUiVD\nVJKkSoaoJEmVDFFJkioZopIkVTJEJUmqZIhKklTJEJUkqZIhKklSJUNUkqRKhqgkSZUMUUmSKhmi\nkiRVMkQlSapkiEqSVMkQlSSpkiEqSVIlQ1SSpEqGqCRJlQxRSZIqGaKSJFUyRCVJqmSISpJUacqG\naEQcHRH7RMQuEfH1tuuRJE09Q20XsCki4jHA04EVwGLgOuCEzExgFeX1zQK2aK1ISdKUNdAhmpm/\nAH7Rux0RhwBPA34EnAcsb+5aNPnVSZKmuqk2nDsDuKi5/gRgTnP94nbKkSRNZVMmRCNia+BVwCMi\n4h2UHurNzd3PbK0wSdKUNWVCFHgJcAowH7gF2JHhfaE3tFWUJGnqGuh9oj0R8SBga+A/M/PWZtsx\nDO8LvWQ9jz0MOAxgzpwtJ7hSSdJUMvA90Yh4AHAA8KNegDZmATM39PjMXJiZe2XmXrNnLZioMiVJ\nU9BA90QjYh9gX+DMzDx3xN0nA0ub6w+e1MIkSdPCoPdE9we+lZk/G+W+fYC5zfULJ68kSdJ0MdA9\n0cw8aj13/wZY0lzfdhLKkSRNM4PeE12fVwK9nZwe4iJJGncD3RNdn8z8K4CI2IXh40UlSRo3UzZE\nezLzD8CL265DkjT1TOXhXEmSJpQhKklSJUNUkqRKhqgkSZUMUUmSKhmikiRVMkQlSapkiEqSVMkQ\nlSSpkiEqSVIlQ1SSpEqGqCRJlQxRSZIqGaKSJFUyRCVJqmSISpJUyRCVJKmSISpJUiVDVJKkSoao\nJEmVDFFJkioZopIkVTJEJUmqZIhKklTJEJUkqZIhKklSJUNUkqRKhqgkSZUMUUmSKhmikiRVGmq7\ngC7ZfOfFPPGT57RdxsB5+ov/se0SBs7sS65pu4SBdNWntmm7hIG02c6z2y5h4Ky+IDaqnT1RSZIq\nGaKSJFUyRCVJqmSISpJUyRCVJKmSISpJUiVDVJKkSoaoJEmVDFFJkioZopIkVTJEJUmqZIhKklTJ\nEJUkqZIhKklSJUNUkqRKhqgkSZUMUUmSKhmikiRVMkQlSapkiEqSVMkQlSSpkiEqSVIlQ1SSpEqG\nqCRJlQxRSZIqGaKSJFUyRCVJqmSISpJUyRCVJKmSISpJUiVDVJKkSoaoJEmVDFFJkioZopIkVRpq\nu4CRIuIo4FAggBXAHGApsBlwN/C8zPzNiMccDDwP2LVptwS4BvgF8PHMXDpZ9UuSpo/OhShwA3Dt\nKNvvAOYDt/VvjIinAN8EbgL+0Nw/C3gk8HxKEH9kAuuVJE1TnQvRzDwOOA4gImYBewNPpgToiZl5\n3YiHPAP4GPCmzMz+OyLiQGDbCS9akjQtdSpEI2IL4JXAvsAiYC5wJPAB4DLgzRFxJHB8Zn6jedg7\ngK8A10TEtZSh3yHg3sAtwL9P6ouQJE0bnQpRYDbwc2BhZt4eEUPA/Mx8O0BEfB54ILCy94DMXBYR\nLwf2BO4HbAdsDpwL/DIzb5zk1yBJmiY6E6IR8Vrg4L7bUCYXnR0Rp4/S/qeUYd512a/veaAE6ptG\neZ7DgMMAttxhXl3xkqRpqTMhCvyAMmQ7RAnPbSkzbQO4nNL7vL65vbL5+TWl99rrmc4CFgNJOXxn\nCJjZXL9mtF+amQuBhQA7PXyrHK2NJEmj6UyIZuaVwJW92xFxHGXS0G3A1sCvMvOgkY+LiCMos3BH\nsxr4SWb+2/hXLEma7joToj0R8QJK7/Ek4BTK5KKVwLKIeB6wADi5b1/nWZQJREHpic4CllF6ozcA\nL5zUFyBJmjY6F6LAEcCWlOCcSQnHpPQqZzQ/vwR6Ibot8DJGfy3zgC9NbLmSpOmqcyGamftGxPbA\nFgzv79wF+C1wfWauHvGQhwNnA9c17W9tts9qLn854UVLkqalzoVo41nALpn57ub25RFxCPB74JwR\nbbcEngvcvI7nOjQiDsnM6yekUknStNXVEN0OmBkRxwJ/AXwLeBRl7dyRIfo7yizezShDvXMoQ7+r\nKPtG76TsV5UkaVx1MkQz80MR8bfAsymLyT8BOB349ChtvwZ8bVILlCSJjoYoQLOs3zc22FCSpJZ4\nPlFJkioZopIkVTJEJUmqZIhKklTJEJUkqZIhKklSJUNUkqRKhqgkSZUMUUmSKhmikiRVMkQlSapk\niEqSVMkQlSSpkiEqSVIlQ1SSpEqGqCRJlQxRSZIqGaKSJFUyRCVJqmSISpJUyRCVJKmSISpJUiVD\nVJKkSoaoJEmVDFFJkioZopIkVTJEJUmqZIhKklRpqO0CuuT26zfnf97/pLbLGDj/8ukvt13CwHnL\nCYe2XcJA+u5eH2m7hIH0poMOCy7YAAAJyElEQVQf13YJA2dmLt6odvZEJUmqZIhKklTJEJUkqZIh\nKklSJUNUkqRKhqgkSZUMUUmSKhmikiRVMkQlSapkiEqSVMkQlSSpkiEqSVIlQ1SSpEqGqCRJlQxR\nSZIqGaKSJFUyRCVJqmSISpJUyRCVJKmSISpJUiVDVJKkSoaoJEmVDFFJkioZopIkVTJEJUmqZIhK\nklTJEJUkqZIhKklSJUNUkqRKhqgkSZUMUUmSKhmikiRVMkQlSao05UM0Ir4fEXu3XYckaeoZaruA\nTRER9wUOAlYDM4G7gc8Cr8/MY5pm5zb3SZI0rgY6RDPzWuDY/m0R8SDgu32bbgFiMuuSJE0PAx2i\nI0XEgcBuwOsiYq9m8/2Bl7ZXlSRpqpoyIRoRDwGeDxwCvABYRdnn+xPKcK8kSeNqSoRoRDwd+Gfg\nwMxcCfx380NEHA/8fD2PPQw4DGD2/K0nvlhJ0pQx0CHa9D7fBzwUeCOjTyA6GVixrufIzIXAQoDN\ntrlfTkCZkqQpaqBDFNgZ+DXweeCCzLw7ImYB8zLzjqbNUqbBoTySpMk30CGamacAp4zY/AJKuP57\nc/uXwPWTWZckaXqYUj20iJgHPBiYFxF7AGTmlZm5tN3KJElT0UD3RPtFxE7AK4BfAJcCz4+IeZl5\nXruVSZKmqoEP0YjYHXgZsCtwbGae1Ww/vrl8M2XVorMy00NdJEnjZioM5z4U2Bp4dy9AATLzOmA7\nSq90b2CndsqTJE1VA98TzczvAd9bx30XAhcCJ01qUZKkaWEq9EQlSWqFISpJUiVDVJKkSoaoJEmV\nDFFJkioZopIkVTJEJUmqZIhKklTJEJUkqZIhKklSJUNUkqRKhqgkSZUMUUmSKhmikiRVMkQlSapk\niEqSVMkQlSSpkiEqSVIlQ1SSpEqGqCRJlQxRSZIqGaKSJFUyRCVJqmSISpJUyRCVJKmSISpJUiVD\nVJKkSoaoJEmVDFFJkioNtV1Al8y4dTGbf+PctssYOB95xf5tlzBwLn/Jp9ouYSCds3R22yUMpBVP\n3bPtEgZOnnP2RrWzJypJUiVDVJKkSoaoJEmVDFFJkioZopIkVTJEJUmqZIhKklTJEJUkqZIhKklS\nJUNUkqRKhqgkSZUMUUmSKhmikiRVMkQlSapkiEqSVMkQlSSpkiEqSVIlQ1SSpEqGqCRJlQxRSZIq\nGaKSJFUyRCVJqmSISpJUyRCVJKmSISpJUiVDVJKkSoaoJEmVDFFJkioZopIkVTJEJUmqZIhKklTJ\nEJUkqZIhKklSpdZCNCJ2j4hvjqH9vIhYNJE1SZI0FkMT9cQR8VzgkUACq5vNTwFelZmXArOBg0c8\nZi/gaOA+wJXAf2Tm6c3dM4FT1/G7Hgg8AfhC3+avAEcCf8zM3PRXJEnSmiYsRDPzBOCE3u2I2AJ4\nBHB5X7M3R8TDgWXAcmB/4DWZeUlE7AfsFxFXZeZVTfuDgLf2/56IeCTwnOZ5HwBcC8wH9gBeBVzB\nmuEqSdK4mMzh3NcDl2Tmqr5tDwLOooTc/YFrMvMSgMw8jRKAD+9r/+tRnncX4ILM/GpmXpmZKzLz\n9sw8EzgK2C0iHjH+L0eSNN1NeIhGcSQlAD884u6vZOZWmfnXlOHd60bcfzXwgr7b9x3lVzwV2HId\nv34u8DKcQCVJmgATNpwLEBEPAP4VOARYkJl3j2jyWWD35voTgO+NuP9GoH8y0a8j4j8o+1nfl5l3\nAV8E/i0ikhLCQQnNLSi92G9n5oXj96okSSomcmLR54DTKPswX7KOyT3v7Lt+OjBrxP07Ak8EPtHc\n3icz9+hvkJnnR8ThwBHAP1P2h64E/gx8GXjvBuo8DDgMYC7zN/i6JEnqmcie6HeAy4D3APdreopB\n6UUmsDlwN/D1pv29KcOy5/U9x86U3mrPiev4XVsAP8nMN/Y2RMSOwEEbmpmbmQuBhQBbxDbO4pUk\nbbSJnJ17UkTMBu4EjqP0DmcAK5rrALf1PeQM4B8j4gDgJ8AzgcdShoN7ngq8fZRft4C1e7HzKbN9\nP7Vpr0SSpNFN6D5RSmg+JTPfsKGGmXldRPwA+ADwGeBC4MjMvLav2T0ze5ue5m6UXu0jgS0i4s6+\ntvcBzmoOlVkFzGxm/EqSNC4mOkQBTmtm566iDOfOpMyaTeA3mfmNXsPMPB942nqe65S+638NfKh5\nnpWUnmcwPGQ8gzJcPLe5fzkldCVJGhcTHaIJPJq+HuQIT4yIUzPz5o18vqOAdwE04fuN9TeXJGni\nTGiIZuayiAiGD2NZ0fzMAOY0l3M38ulWA/877kVKklRpwodzm4UUxuN57qYcSypJUie4ko8kSZUM\nUUmSKhmikiRVMkQlSapkiEqSVMkQlSSpkiEqSVIlQ1SSpEqGqCRJlQxRSZIqGaKSJFUyRCVJqmSI\nSpJUyRCVJKmSISpJUiVDVJKkSoaoJEmVDFFJkioZopIkVTJEJUmqZIhKklTJEJUkqZIhKklSJUNU\nkqRKhqgkSZUMUUmSKhmikiRVMkQlSapkiEqSVMkQlSSp0lDbBXTJjnss5j3fX9R2GQPnXbu1XcHg\nueHqxW2XMJBWMa/tEgbSvEv+2HYJA2fG0uUb126C65AkacoyRCVJqmSISpJUyRCVJKmSISpJUiVD\nVJKkSoaoJEmVDFFJkioZopIkVTJEJUmqZIhKklTJEJUkqZIhKklSJUNUkqRKhqgkSZUMUUmSKhmi\nkiRVMkQlSapkiEqSVMkQlSSpkiEqSVIlQ1SSpEqGqCRJlQxRSZIqGaKSJFUyRCVJqmSISpJUyRCV\nJKmSISpJUiVDVJKkSoaoJEmVDFFJkioZopIkVTJEJUmqZIhKklTJEJUkqZIhKklSJUNUkqRK0z5E\nI+KwiFgUEYtuu3lV2+VIkgbItA/RzFyYmXtl5l5bbTuz7XIkSQNk2oeoJEm1DFFJkioZopIkVTJE\nJUmqZIhKklTJEJUkqZIhKklSJUNUkqRKhqgkSZUMUUmSKhmikiRVMkQlSapkiEqSVMkQlSSpkiEq\nSVIlQ1SSpEqGqCRJlQxRSZIqGaKSJFUyRCVJqmSISpJUyRCVJKmSISpJUiVDVJKkSoaoJEmVDFFJ\nkioZopIkVTJEJUmqZIhKklTJEJUkqZIhKklSJUNUkqRKhqgkSZUMUUmSKhmikiRVMkQlSapkiEqS\nVCkys+0aOiMibgT+0HYd67AdcFPbRQwY37M6vm91fN/Grsvv2S6Zuf2GGhmiAyIiFmXmXm3XMUh8\nz+r4vtXxfRu7qfCeOZwrSVIlQ1SSpEqG6OBY2HYBA8j3rI7vWx3ft7Eb+PfMfaKSJFWyJypJUiVD\nVJKkSoaoJEmVDFFJkioZopIkVfr/Ucpr40tRY7cAAAAASUVORK5CYII=\n","text/plain":["<Figure size 720x720 with 1 Axes>"]},"metadata":{"tags":[]}}]},{"cell_type":"code","metadata":{"id":"qZr8ELYq8SxF","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]}]}