10PretrainedModelForEmbeddings.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "10PretrainedModelForEmbeddings.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3fGqwyWhn3Bj",
        "colab_type": "text"
      },
      "source": [
        "# 1) Import the libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MV_Fvqw2n4GG",
        "colab_type": "code",
        "outputId": "0eaa97c6-54d0-4136-9656-8107f5d69616",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "import tensorflow as tf\n",
        "print(tf.__version__)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.2.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LRLOb-Ntn4Ig",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import gensim\n",
        "from gensim.models import Word2Vec, KeyedVectors"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dum09TbwoMdd",
        "colab_type": "text"
      },
      "source": [
        "# 2) Data Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IhiTDbRQoNXN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# google pretrained model\n",
        "\n",
        "# https://www.kaggle.com/umbertogriffo/googles-trained-word2vec-model-in-python"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E5qaZvIeoNaR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# install kaggle API\n",
        "! pip install -q kaggle"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vbCWdojhoNc5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# create a directory as kaggle\n",
        "! mkdir -p ~/.kaggle"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qqGhLnFNoNe2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# copy API key to kaggle directory\n",
        "! cp kaggle.json ~/.kaggle"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e9F3g_xkoNho",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# disable the API key\n",
        "! chmod 600 /root/.kaggle/kaggle.json"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4ZxBwxJXoX-4",
        "colab_type": "code",
        "outputId": "e3e44b1e-30c8-4b9c-9c63-20e6ba805ca2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "# import the dataset\n",
        "! kaggle datasets download -d umbertogriffo/googles-trained-word2vec-model-in-python"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading googles-trained-word2vec-model-in-python.zip to /content\n",
            " 99% 3.16G/3.17G [01:26<00:00, 45.1MB/s]\n",
            "100% 3.17G/3.17G [01:26<00:00, 39.3MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CY-sGCLToYBc",
        "colab_type": "code",
        "outputId": "2b63a27e-ae5e-4632-fb2f-702b34d1e011",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "# unzip the dataset\n",
        "! unzip /content/googles-trained-word2vec-model-in-python"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/googles-trained-word2vec-model-in-python.zip\n",
            "  inflating: GoogleNews-vectors-negative300.bin  \n",
            "  inflating: GoogleNews-vectors-negative300.bin.gz  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nXDTMbm2o2SF",
        "colab_type": "text"
      },
      "source": [
        "# 3) Build the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "94q1L1jCoYGo",
        "colab_type": "code",
        "outputId": "be9eade1-d14e-444b-8916-493cc53038a3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "model = KeyedVectors.load_word2vec_format('/content/GoogleNews-vectors-negative300.bin', binary=True, limit=100000)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:253: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Trl53YMUqBa3",
        "colab_type": "text"
      },
      "source": [
        "# 4) Predict the output"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nIs85s2AqCZG",
        "colab_type": "code",
        "outputId": "1f917403-ef4e-47c9-d35c-38939809bf18",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        }
      },
      "source": [
        "# find 10 closest words in the vector space that we have created\n",
        "model.most_similar('man')"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
            "  if np.issubdtype(vec.dtype, np.int):\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('woman', 0.7664012908935547),\n",
              " ('boy', 0.6824870109558105),\n",
              " ('teenager', 0.6586930155754089),\n",
              " ('teenage_girl', 0.6147903800010681),\n",
              " ('girl', 0.5921714305877686),\n",
              " ('robber', 0.5585119128227234),\n",
              " ('teen_ager', 0.5549196600914001),\n",
              " ('men', 0.5489763021469116),\n",
              " ('guy', 0.5420035123825073),\n",
              " ('person', 0.5342026352882385)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e3mnVjCCqCbw",
        "colab_type": "code",
        "outputId": "cb529b6a-a85d-41e6-87d9-39abe1296aab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# see the vector\n",
        "model['man']\n",
        "\n",
        "# so this is how man is represented in our vector space"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 0.32617188,  0.13085938,  0.03466797, -0.08300781,  0.08984375,\n",
              "       -0.04125977, -0.19824219,  0.00689697,  0.14355469,  0.0019455 ,\n",
              "        0.02880859, -0.25      , -0.08398438, -0.15136719, -0.10205078,\n",
              "        0.04077148, -0.09765625,  0.05932617,  0.02978516, -0.10058594,\n",
              "       -0.13085938,  0.001297  ,  0.02612305, -0.27148438,  0.06396484,\n",
              "       -0.19140625, -0.078125  ,  0.25976562,  0.375     , -0.04541016,\n",
              "        0.16210938,  0.13671875, -0.06396484, -0.02062988, -0.09667969,\n",
              "        0.25390625,  0.24804688, -0.12695312,  0.07177734,  0.3203125 ,\n",
              "        0.03149414, -0.03857422,  0.21191406, -0.00811768,  0.22265625,\n",
              "       -0.13476562, -0.07617188,  0.01049805, -0.05175781,  0.03808594,\n",
              "       -0.13378906,  0.125     ,  0.0559082 , -0.18261719,  0.08154297,\n",
              "       -0.08447266, -0.07763672, -0.04345703,  0.08105469, -0.01092529,\n",
              "        0.17480469,  0.30664062, -0.04321289, -0.01416016,  0.09082031,\n",
              "       -0.00927734, -0.03442383, -0.11523438,  0.12451172, -0.0246582 ,\n",
              "        0.08544922,  0.14355469, -0.27734375,  0.03662109, -0.11035156,\n",
              "        0.13085938, -0.01721191, -0.08056641, -0.00708008, -0.02954102,\n",
              "        0.30078125, -0.09033203,  0.03149414, -0.18652344, -0.11181641,\n",
              "        0.10253906, -0.25976562, -0.02209473,  0.16796875, -0.05322266,\n",
              "       -0.14550781, -0.01049805, -0.03039551, -0.03857422,  0.11523438,\n",
              "       -0.0062561 , -0.13964844,  0.08007812,  0.06103516, -0.15332031,\n",
              "       -0.11132812, -0.14160156,  0.19824219, -0.06933594,  0.29296875,\n",
              "       -0.16015625,  0.20898438,  0.00041771,  0.01831055, -0.20214844,\n",
              "        0.04760742,  0.05810547, -0.0123291 , -0.01989746, -0.00364685,\n",
              "       -0.0135498 , -0.08251953, -0.03149414,  0.00717163,  0.20117188,\n",
              "        0.08300781, -0.0480957 , -0.26367188, -0.09667969, -0.22558594,\n",
              "       -0.09667969,  0.06494141, -0.02502441,  0.08496094,  0.03198242,\n",
              "       -0.07568359, -0.25390625, -0.11669922, -0.01446533, -0.16015625,\n",
              "       -0.00701904, -0.05712891,  0.02807617, -0.09179688,  0.25195312,\n",
              "        0.24121094,  0.06640625,  0.12988281,  0.17089844, -0.13671875,\n",
              "        0.1875    , -0.10009766, -0.04199219, -0.12011719,  0.00524902,\n",
              "        0.15625   , -0.203125  , -0.07128906, -0.06103516,  0.01635742,\n",
              "        0.18261719,  0.03588867, -0.04248047,  0.16796875, -0.15039062,\n",
              "       -0.16992188,  0.01831055,  0.27734375, -0.01269531, -0.0390625 ,\n",
              "       -0.15429688,  0.18457031, -0.07910156,  0.09033203, -0.02709961,\n",
              "        0.08251953,  0.06738281, -0.16113281, -0.19628906, -0.15234375,\n",
              "       -0.04711914,  0.04760742,  0.05908203, -0.16894531, -0.14941406,\n",
              "        0.12988281,  0.04321289,  0.02624512, -0.1796875 , -0.19628906,\n",
              "        0.06445312,  0.08935547,  0.1640625 , -0.03808594, -0.09814453,\n",
              "       -0.01483154,  0.1875    ,  0.12792969,  0.22753906,  0.01818848,\n",
              "       -0.07958984, -0.11376953, -0.06933594, -0.15527344, -0.08105469,\n",
              "       -0.09277344, -0.11328125, -0.15136719, -0.08007812, -0.05126953,\n",
              "       -0.15332031,  0.11669922,  0.06835938,  0.0324707 , -0.33984375,\n",
              "       -0.08154297, -0.08349609,  0.04003906,  0.04907227, -0.24121094,\n",
              "       -0.13476562, -0.05932617,  0.12158203, -0.34179688,  0.16503906,\n",
              "        0.06176758, -0.18164062,  0.20117188, -0.07714844,  0.1640625 ,\n",
              "        0.00402832,  0.30273438, -0.10009766, -0.13671875, -0.05957031,\n",
              "        0.0625    , -0.21289062, -0.06542969,  0.1796875 , -0.07763672,\n",
              "       -0.01928711, -0.15039062, -0.00106049,  0.03417969,  0.03344727,\n",
              "        0.19335938,  0.01965332, -0.19921875, -0.10644531,  0.01525879,\n",
              "        0.00927734,  0.01416016, -0.02392578,  0.05883789,  0.02368164,\n",
              "        0.125     ,  0.04760742, -0.05566406,  0.11572266,  0.14746094,\n",
              "        0.1015625 , -0.07128906, -0.07714844, -0.12597656,  0.0291748 ,\n",
              "        0.09521484, -0.12402344, -0.109375  , -0.12890625,  0.16308594,\n",
              "        0.28320312, -0.03149414,  0.12304688, -0.23242188, -0.09375   ,\n",
              "       -0.12988281,  0.0135498 , -0.03881836, -0.08251953,  0.00897217,\n",
              "        0.16308594,  0.10546875, -0.13867188, -0.16503906, -0.03857422,\n",
              "        0.10839844, -0.10498047,  0.06396484,  0.38867188, -0.05981445,\n",
              "       -0.0612793 , -0.10449219, -0.16796875,  0.07177734,  0.13964844,\n",
              "        0.15527344, -0.03125   , -0.20214844, -0.12988281, -0.10058594,\n",
              "       -0.06396484, -0.08349609, -0.30273438, -0.08007812,  0.02099609],\n",
              "      dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zjBCVKGCqCe8",
        "colab_type": "code",
        "outputId": "72375142-7499-4c36-822a-0f645f680d8f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        }
      },
      "source": [
        "# let us try the famous relationship\n",
        "vec = model['king'] - model['man'] + model['women']\n",
        "model.most_similar([vec])"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
            "  if np.issubdtype(vec.dtype, np.int):\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('king', 0.6478992104530334),\n",
              " ('queen', 0.5354937314987183),\n",
              " ('women', 0.5233659148216248),\n",
              " ('kings', 0.5162314176559448),\n",
              " ('queens', 0.499536395072937),\n",
              " ('princes', 0.46233272552490234),\n",
              " ('monarch', 0.4528029263019562),\n",
              " ('monarchy', 0.4293174147605896),\n",
              " ('crown_prince', 0.42302513122558594),\n",
              " ('womens', 0.41756653785705566)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H-150DnQqChB",
        "colab_type": "code",
        "outputId": "74097fea-2b85-4647-9122-990158ef6bf5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        }
      },
      "source": [
        "# Relationship\n",
        "vec = model['Germany'] - model['Berlin'] + model['Paris']\n",
        "model.most_similar([vec])"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
            "  if np.issubdtype(vec.dtype, np.int):\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('France', 0.7724405527114868),\n",
              " ('Paris', 0.6798242926597595),\n",
              " ('Belgium', 0.598486065864563),\n",
              " ('Germany', 0.5652832984924316),\n",
              " ('Spain', 0.550815761089325),\n",
              " ('Italy', 0.5462925434112549),\n",
              " ('Marseille', 0.5372347831726074),\n",
              " ('Switzerland', 0.5364958047866821),\n",
              " ('French', 0.5346114039421082),\n",
              " ('Morocco', 0.5051252841949463)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gQ-9rTIwqCjW",
        "colab_type": "code",
        "outputId": "43a1aae3-197a-4c53-89c4-8b96afdb54a0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        }
      },
      "source": [
        "# Relationship\n",
        "\n",
        "vec = model['Messi'] - model['Football'] + model['Cricket']\n",
        "model.most_similar([vec])"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
            "  if np.issubdtype(vec.dtype, np.int):\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('Messi', 0.7382575869560242),\n",
              " ('Sehwag', 0.6777455806732178),\n",
              " ('Tendulkar', 0.6748222708702087),\n",
              " ('Xavi', 0.6610832810401917),\n",
              " ('Dravid', 0.6569646596908569),\n",
              " ('Dhoni', 0.6550688743591309),\n",
              " ('Lionel_Messi', 0.6407608985900879),\n",
              " ('Forlan', 0.640610933303833),\n",
              " ('Yuvraj', 0.6390379071235657),\n",
              " ('Ponting', 0.6390198469161987)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    }
  ]
}