URL Categorization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import html2text\n",
    "from flashtext.keyword import KeywordProcessor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "url='https://powerseductionandwar.com/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    page = requests.get(url)        #to extract page from website\n",
    "    html_code = page.content        #to extract html code from page\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    soup = BeautifulSoup(html_code, 'html.parser')  #Parse html code\n",
    "    texts = soup.findAll(text=True)                 #find all text\n",
    "    text_from_html = ' '.join(texts)                   #join all text\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_text(html, rehtml=False):\n",
    "    parser = html2text.HTML2Text()\n",
    "    parser.wrap_links = False\n",
    "    parser.skip_internal_links = True\n",
    "    parser.inline_links = True\n",
    "    parser.ignore_anchors = True\n",
    "    parser.ignore_images = True\n",
    "    parser.ignore_emphasis = True\n",
    "    parser.ignore_links = True\n",
    "    text = parser.handle(html)\n",
    "    text = text.strip(' \\t\\n\\r')\n",
    "    if rehtml:\n",
    "        text = text.replace('\\n', '<br/>')\n",
    "        text = text.replace('\\\\', '')\n",
    "    return text "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_text = to_text(html_code.decode()).replace('\\r', ' ').replace('\\n',' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"|    search    * Home   * About   * Books   * Blog   * Contact  Filter your search by Category : Tag : Date  Category Uncategorized Blog Laws of Human Nature  Tags    * Home   * About   * Books   * Blog   * Contact  #  # Robert Greene  Robert Greene is the author of the New York Times bestsellers The 48 Laws of Power, The Art of Seduction, The 33 Strategies of War, The 50th Law, and Mastery.  In his highly anticipated sixth book, The Laws of Human Nature, he turns to the most important subject of all - understanding people's drives and motivations, even when they are unconscious of them themselves.  #### Learn More  ### Learn The Secret Strategies That Help Robert Greene Write & Think  Learn More  ## Coming Oct 23rd  We are social animals. Our very lives depend on our relationships with people. Knowing why people do what they do is the most important tool we can possess, without which our other talents can only take us so far. Drawing from the ideas and examples of Pericles, Queen Elizabeth I, Martin Luther King Jr, and many others, Greene teaches us how to detach ourselves from our own emotions and master self-control, how to develop the empathy that leads to insight, how to look behind people's masks, and how to resist conformity to develop your singular sense of purpose. Whether at work, in relationships, or in shaping the world around you, The Laws of Human Nature offers brilliant tactics for success, self-improvement, and self-defense.  #### Learn More  ## More Books by Robert Greene  Learn More  Learn More  Learn More  Learn More  Learn More    * Home   * About   * Books   * Blog   * Contact     VIEW POST  View more  Laws of Human Nature  ### Introducing: The Laws of Human Nature (And Exclusive Pre-Order Bonuses)  By  Robert Greene  on  October 3, 2018  I’m very excited to announce that my newest book The Laws of Human Nature is available for pre-order.   Amazon   Barnes & Noble   IndieBound   Books-A-Million   Apple Books   PRH.com   UK  Waterstones   Amazon UK   The Laws of…  READ ARTICLE     VIEW POST  View more  Uncategorized  ### “How Can I Be More Productive?”  By  Robert Greene  on  September 4, 2014  After being asked this question countless times over the years, I finally compiled all my recommendations in one document The Three Secrets That Help…  READ ARTICLE     VIEW POST  View more  Uncategorized  ### Mastery paperback is out now + FREE book (US only)  By  Robert Greene  on  December 5, 2013  The paperback of the #1 New York Times Bestseller Mastery is in stores now and available on Amazon.   And as a thank you to my…  READ ARTICLE  1 2 3 … 25 »  (C) 2019 Power, Seduction and War\""
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification - Fuddu Method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "Class_1_keywords = ['Psychology', 'Business', 'Human Nature', 'Addiction', 'Electronics', 'Cell', 'Business', 'Education', 'Classroom']\n",
    "Class_2_keywords = ['Restaurant', 'Hospitality', 'Tub', 'Drain', 'Pool', 'Filtration', 'Floor', 'Restroom', 'Consumer', 'Care', 'Bags', 'Disposables']\n",
    "Class_3_keywords = ['Pull', 'Lifts', 'Pneumatic', 'Emergency', 'Finishing', 'Hydraulic', 'Lockout', 'Towers', 'Drywall', 'Tools', 'Packaging', 'Measure', 'Tag ']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "keywords=Class_1_keywords + Class_2_keywords + Class_3_keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "kp0=KeywordProcessor()\n",
    "for word in keywords:\n",
    "    kp0.add_keyword(word)\n",
    "    \n",
    "kp1=KeywordProcessor()\n",
    "for word in Class_1_keywords:\n",
    "    kp1.add_keyword(word)\n",
    "    \n",
    "kp2=KeywordProcessor()\n",
    "for word in Class_2_keywords:\n",
    "    kp2.add_keyword(word)\n",
    "    \n",
    "kp3=KeywordProcessor()\n",
    "for word in Class_3_keywords:\n",
    "    kp3.add_keyword(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def percentage_match(dum0,dumx):\n",
    "    try:\n",
    "        ans=float(dumx)/float(dum0)\n",
    "        ans=ans*100\n",
    "    except:\n",
    "        return 0\n",
    "    else:\n",
    "        return ans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_class():\n",
    "    x=str(text_from_html)\n",
    "    y0 = len(kp0.extract_keywords(x))\n",
    "    y1 = len(kp1.extract_keywords(x))\n",
    "    y2 = len(kp2.extract_keywords(x))\n",
    "    y3 = len(kp3.extract_keywords(x))\n",
    "    Total_matches=y0   \n",
    "    per1 = float(percentage1(y0,y1))\n",
    "    per2 = float(percentage1(y0,y2))\n",
    "    per3 = float(percentage1(y0,y3))\n",
    "    if y0==0:\n",
    "        Category='None'\n",
    "    else:\n",
    "        if per1>=per2 and per1>=per3:\n",
    "            Category='Class_1'\n",
    "        elif per2>=per3 and per2>=per1:\n",
    "            Category='Class_2'\n",
    "        elif per3>=per1 and per3>=per2:\n",
    "            Category='Class_3'\n",
    "    return Category"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NLTK Comes HERE!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}