Belati.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#   Belati is tool for Collecting Public Data & Public Document from Website and other service for OSINT purpose.
#   This tools is inspired by Foca and Datasploit for OSINT
#   Copyright (C) 2017  cacaddv@gmail.com (Petruknisme a.k.a Aan Wahyu)

#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 2 of the License, or
#    (at your option) any later version.

#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.

#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.


# We need to check Dependency first
from plugins.dep_check import DepCheck

dep_check = DepCheck()
dep_check.check_dependency()

import argparse
import datetime
import urllib2
import sys, signal, socket, re
import time
import dns.resolver
import tldextract
import shlex, subprocess

from plugins.about_project import AboutProject
from plugins.banner_grab import BannerGrab
from plugins.check_domain import CheckDomain
from plugins.config import Config
from plugins.common_service_check import CommonServiceCheck
from plugins.database import Database
from plugins.gather_company import GatherCompany
from plugins.git_finder import GitFinder
from plugins.harvest_email import HarvestEmail
from plugins.harvest_public_document import HarvestPublicDocument
from plugins.json_beautifier import JsonBeautifier
from plugins.logger import Logger
from plugins.meta_exif_extractor import MetaExifExtractor
from plugins.robots_scraper import RobotsScraper
from plugins.scan_nmap import ScanNmap
from plugins.subdomain_enum import SubdomainEnum
from plugins.svn_finder import SVNFinder
from plugins.updater import Updater
from plugins.url_request import URLRequest
from plugins.util import Util
from plugins.wappalyzer import Wappalyzer

from lib.CheckMyUsername.check_my_username import CheckMyUsername
from dnsknife.scanner import Scanner
from urlparse import urlparse

from cmd2 import Cmd
from tabulate import tabulate
from texttable import Texttable

# Console color
G = '\033[92m'  # green
Y = '\033[93m'  # yellow
B = '\033[94m'  # blue
R = '\033[91m'  # red
W = '\033[0m'   # white
UNDERLINE = '\033[4m'
ENDC = '\033[0m'

log = Logger()
util = Util()

class Belati(Cmd):
    def __init__(self):
        self.about = AboutProject()
        self.url_req = URLRequest()

        Cmd.doc_header = "Core Commands"
        Cmd.prompt = "{}belati{} > ".format(UNDERLINE, ENDC)
        Cmd.path_complete

        Cmd.__init__(self)

        self.list_parameter = ['domain', 'username', 'email', 'orgcomp', 'proxy', 'proxy_file']
        self.parameters = {}
        self.multiple_proxy_list = []
        self.current_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')

        self.show_banner()
        self.conf = Config()
        self.db = Database()


    def show_banner(self):
        banner = """
        {}

         /$$$$$$$  /$$$$$$$$ /$$        /$$$$$$  /$$$$$$$$     .
        | $$__  $$| $$_____/| $$       /$$__  $$|__  $$__/    J:L
        | $$  \ $$| $$      | $$      | $$  \ $$   | $$       |:|
        | $$$$$$$ | $$$$$   | $$      | $$$$$$$$   | $$       |:|
        | $$__  $$| $$__/   | $$      | $$__  $$   | $$       |:|
        | $$  \ $$| $$      | $$      | $$  | $$   | $$       |:|
        | $$$$$$$/| $$$$$$$$| $$$$$$$$| $$  | $$   | $$   /]  |:|  [\ 
        |_______/ |________/|________/|__/  |__/   |__/   \:-'\"""'-:/
                                                            ""III""
                                                              III
                                                              III
                                                              III
                                                             (___)

                                The Traditional Swiss Army Knife for OSINT

        =[ {} {} by {}]=

        + -- --=[ {} ]=-- -- +
        + -- --=[ {} ]=-- -- +
        {}
        """

        warning_message = """
        {}
        This tool is for educational purposes only.
        Any damage you make will not affect the author.
        Do It With Your Own Risk!

        For Better Privacy, Please Use proxychains or other proxy service!
        {}
        """

        log.console_log(banner.format(G, self.about.__name__, self.about.__version__, self.about.__author__, self.about.__info__, self.about.__authorurl__, W))
        log.console_log(warning_message.format(R, W))

    def do_help(self, line):
    	'print help message'

    	print("\nCore commands")
    	print("==============\n")
    	print tabulate([["Name","Description"],
    		["?", "Help menu"],
    		["!", "Run OS Command"],
    		["history", "Show command history"],
    		["set", "Set parameters option value"],
    		["show", "Display list available parameter option"],
    		["start", "Start Automatic Scanning Belati"],
    		["startws", "Start Web Server Only Mode"],
    		["version", "Show application version number"],
    		["quit", "Exit the application"]],
			headers="firstrow")


    def do_set(self, arg, opts=None):
    	'''Set Variable for Belati parameters.\nUsage: set [option] [value]\n\nAvailable options:\ndomain, username, email, orgcomp, proxy, proxy_file'''

    	if not arg:
    		log.console_log('{} Set Variable for Belati parameters.\nUsage: set [option] [value]\n\nAvailable options:\ndomain, username, email, orgcomp, proxy, proxy_file {}'.format(W, W))
    	else:
            param = shlex.split(arg)
            key = param[0]
            value = param[1]
            if key in self.list_parameter:
                self.parameters[key] = value
                log.console_log('{} => {}'.format(self.parameters[key], value))
            else:
                log.console_log("Available parameters: domain, username, email, orgcomp, proxy, proxy_file")

    def do_show(self, arg, opts=None):
    	'Show available parameter options'
    	
    	domain_val = self.parameters['domain'] if 'domain' in self.parameters else None
    	orgcomp = self.parameters['orgcomp'] if 'orgcomp' in self.parameters else None
     	email = self.parameters['email'] if 'email' in self.parameters else None
     	username = self.parameters['username'] if 'username' in self.parameters else None
     	proxy = self.parameters['proxy'] if 'proxy' in self.parameters else None
     	proxy_file = self.parameters['proxy_file'] if 'proxy_file' in self.parameters else None
     	org_val = ""
    	arg = shlex.split(arg)

    	if not arg:
    		print("Please use command 'show options' to see list of option parameters")

    	elif arg[0] == "options":
            print tabulate([["Name","Value", "Required", "Description"],
    			["domain", domain_val, "Yes", "Domain name for OSINT"],
    			["orgcomp", orgcomp, "Yes", "Organization/Company name for OSINT"],
    			["email", email, "Optional", "Email address for OSINT"],
    			["username", username, "Optional", "Username for OSINT"],
    			["proxy", proxy, "Optional", "Proxy server(e.g http://127.0.0.1:8080)"],
    			["proxy_file", proxy_file, "Optional", "Proxy file list location"]],
    			headers="firstrow")

    def do_startws(self, line):
    	'Start Belati in Web Server Only Mode'
    	
    	log.console_log("{}[*] Entering Web Server Only Mode...{}".format(Y,W))
    	self.start_web_server()
    	sys.exit()

    def do_version(self, line):
    	'Check current Belati version'
    	
    	log.console_log('{} {} by {}\n'.format(self.about.__name__, self.about.__version__, self.about.__author__))
    	log.console_log('Project URL: {}'.format(self.about.__giturl__))

    def do_start(self, line):
    	'Start automatic scanning'
    	domain = self.parameters['domain'] if 'domain' in self.parameters else None
    	orgcomp = self.parameters['orgcomp'] if 'orgcomp' in self.parameters else None
     	email = self.parameters['email'] if 'email' in self.parameters else None
     	username = self.parameters['username'] if 'username' in self.parameters else None
     	proxy = self.parameters['proxy'] if 'proxy' in self.parameters else ''
     	proxy_file = self.parameters['proxy_file'] if 'proxy_file' in self.parameters else ''

    	if domain is None and orgcomp is None:
    		log.console_log("{}[-] Please specify domain/organization {}".format(R,W))
    		sys.exit()
			
		log.console_log("{}[*] Starting at: {} {}".format(Y, self.current_time , W))

        self.updater = Updater()
        self.updater.check_update(self.about.__version__)

        # Setup project
        self.project_id = self.db.create_new_project(domain, orgcomp, self.current_time)
        log.console_log("{}[+] Creating New Belati Project... {}".format(G, W))
        log.console_log("---------------------------------------------------------")
        log.console_log("Project ID: {}".format(str(self.project_id)))
        log.console_log("Project Domain: {}".format(domain))
        log.console_log("Project Organization/Company: {}".format(orgcomp))
        log.console_log("---------------------------------------------------------")

        if domain is not None:
            if proxy is not '':
                log.console_log("{}[*] Checking Proxy Status... {}".format(G, W))
                if self.check_single_proxy_status(proxy, "http://" + str(domain)) == 'ok':
                    pass
                else:
                    log.console_log('{}[-] Please use another proxy or disable proxy! {}'.format(R, W))
                    sys.exit()

            if proxy_file is not '':
                log.console_log("{}[*] Checking Proxy Status from file {}{}".format(G, proxy_file, W))
                self.check_multiple_proxy_status(proxy_file, "http://" + str(domain))
                proxy = self.multiple_proxy_list

            extract_domain = tldextract.extract(domain)

            self.check_domain(self.url_req.ssl_checker(domain), proxy)
            self.banner_grab(self.url_req.ssl_checker(domain), proxy)

            if extract_domain.subdomain == "":
                self.robots_scraper(self.url_req.ssl_checker(domain), proxy)
                self.enumerate_subdomains(domain, proxy)
                self.scan_DNS_zone(domain)
                self.harvest_email_search(domain, proxy)
                self.harvest_email_pgp(domain, proxy)
            else:
                domain = extract_domain.domain + '.' + extract_domain.suffix
                self.robots_scraper(self.url_req.ssl_checker(domain), proxy)
                self.enumerate_subdomains(domain, proxy)
                self.scan_DNS_zone(domain)
                self.harvest_email_search(domain, proxy)
                self.harvest_email_pgp(domain, proxy)

            self.harvest_document(domain, proxy)

        if username is not None:
            self.username_checker(username)

        if orgcomp is not None:
            self.gather_company(orgcomp, proxy)

        if email is not None:
            log.console_log("This feature will be coming soon. Be patient :)")

        log.console_log("{}All done sir! All logs saved in {}logs{} directory and dowloaded file saved in {}belatiFiles{} {}".format(Y, B, Y, B, Y, W))

        self.start_web_server()

    def check_domain(self, domain_name, proxy_address):
        check = CheckDomain()

        log.console_log(G + "{}[*] Checking Domain Availability... {}".format(G, W) , 0)
        log.console_log(check.domain_checker(domain_name, proxy_address))
        
        log.console_log("{}[*] Checking URL Alive... {}".format(G, W), 0)
        log.console_log(check.alive_check(domain_name, proxy_address))

        log.console_log("{}[*] Perfoming Whois... {}".format(G, W))
        whois_result = check.whois_domain(domain_name)
        log.console_log(whois_result)
        email = re.findall(r'[a-zA-Z0-9._+-]+@[a-zA-Z0-9._+-]+\s*', str(whois_result))

        # JSON Beautifier
        json_bf = JsonBeautifier()
        json_whois = json_bf.beautifier(str(whois_result))
        self.db.insert_domain_result(self.project_id, util.strip_scheme(domain_name), str(json_whois), util.clean_list_string(email))

    def banner_grab(self, domain_name, proxy_address):
        banner = BannerGrab()
        log.console_log("{}[*] Perfoming HTTP Banner Grabbing... {}".format(G, W))
        banner_info = banner.show_banner(domain_name, proxy_address)
        log.console_log(banner_info)
        self.db.insert_banner(domain_name, self.project_id, str(banner_info))

    def enumerate_subdomains(self, domain_name, proxy):
        log.console_log("{}[*] Perfoming Subdomains Enumeration...{}".format(G, W))
        sub_enum = SubdomainEnum()
        log.console_log("{}[+] Grabbing data from dnsdumpster...{}\n".format(B, W))
        dnsdumpster = sub_enum.scan_dnsdumpster(domain_name)
        subdomain_list = []
        data_table = [["Domain", "IP", "Provider", "Country"]]
        for entry in dnsdumpster['dns_records']['host']:
            data_table.extend([[entry['domain'], entry['ip'], entry['provider'], entry['country']]])
            subdomain_list.append(entry['domain'])

        log.console_log( tabulate(data_table, headers='firstrow') )

        log.console_log("{}[+] Grabbing data from crt.sh...{}\n".format(B, W))
        crt_list = sub_enum.scan_crtsh(domain_name, proxy)
        
        if crt_list is not None:
            log.console_log("\n".join(crt_list))
            subdomain_list = list(set(subdomain_list + crt_list))
        
        log.console_log("{}[+] Grabbing data from findsubdomains.com...{}\n".format(B, W))
        findsubdomains_list = sub_enum.scan_findsubdomainsCom(domain_name,proxy)
        
        if findsubdomains_list is not None:
            log.console_log("\n".join(findsubdomains_list))
            subdomain_list = list(set(subdomain_list + findsubdomains_list))

        subdomain_ip_list = []

        for subdomain in subdomain_list:
            self.banner_grab(self.url_req.ssl_checker(subdomain), proxy)
            self.robots_scraper(self.url_req.ssl_checker(subdomain), proxy)
            self.wappalyzing_webpage(subdomain)
            self.public_git_finder(subdomain, proxy)
            self.public_svn_finder(subdomain, proxy)
            try:
                subdomain_ip_list.append(socket.gethostbyname(subdomain))
                self.db.update_subdomain_ip(self.project_id, subdomain, str(socket.gethostbyname(subdomain)))
            except socket.gaierror:
                pass

        subdomain_ip_listFix = list(set(subdomain_ip_list))

        # check common service port TODO
        #for ipaddress in subdomain_ip_listFix:
            #self.common_service_check(ipaddress)

        for ipaddress in subdomain_ip_listFix:
            self.service_scanning(ipaddress)

    def wappalyzing_webpage(self, domain):
        log.console_log("{}[*] Wapplyzing on domain {}{}".format(G, domain, W))
        wappalyzing = Wappalyzer()
        targeturl = self.url_req.ssl_checker(domain)
        try:
            data = wappalyzing.run_wappalyze(targeturl)
            self.db.insert_wappalyzing(self.project_id, domain, data)
        except urllib2.URLError as exc:
            log.console_log('URL Error: {0}'.format(str(exc)))
        except urllib2.HTTPError as exc:
            log.console_log('HTTP Error: {0}'.format(str(exc)))
        except Exception as exc:
            log.console_log('Unknown error: {0}'.format(str(exc)))

    def service_scanning(self, ipaddress):
        scan_nm = ScanNmap()
        log.console_log("{}[*] Perfoming Nmap Full Scan on IP {}{}".format(G, ipaddress, W))
        log.console_log("{}[*] nmap -sS -A -Pn {}{}".format(G, ipaddress, W))
        scan_nm.run_scanning(ipaddress)

    def scan_DNS_zone(self, domain_name):
        log.console_log("{}[*] Perfoming DNS Zone Scanning... {}".format(G, W))
        log.console_log("{}[*] Please wait, maximum timeout for checking is 1 minutes {}".format(G, W))
        signal.signal(signal.SIGALRM, self.timeLimitHandler)
        signal.alarm(60)
        try:
            scan_list = str(list(Scanner(domain_name).scan()))
            ns_record_list = []
            mx_record_list = []
            log.console_log("{}{}{}".format(G, scan_list.replace(",","\n"), W))
            log.console_log("{}DNS Server:{}".format(G, W))
            for ns in dns.resolver.query(domain_name, 'NS'):
                log.console_log(G + ns.to_text() + W)
                ns_record_list.append(ns.to_text())

            log.console_log("{}MX Record:{}".format(G, W))
            for ns in dns.resolver.query(domain_name, 'MX'):
                log.console_log("{}{}{}".format(G, ns.to_text(), W))
                mx_record_list.append(ns.to_text())

            self.db.update_dns_zone(self.project_id, domain_name, util.clean_list_string(ns_record_list), util.clean_list_string(mx_record_list))

        except Exception, exc:
            print("{}[*] No response from server... SKIP!{}".format(R, W))

    def harvest_email_search(self, domain_name, proxy_address):
        log.console_log("{}[*] Perfoming Email Harvest from Google Search...{}".format(G, W) )
        harvest = HarvestEmail()
        harvest_result = harvest.crawl_search(domain_name, proxy_address)
        try:
            log.console_log("{}[*] Found {} emails on domain {}{}".format(Y, str(len(harvest_result)), domain_name, W))
            log.console_log("{}{}{}".format(R, '\n'.join(harvest_result), W))
            self.db.insert_email_result(self.project_id, util.clean_list_string(harvest_result))
        except Exception, exc:
            log.console_log("{}[-] Not found or Unavailable. {}{}".format(R, str(harvest_result), W ))

    def harvest_email_pgp(self, domain_name, proxy_address):
        log.console_log("{}[*] Perfoming Email Harvest from PGP Server...{}".format(G, W) )
        harvest = HarvestEmail()
        harvest_result = harvest.crawl_pgp_mit_edu(domain_name, proxy_address)
        try:
            log.console_log("{}[*] Found {} emails on domain {}{}".format(Y, str(len(harvest_result)), domain_name, W))
            log.console_log("{}{}{}".format(R, '\n'.join(harvest_result), W))
            self.db.update_pgp_email(self.project_id, util.clean_list_string(harvest_result))
        except Exception, exc:
            log.console_log("{}[-] Not found or Unavailable. {}{}".format(R, str(harvest_result), W ))

    def harvest_document(self, domain_name, proxy_address):
        log.console_log("{}[*] Perfoming Public Document Harvest from Google... {}".format(G, W))
        public_doc = HarvestPublicDocument()
        public_doc.init_crawl(domain_name, proxy_address, self.project_id)

    def username_checker(self, username):
        log.console_log("{}[*] Perfoming Username Availability Checker... {}".format(G, W))
        user_check = CheckMyUsername()
        username_status_result = user_check.check_username_availability(username)

        for result in username_status_result:
            log.console_log(G + "[+] " + result[0] + " => " + result[1] + ": " + result[2])

    def check_single_proxy_status(self, proxy_address, domain_check):
        try:
            parse = urlparse(proxy_address)
            proxy_scheme = parse.scheme
            proxy = str(parse.hostname) + ':' + str(parse.port)
            proxy_handler = urllib2.ProxyHandler({ proxy_scheme: proxy})
            opener = urllib2.build_opener(proxy_handler)
            opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36')]
            urllib2.install_opener(opener)
            req = urllib2.Request(domain_check)
            start_time = time.time()
            sock = urllib2.urlopen(req)
            end_time = time.time()
            diff_time = round(end_time - start_time, 3)
            log.console_log(Y + "{}[+] {} OK! Response Time : {}s".format(Y, proxy_address, str(diff_time), W ))
            return 'ok'
        except urllib2.HTTPError, e:
            print('Error code: ' + str(e.code))
            return e.code
        except Exception, detail:
            print('ERROR ' +  str(detail))
            return 1

    def check_multiple_proxy_status(self, file_location, domain_check):
        with open(file_location) as data:
            text = [line.rstrip('\n') for line in data]
            for proxy in text:
                if self.check_single_proxy_status(str(proxy), str(domain_check)) == 'ok':
                     self.multiple_proxy_list.append(proxy)

    def public_git_finder(self, domain, proxy_address):
        log.console_log("{}[*] Checking Public GIT Directory on domain {}{}".format(G, domain, W))
        git_finder = GitFinder()
        if git_finder.check_git(domain, proxy_address) == True:
            log.console_log("{}[+] Gotcha! You are in luck, boy![{}/.git/]{}".format(Y, domain, W))
            self.db.update_git_finder(self.project_id, domain, "Yes")

    def public_svn_finder(self, domain, proxy_address):
        log.console_log("{}[*] Checking Public SVN Directory on domain {}{}".format(G, domain, W))
        svn_finder = SVNFinder()
        if svn_finder.check_svn(domain, proxy_address) == 403:
            log.console_log("{}[+] Um... Forbidden :( {}".format(Y, W))
        if svn_finder.check_svn(domain, proxy_address) == 200:
            log.console_log("{}[+] Gotcha! You are in luck, boy![{}/.svn/]{}".format(Y, domain, W))
            self.db.update_svn_finder(self.project_id, domain, "Yes")

    def robots_scraper(self, domain, proxy_address):
        scraper = RobotsScraper()
        data = scraper.check_robots(domain, proxy_address)
        if data is not None and isinstance(data, int) == False and data.code == 200:
            log.console_log("{}[+] Found interesting robots.txt[ {} ] =>{}".format(Y, domain, W))
            log.console_log(data.read())
            self.db.insert_robots_txt(self.project_id, domain, str(data.read()))

    def gather_company(self, company_name, proxy_address):
        log.console_log("{}[+] Gathering Company Employee {} -> {}".format(G, W, company_name))
        gather_company = GatherCompany()
        gather_company.crawl_company_employee(company_name, proxy_address, self.project_id)

    def start_web_server(self):
        log.console_log("{}Starting Django Web Server at http://127.0.0.1:8000/{}".format(Y, W))
        py_bin = self.conf.get_config("Environment", "py_bin")
        command = "{} web/manage.py runserver 0.0.0.0:8000".format(py_bin)
        process = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE)
        while True:
            output = process.stdout.readline()
            if output == '' and process.poll() is not None:
                break
            if output:
                log.console_log(output.strip())
        rc = process.poll()
        return rc

    def complete_set(self, text, line, start_index, end_index):
        if text:
            return [
                param for param in self.list_parameter
                if param.startswith(text)
            ]
        else:
            return self.list_parameter

    def common_service_check(self, host):
        log.console_log("{}[*] Checking Common Service Check on host {}{}".format(G, host, W))
        service_check = CommonServiceCheck()
        service_check.check_available_service(host)

    def timeLimitHandler(self, signum, frame):
        print("No Response...")

if __name__ == '__main__':
    BelatiApp = Belati()
    BelatiApp.cmdloop()