Add Django Web Management

Update Auto Configuration
Add auto start Django
Update output result
Update Gather Company Info

.gitignore

@@ -8,3 +8,4 @@ lib/.directory
 plugins/.directory
 proxy.txt
 belati.conf
+beta/*

Belati.py

@@ -32,6 +32,7 @@
 import time
 import dns.resolver
 import tldextract
+import shlex, subprocess
 
 from plugins.about_project import AboutProject
 from plugins.banner_grab import BannerGrab
@@ -43,6 +44,7 @@
 from plugins.git_finder import GitFinder
 from plugins.harvest_email import HarvestEmail
 from plugins.harvest_public_document import HarvestPublicDocument
+from plugins.json_beautifier import JsonBeautifier
 from plugins.logger import Logger
 from plugins.robots_scraper import RobotsScraper
 from plugins.scan_nmap import ScanNmap
@@ -94,11 +96,11 @@ def __init__(self):
 
         self.show_banner()
 
-        conf = Config()
+        self.conf = Config()
         self.db = Database()
 
         # Setup project
-        self.project_id = self.db.create_new_project(domain, orgcomp, datetime.datetime.now())
+        self.project_id = self.db.create_new_project(domain, orgcomp, datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
         log.console_log("{}[+] Creating New Belati Project... {}".format(G, W))
         log.console_log("---------------------------------------------------------")
         log.console_log("Project ID: {}".format(str(self.project_id)))
@@ -151,6 +153,7 @@ def __init__(self):
             log.console_log("This feature will be coming soon. Be patient :)")
 
         log.console_log("{}All done sir! All log saved in log directory and dowloaded file saved in belatiFiles {}".format(Y, W))
+        self.start_web_server()
 
     def show_banner(self):
         banner = """
@@ -196,7 +199,11 @@ def check_domain(self, domain_name, proxy_address):
         log.console_log("{}[*] Perfoming Whois... {}".format(G, W))
         whois_result = check.whois_domain(domain_name)
         email = re.findall(r'[a-zA-Z0-9._+-]+@[a-zA-Z0-9._+-]+\s*', str(whois_result))
-        self.db.insert_domain_result(self.project_id, self.strip_scheme(domain_name), str(whois_result), str(email) )
+
+        # JSON Beautifier
+        json_bf = JsonBeautifier()
+        json_whois = json_bf.beautifier(str(whois_result))
+        self.db.insert_domain_result(self.project_id, self.strip_scheme(domain_name), str(json_whois), str(email).strip("\'.\'[]") )
 
 
     def banner_grab(self, domain_name, proxy_address):
@@ -271,7 +278,7 @@ def scan_DNS_zone(self, domain_name):
                 log.console_log("{}{}{}".format(G, ns.to_text(), W))
                 mx_record_list.append(ns.to_text())
 
-            self.db.update_dns_zone(self.project_id, domain_name, str(ns_record_list), str(mx_record_list))
+            self.db.update_dns_zone(self.project_id, domain_name, str(ns_record_list).strip("\'.\'[]"), str(mx_record_list).strip("\'[]"))
 
         except Exception, exc:
             print("{}[*] No response from server... SKIP!{}".format(R, W))
@@ -286,9 +293,10 @@ def harvest_email_search(self, domain_name, proxy_address):
         except Exception, exc:
             log.console_log("{}[-] Not found or Unavailable. {}{}".format(R, str(harvest_result), W ))
 
-        self.db.insert_email_result(self.project_id, str(harvest_result))
+        self.db.insert_email_result(self.project_id, str(harvest_result).strip("\'.\'[]"))
 
     def harvest_email_pgp(self, domain_name, proxy_address):
+        log.console_log("{}[*] Perfoming Email Harvest from PGP Server...{}".format(G, W) )
         harvest = HarvestEmail()
         harvest_result = harvest.crawl_pgp_mit_edu(domain_name, proxy_address)
         try:
@@ -297,7 +305,7 @@ def harvest_email_pgp(self, domain_name, proxy_address):
         except Exception, exc:
             log.console_log("{}[-] Not found or Unavailable. {}{}".format(R, str(harvest_result), W ))
 
-        self.db.update_pgp_email(self.project_id, str(harvest_result))
+        self.db.update_pgp_email(self.project_id, str(harvest_result).strip("\'.\'[]"))
 
     def harvest_document(self, domain_name, proxy_address):
         log.console_log("{}[*] Perfoming Public Document Harvest from Google... {}".format(G, W))
@@ -371,6 +379,20 @@ def gather_company(self, company_name, proxy_address):
         gather_company = GatherCompany()
         gather_company.crawl_company_employee(company_name, proxy_address, self.project_id)
 
+    def start_web_server(self):
+        log.console_log("{}Starting Django Web Server at http://127.0.0.1:8000/{}".format(Y, W))
+        py_bin = self.conf.get_config("Environment", "py_bin")
+        command = "{} web/manage.py runserver".format(py_bin)
+        process = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE)
+        while True:
+            output = process.stdout.readline()
+            if output == '' and process.poll() is not None:
+                break
+            if output:
+                log.console_log(output.strip())
+        rc = process.poll()
+        return rc
+
     def check_update(self, version):
         log.console_log("{} Checking Version Update for Belati... {}".format(G, W))
         # TODO

CHANGELOG.md

@@ -20,3 +20,11 @@ Update Creating Project Info
 Update Sublist3r
 Update Gather Company Plugin
 Update README
+
+v0.2.2-dev:
+
+Add Django Web Management
+Update Auto Configuration
+Add auto start Django
+Update output result
+Update Gather Company Info

README.md

@@ -5,7 +5,11 @@ Belati is tool for Collecting Public Data & Public Document from Website and oth
 
 ## Version
 
-v.0.2.1-dev
+v.0.2.2-dev
+
+## Belati In Action
+
+[Belati In Action Preview](https://www.youtube.com/watch?v=AGvsIWoaX_k)
 
 ## Why I Made this?
 Just for learning stuff and OSINT purpose. Correct me if i'm wrong
@@ -28,6 +32,7 @@ Just for learning stuff and OSINT purpose. Correct me if i'm wrong
 - Gather Public Company Info & Employee
 - SQLite3 Database Support for storing Belati Results
 - Setup Wizard/Configuration for Belati
+- Django Web Management
 
 ## TODO
 - Automatic OSINT with Username and Email support
@@ -38,7 +43,6 @@ Just for learning stuff and OSINT purpose. Correct me if i'm wrong
 - Email Harvesting with multiple content(github, linkedin, etc)
 - Scrapping Public Document with multiple search engine(yahoo, yandex, bing etc)
 - Metadata Extractor
-- Web version with Django
 - Scanning Report export to PDF
 - domain or subdomain reputation checker
 - Reporting Support to JSON, PDF
@@ -88,6 +92,35 @@ Don't worry. Belati still running when you are run with normal user ;)
 - tldextract
 - fake-useragent
 
+## Missing Dependencies?
+
+If you are seeing this
+
+```
+$ python Belati.py
+
+            You are missing a module required for Belati. In order to continue using Belati, please install them with:
+
+            `pip install -r requirements.txt`
+
+            or manually install missing modules with:
+
+            `pip install dnspython requests termcolor colorama future beautifulsoup4 futures`
+```
+
+and this
+
+```
+You are using pip version 8.1.2, however version 9.0.1 is available.
+You should consider upgrading via the 'pip install --upgrade pip' command.
+```        
+
+Please upgrade pip version:
+
+```
+pip install --upgrade pip
+```
+
 ## System Dependencies
 
 For CentOS/Fedora user, please install this:

plugins/about_project.py

@@ -24,7 +24,7 @@ class AboutProject(object):
     def __init__(self):
         self.__info__ = 'Collecting Public Data & Public Document for OSINT purpose'
         self.__author__ = 'Petruknisme'
-        self.__version__ = 'v0.2.1-dev'
+        self.__version__ = 'v0.2.2-dev'
         self.__name__= "Belati"
         self.__giturl__ = "https://github.com/aancw/Belati"
         self.__authorurl__ = "https://petruknisme.com"

plugins/config.py

@@ -38,7 +38,10 @@ class Config(object):
     def __init__(self):
         self.config_file = "belati.conf"
         if os.path.isfile(self.config_file):
-            pass
+            db_string = self.get_config("Database", "db_location")
+            if db_string == "belati.db":
+                log.console_log("{}[-] Hm... You are using old Belati configuration{}".format(Y, W))
+                self.init_config_file()
         else:
             log.console_log("{}[-] No Configuration file found. Setting up...{}".format(Y, W))
             self.init_config_file()
@@ -56,7 +59,19 @@ def set_config(self, conf_section, conf_key, conf_value):
 
     def init_config_file(self):
         log.console_log("\n{} -----> Initiating Configuration <-----\n{}".format(Y, W))
-        db_location = raw_input("Please enter Belati Database Location [belati.db]:") or "belati.db"
 
-        config.add_section("Database")
-        self.set_config("Database", "db_location", db_location)
+        if config.has_section("Database"):
+            pass
+        else:
+            config.add_section("Database")
+
+        self.set_config("Database", "db_location", "web/db.sqlite3")
+        log.console_log("\n{} Setting database location to {}\n{}".format(Y,"web/db.sqlite3", W))
+        python_binary = raw_input("Please enter Python v2.x Binary name [python]:") or "python"
+
+        if config.has_section("Environment"):
+            pass
+        else:
+            config.add_section("Environment")
+
+        self.set_config("Environment", "py_bin", python_binary)

plugins/database.py

@@ -41,6 +41,7 @@ def __init__(self):
         db_location = conf.get_config("Database", "db_location")
         try:
             self.conn = db.connect(db_location)
+            self.conn.text_factory = str
         except db.Error, e:
             print("Error: " +  str(e.args[0]))
             sys.exit()
@@ -152,9 +153,9 @@ def insert_linkedin_company_info(self, project_id, company_name, company_linkedi
 
         return cur.lastrowid
 
-    def insert_company_employees(self, company_id, name, job_title, linkedin_url ):
+    def insert_company_employees(self, project_id, name, job_title, linkedin_url ):
         cur = self.conn.cursor()
-        cur.execute("INSERT INTO linkedin_company_employees(`company_id`, `name`, `job_title`, `linkedin_url`) VALUES(?, ?, ?, ?)", (company_id, name, job_title, linkedin_url))
+        cur.execute("INSERT INTO linkedin_company_employees(`project_id`, `name`, `job_title`, `linkedin_url`) VALUES(?, ?, ?, ?)", (project_id, name, job_title, linkedin_url))
         self.conn.commit()
 
     # def read(table, **kwargs):

plugins/gather_company.py

@@ -61,9 +61,17 @@ def crawl_company_employee(self, company_name, proxy_address, project_id):
             url_tag = soup2.find_all("a")[0].string
 
             # Check if URL is match with one of the string from company name(?)
-            if company_linkedin_url and any(x in company_name for x in url_tag):
-                company_linkedin_url_list.append(company_linkedin_url)
-                self.company_id = self.db.insert_linkedin_company_info(self.project_id, str(company_name), str(company_linkedin_url), "Lorem ipsum")
+            if company_linkedin_url:
+                is_contain_name = 0
+                for x in company_name.split():
+                    if x in url_tag:
+                        print(" X" + str(x) + "url tag " + str(url_tag))
+                        is_contain_name = 1
+                        break
+
+                if is_contain_name == 1:
+                    company_linkedin_url_list.append(company_linkedin_url)
+                    self.company_id = self.db.insert_linkedin_company_info(self.project_id, str(company_name), str(company_linkedin_url), "Lorem ipsum")
 
             # Get data when linkedin url is like this : *.linkedin.com/in
             if not linkedin_url:
@@ -80,7 +88,7 @@ def crawl_company_employee(self, company_name, proxy_address, project_id):
                 log.console_log("Url: {}".format( linkedin_url_fix ))
                 log.console_log("{}[+] --------------------------------------------------- [+]{}\n".format(Y, W))
 
-                self.db.insert_company_employees(self.company_id, name_fix, job_title_fix, linkedin_url_fix)
+                self.db.insert_company_employees(self.project_id, name_fix, job_title_fix, linkedin_url_fix)
 
         log.console_log("\n\n{}[+] --------------------------------------------------- [+]{}".format(Y, W))
         log.console_log("{}[+] Found LinkedIn Company URL: {}".format(Y, W))

plugins/harvest_email.py

@@ -37,7 +37,7 @@ def crawl_search(self, domain, proxy_address):
             dataStrip1 =  re.findall(r'[a-zA-Z0-9._+-]+@[a-zA-Z0-9._+-]+' + domain, dataStrip)
             dataStrip2 = re.findall(r'[a-zA-Z0-9._+-]+@' + domain, dataStrip)
             dataEmail = set(dataStrip1 + dataStrip2)
-            dataFix = [x for x in dataEmail if not x.startswith('x22') and not x.startswith('3D') and not x.startswith('x3d')] # ignore email because bad parsing
+            dataFix = [x for x in dataEmail if not x.startswith('x22') and not x.startswith('3D') and not x.startswith('x3d') and not x.startswith('Cached') and not x.startswith('page')] # ignore email because bad parsing
             return list(dataFix)
         except:
             pass

plugins/harvest_public_document.py

@@ -67,8 +67,8 @@ def harvest_public_doc(self, domain, extension, proxy_address):
         data = re.findall(regex, data)
         list_files_download = list(set(data))
         total_files = str(len(list_files_download))
-        log.console_log("{}[*] Found {} {} files!".format(G, total_files, extension.upper(), W) )
         if total_files != "0":
+            log.console_log("{}[*] Found {} {} files!".format(G, total_files, extension.upper(), W) )
             log.console_log("{}[*] Please wait, lemme download it for you ;) {}[NO PROXY] {}".format(G, Y, W))
             for files_download in list_files_download:
                 log.no_console_log(files_download.split('/')[-1])

plugins/json_beautifier.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+#   Belati is tool for Collecting Public Data & Public Document from Website and other service for OSINT purpose.
+#   This tools is inspired by Foca and Datasploit for OSINT
+#   Copyright (C) 2017  cacaddv@gmail.com (Petruknisme a.k.a Aan Wahyu)
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 2 of the License, or
+#    (at your option) any later version.
+
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# This file is part of Belati project
+
+import json
+
+class JsonBeautifier(object):
+	def beautifier(self, json_data):
+		parsed = json.loads(json_data)
+		return json.dumps(parsed, indent=4, sort_keys=True)

requirements.txt

@@ -16,3 +16,4 @@ future
 beautifulsoup4
 python-whois
 futures
+django

version

@@ -1 +1 @@
-v0.2.0-dev
+v0.2.2-dev

web/manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "web.settings")
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError:
+        # The above import may fail for some other reason. Ensure that the
+        # issue is really that Django is missing to avoid masking other
+        # exceptions on Python 2.
+        try:
+            import django
+        except ImportError:
+            raise ImportError(
+                "Couldn't import Django. Are you sure it's installed and "
+                "available on your PYTHONPATH environment variable? Did you "
+                "forget to activate a virtual environment?"
+            )
+        raise
+    execute_from_command_line(sys.argv)