From 0c3ac1038aea666bbae15cac40d947a160eca89a Mon Sep 17 00:00:00 2001 From: Jonathan Claudius Date: Thu, 21 Mar 2019 10:38:03 -0400 Subject: [PATCH 1/3] Format NMAP JSON for humans --- lib/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/task.py b/lib/task.py index cd81dd6..98f88ae 100644 --- a/lib/task.py +++ b/lib/task.py @@ -192,7 +192,7 @@ class NmapTask(Task): if results: try: nmap_output = open("/app/results/" + self.tasktarget.targetdomain + "/" + "nmap_tcp.json", "w+") - nmap_output.write(json.dumps(results)) + nmap_output.write(json.dumps(results, indent=4, sort_keys=True)) return True except Exception: logger.error("[-] Could not open file for nmap output!") From d688dfdba5b1f6fa69b56dc225bacc998bf49df1 Mon Sep 17 00:00:00 2001 From: Guillaume Destuynder Date: Thu, 21 Mar 2019 13:28:21 -0700 Subject: [PATCH 2/3] allow to add custom args (tox -- myarg) when running tests --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9605af4..4cf1c6a 100644 --- a/tox.ini +++ b/tox.ini @@ -8,4 +8,4 @@ skipsdist=True deps= .[test] ./ -commands=pytest tests/ --cov=lib +commands=pytest tests/ --cov=lib {posargs} From 9c8c7e55ba95eed31acf1abe05a0fc94a424a702 Mon Sep 17 00:00:00 2001 From: Guillaume Destuynder Date: Thu, 21 Mar 2019 13:44:18 -0700 Subject: [PATCH 3/3] Add support for WebSearchTask --- lib/target.py | 8 +++++++- lib/task.py | 31 ++++++++++++++++++++++++++++++- requirements.txt | 1 + run.py | 32 +++++++++++++++++++------------- setup.py | 2 +- 5 files changed, 58 insertions(+), 16 deletions(-) diff --git a/lib/target.py b/lib/target.py index f07518e..4a058a2 100644 --- a/lib/target.py +++ b/lib/target.py @@ -133,6 +133,12 @@ class Target: logger.info("[+] HTTP Observatory scan successfully ran.") self.resultsdict.update({"httpobs": True}) + elif isinstance(one_task, task.WebSearchTask): + websearch_results = one_task.runWebSearchScan() + if websearch_results: + logger.info("[+] WebSearch scan successfully ran.") + self.resultsdict.update({"websearch": True}) + elif isinstance(one_task, task.SSHScanTask): sshscan_results = one_task.runSSHScan() if sshscan_results and sshscan_results.returncode == 0: @@ -146,7 +152,7 @@ class Target: self.resultsdict.update({"dirbrute": True}) else: - logger.error("[-] No or unidentified task specified!") + logger.error("[-] No or unidentified task specified! Task was: {}".format(one_task)) return False # Need to check if the current Nessus scan is complete diff --git a/lib/task.py b/lib/task.py index cd81dd6..03f1ae2 100644 --- a/lib/task.py +++ b/lib/task.py @@ -1,6 +1,7 @@ import os import logging import coloredlogs +import googlesearch import json import nmap import subprocess @@ -45,6 +46,34 @@ class Task: time.sleep(interval) +class WebSearchTask(Task): + def __init__(self, target_obj): + super().__init__(target_obj) + + def runWebSearchScan(self): + # Limit max amount of results + result_nr_max = 15 + # Search for security hits but without the host domain (ie "not their pages") + search_results = [] + logger.info("[+] Running WebSearch scan...") + for m in googlesearch.search(query="{} security -site:{}".format(self.tasktarget.targetdomain), num=15): + search_results.append(m) + if len(search_results) >= result_nr_max: + break + + if len(search_results) > 0: + try: + with open("/app/results/" + self.tasktarget.targetdomain + "/" + "websearch.txt", "w+") as fd: + for i in search_results: + fd.write(i + "\n") + return True + except Exception: + logger.error("[-] Could not open file for websearch output!") + return False + else: + logger.error("[-] No results from websearch!") + + class NmapTask(Task): def __init__(self, target_obj, scan_type="full"): super().__init__(target_obj) @@ -192,7 +221,7 @@ class NmapTask(Task): if results: try: nmap_output = open("/app/results/" + self.tasktarget.targetdomain + "/" + "nmap_tcp.json", "w+") - nmap_output.write(json.dumps(results)) + nmap_output.write(json.dumps(results, indent=4, sort_keys=True)) return True except Exception: logger.error("[-] Could not open file for nmap output!") diff --git a/requirements.txt b/requirements.txt index 3e5c8f4..61ae375 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ python_nmap==0.6.1 netaddr==0.7.19 coloredlogs==10.0 pytest==4.1.1 +google==2.0.2 diff --git a/run.py b/run.py index efdb3d9..b083615 100644 --- a/run.py +++ b/run.py @@ -10,9 +10,13 @@ from lib import target, task, utils # Logging in UTC logger = logging.getLogger(__name__) -coloredlogs.install(level='INFO', logger=logger, reconfigure=True, - fmt='[%(hostname)s] %(asctime)s %(levelname)-8s %(message)s', - datefmt="%Y-%m-%d %I:%M:%S %p %Z") +coloredlogs.install( + level="INFO", + logger=logger, + reconfigure=True, + fmt="[%(hostname)s] %(asctime)s %(levelname)-8s %(message)s", + datefmt="%Y-%m-%d %I:%M:%S %p %Z", +) def setupVA(va_target): @@ -23,12 +27,13 @@ def setupVA(va_target): # Also kicking of Nessus scan as the first task as it takes time va_target.addTask(task.NessusTask(va_target)) va_target.addTask(task.NmapTask(va_target)) - + if "URL" in va_target.getType(): # We have a URL, means HTTP Obs, TLS Obs, # and directory brute scans are a go if va_target.getType() == "FQDN|URL": # We can run all tools/tasks + va_target.addTask(task.WebSearchTask(va_target)) va_target.addTask(task.MozillaHTTPObservatoryTask(va_target)) va_target.addTask(task.MozillaTLSObservatoryTask(va_target)) va_target.addTask(task.DirectoryBruteTask(va_target, tool="dirb")) @@ -36,25 +41,26 @@ def setupVA(va_target): va_target.addTask(task.MozillaTLSObservatoryTask(va_target)) va_target.addTask(task.DirectoryBruteTask(va_target, tool="dirb")) # HTTP Observatory does not like IPs as a target, skipping - va_target.resultsdict.update({'httpobs': "PASS"}) + va_target.resultsdict.update({"httpobs": "PASS"}) + va_target.resultsdict.update({"websearch": "PASS"}) elif va_target.getType() == "IPv4": va_target.addTask(task.MozillaTLSObservatoryTask(va_target)) va_target.addTask(task.DirectoryBruteTask(va_target, tool="dirb")) # Again, HTTP Observatory does not like IPs as a target, skipping - va_target.resultsdict.update({'httpobs': "PASS"}) + va_target.resultsdict.update({"httpobs": "PASS"}) else: # FQDN, we can run all tools/tasks + va_target.addTask(task.WebSearchTask(va_target)) va_target.addTask(task.MozillaHTTPObservatoryTask(va_target)) va_target.addTask(task.MozillaTLSObservatoryTask(va_target)) va_target.addTask(task.DirectoryBruteTask(va_target, tool="dirb")) - + return va_target def showScanSummary(result_dictionary): - coloredlogs.install(level='INFO', logger=logger, reconfigure=True, - fmt='%(levelname)-10s %(message)s') + coloredlogs.install(level="INFO", logger=logger, reconfigure=True, fmt="%(levelname)-10s %(message)s") print("\n====== SCAN SUMMARY ======") for one_task, status in result_dictionary.items(): @@ -65,7 +71,7 @@ def showScanSummary(result_dictionary): logger.info("[+] [\o/] " + one_task + " scan completed successfully!") else: logger.error("[-] [ :( ] " + one_task + " scan failed to run. Please investigate or run manually.") - + print("====== END OF SCAN =======\n") @@ -85,8 +91,8 @@ def runVA(scan_with_tasks, outpath): def main(): - - results = {'nmap': False, 'nessus': False, 'tlsobs': False, 'httpobs': False, 'sshscan': False, 'dirbrute': False} + + results = {"nmap": False, "nessus": False, "tlsobs": False, "httpobs": False, "sshscan": False, "dirbrute": False} destination = sys.argv[1] output_path = "/app/results/" + destination + "/" va_target = target.Target(destination, results) @@ -105,7 +111,7 @@ def main(): os.stat(output_path) except Exception: os.mkdir(output_path) - + va_scan = setupVA(va_target) runVA(va_scan, output_path) diff --git a/setup.py b/setup.py index 4e9a89d..56a893c 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ from setuptools import setup, find_packages with open("Readme.md", "r") as fh: long_description = fh.read() -requirements = ["coloredlogs", "netaddr", "nmap", "tenable_io"] +requirements = ["google", "coloredlogs", "netaddr", "nmap", "tenable_io"] test_requirements = ["pytest", "pytest-watch", "pytest-cov", "flake8"] setup_requirements = ["pytest-runner", "setuptools>=40.5.0"]