diff --git a/README.md b/README.md index 0c2dbc0..72c0d3b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,10 @@ -## About Sublist3r +## About this Fork + +This fork differs from the original Sublist3r project in the following ways: +* Includes Bogka IP lookups for subdomains discovered +* Output to a file in Comma-delimited format that can be ingested by many other tools + +## About Sublist3r Sublist3r is a python tool designed to enumerate subdomains of websites using OSINT. It helps penetration testers and bug hunters collect and gather subdomains for the domain they are targeting. Sublist3r enumerates subdomains using many search engines such as Google, Yahoo, Bing, Baidu, and Ask. Sublist3r also enumerates subdomains using Netcraft, Virustotal, ThreatCrowd, DNSdumpster, and ReverseDNS. @@ -88,7 +94,7 @@ sudo apt-get install python-argparse - Install for Centos/Redhat: ``` sudo yum install python-argparse -``` +``` - Install using pip: ``` @@ -111,7 +117,8 @@ Short Form | Long Form | Description -t | --threads | Number of threads to use for subbrute bruteforce -e | --engines | Specify a comma-separated list of search engines -o | --output | Save the results to text file --h | --help | show the help message and exit +-h | --help | Show the help message and exit +-i | --findip | Find IP address of each subdomain ### Examples @@ -123,10 +130,14 @@ Short Form | Long Form | Description ``python sublist3r.py -d example.com`` -* To enumerate subdomains of specific domain and show only subdomains which have open ports 80 and 443 : +* To enumerate subdomains of specific domain and check for ports 80 and 443: ``python sublist3r.py -d example.com -p 80,443`` +* To enumerate subdomains and enable IP address resolution: + +``python sublist3r.py -v -i -d example.com`` + * To enumerate subdomains of specific domain and show the results in realtime: ``python sublist3r.py -v -d example.com`` @@ -145,7 +156,7 @@ Short Form | Long Form | Description **Example** ```python -import sublist3r +import sublist3r subdomains = sublist3r.main(domain, no_threads, savefile, ports, silent, verbose, enable_bruteforce, engines) ``` The main function will return a set of unique subdomains found by Sublist3r @@ -161,7 +172,7 @@ The main function will return a set of unique subdomains found by Sublist3r Example to enumerate subdomains of Yahoo.com: ```python -import sublist3r +import sublist3r subdomains = sublist3r.main('yahoo.com', 40, 'yahoo_subdomains.txt', ports= None, silent=False, verbose= False, enable_bruteforce= False, engines=None) ``` @@ -172,8 +183,8 @@ Sublist3r is licensed under the GNU GPL license. take a look at the [LICENSE](ht ## Credits -* [TheRook](https://github.com/TheRook) - The bruteforce module was based on his script **subbrute**. -* [Bitquark](https://github.com/bitquark) - The Subbrute's wordlist was based on his research **dnspop**. +* [TheRook](https://github.com/TheRook) - The bruteforce module was based on his script **subbrute**. +* [Bitquark](https://github.com/bitquark) - The Subbrute's wordlist was based on his research **dnspop**. ## Thanks diff --git a/sublist3r.py b/sublist3r.py index e3aeeb3..42df1ba 100755 --- a/sublist3r.py +++ b/sublist3r.py @@ -50,10 +50,11 @@ R = '\033[91m' # red W = '\033[0m' # white try: - import win_unicode_console , colorama + import win_unicode_console + import colorama win_unicode_console.enable() colorama.init() - #Now the unicode will work ^_^ + # Now the unicode will work ^_^ except: print("[!] Error: Coloring libraries not installed, no coloring will be used [Check the readme]") G = Y = B = R = W = G = Y = B = R = W = '' @@ -98,15 +99,14 @@ def parse_args(): parser.add_argument('-t', '--threads', help='Number of threads to use for subbrute bruteforce', type=int, default=30) parser.add_argument('-e', '--engines', help='Specify a comma-separated list of search engines') parser.add_argument('-o', '--output', help='Save the results to text file') + parser.add_argument('-i', '--findip', help='Find IP address of each subdomain', action='store_true') return parser.parse_args() -def write_file(filename, subdomains): +def write_file(filename, data): # saving subdomains results to output file - print("%s[-] Saving results to file: %s%s%s%s" % (Y, W, R, filename, W)) - with open(str(filename), 'wt') as f: - for subdomain in subdomains: - f.write(subdomain + os.linesep) + with open(str(filename), 'a') as f: + f.write(data + "\n") def subdomain_sorting_key(hostname): @@ -135,7 +135,14 @@ def subdomain_sorting_key(hostname): return parts, 0 -class enumratorBase(object): +def get_host_by_name(subdomain): + try: + return socket.gethostbyname(subdomain) + except Exception: + return "0.0.0.0" + + +class EnumratorBase(object): def __init__(self, base_url, engine_name, domain, subdomains=None, silent=False, verbose=True): subdomains = subdomains or [] self.domain = urlparse.urlparse(domain).netloc @@ -147,22 +154,20 @@ def __init__(self, base_url, engine_name, domain, subdomains=None, silent=False, self.silent = silent self.verbose = verbose self.headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.8', - 'Accept-Encoding': 'gzip', - } + 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.8', + 'Accept-Encoding': 'gzip', + } self.print_banner() def print_(self, text): if not self.silent: print(text) - return def print_banner(self): """ subclass can override this if they want a fancy banner :)""" self.print_(G + "[-] Searching now in %s.." % (self.engine_name) + W) - return def send_req(self, query, page_no=1): @@ -251,10 +256,10 @@ def enumerate(self, altquery=False): return self.subdomains -class enumratorBaseThreaded(multiprocessing.Process, enumratorBase): +class EnumratorBaseThreaded(multiprocessing.Process, EnumratorBase): def __init__(self, base_url, engine_name, domain, subdomains=None, q=None, lock=threading.Lock(), silent=False, verbose=True): subdomains = subdomains or [] - enumratorBase.__init__(self, base_url, engine_name, domain, subdomains, silent=silent, verbose=verbose) + EnumratorBase.__init__(self, base_url, engine_name, domain, subdomains, silent=silent, verbose=verbose) multiprocessing.Process.__init__(self) self.lock = lock self.q = q @@ -266,7 +271,7 @@ def run(self): self.q.append(domain) -class GoogleEnum(enumratorBaseThreaded): +class GoogleEnum(EnumratorBaseThreaded): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True): subdomains = subdomains or [] base_url = "https://google.com/search?q={query}&btnG=Search&hl=en-US&biw=&bih=&gbv=1&start={page_no}&filter=0" @@ -315,7 +320,7 @@ def generate_query(self): return query -class YahooEnum(enumratorBaseThreaded): +class YahooEnum(EnumratorBaseThreaded): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True): subdomains = subdomains or [] base_url = "https://search.yahoo.com/search?p={query}&b={page_no}" @@ -366,14 +371,14 @@ def generate_query(self): return query -class AskEnum(enumratorBaseThreaded): +class AskEnum(EnumratorBaseThreaded): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True): subdomains = subdomains or [] base_url = 'http://www.ask.com/web?q={query}&page={page_no}&qid=8D6EE6BF52E0C04527E51F64F22C4534&o=0&l=dir&qsrc=998&qo=pagination' self.engine_name = "Ask" self.MAX_DOMAINS = 11 self.MAX_PAGES = 0 - enumratorBaseThreaded.__init__(self, base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose) + EnumratorBaseThreaded.__init__(self, base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose) self.q = q return @@ -408,14 +413,14 @@ def generate_query(self): return query -class BingEnum(enumratorBaseThreaded): +class BingEnum(EnumratorBaseThreaded): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True): subdomains = subdomains or [] base_url = 'https://www.bing.com/search?q={query}&go=Submit&first={page_no}' self.engine_name = "Bing" self.MAX_DOMAINS = 30 self.MAX_PAGES = 0 - enumratorBaseThreaded.__init__(self, base_url, self.engine_name, domain, subdomains, q=q, silent=silent) + EnumratorBaseThreaded.__init__(self, base_url, self.engine_name, domain, subdomains, q=q, silent=silent) self.q = q self.verbose = verbose return @@ -452,14 +457,14 @@ def generate_query(self): return query -class BaiduEnum(enumratorBaseThreaded): +class BaiduEnum(EnumratorBaseThreaded): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True): subdomains = subdomains or [] base_url = 'https://www.baidu.com/s?pn={page_no}&wd={query}&oq={query}' self.engine_name = "Baidu" self.MAX_DOMAINS = 2 self.MAX_PAGES = 760 - enumratorBaseThreaded.__init__(self, base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose) + EnumratorBaseThreaded.__init__(self, base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose) self.querydomain = self.domain self.q = q return @@ -511,7 +516,7 @@ def generate_query(self): return query -class NetcraftEnum(enumratorBaseThreaded): +class NetcraftEnum(EnumratorBaseThreaded): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True): subdomains = subdomains or [] self.base_url = 'https://searchdns.netcraft.com/?restriction=site+ends+with&host={domain}' @@ -581,7 +586,7 @@ def extract_domains(self, resp): return links_list -class DNSdumpster(enumratorBaseThreaded): +class DNSdumpster(EnumratorBaseThreaded): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True): subdomains = subdomains or [] base_url = 'https://dnsdumpster.com/' @@ -595,11 +600,11 @@ def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True): def check_host(self, host): is_valid = False - Resolver = dns.resolver.Resolver() - Resolver.nameservers = ['8.8.8.8', '8.8.4.4'] + resolver = dns.resolver.Resolver() + resolver.nameservers = ['8.8.8.8', '8.8.4.4'] self.lock.acquire() try: - ip = Resolver.query(host, 'A')[0].to_text() + ip = resolver.query(host, 'A')[0].to_text() if ip: if self.verbose: self.print_("%s%s: %s%s" % (R, self.engine_name, W, host)) @@ -660,7 +665,7 @@ def extract_domains(self, resp): return links -class Virustotal(enumratorBaseThreaded): +class Virustotal(EnumratorBaseThreaded): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True): subdomains = subdomains or [] base_url = 'https://www.virustotal.com/en/domain/{domain}/information/' @@ -703,7 +708,7 @@ def extract_domains(self, resp): pass -class ThreatCrowd(enumratorBaseThreaded): +class ThreatCrowd(EnumratorBaseThreaded): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True): subdomains = subdomains or [] base_url = 'https://www.threatcrowd.org/searchApi/v2/domain/report/?domain={domain}' @@ -738,11 +743,11 @@ def extract_domains(self, resp): if self.verbose: self.print_("%s%s: %s%s" % (R, self.engine_name, W, subdomain)) self.subdomains.append(subdomain.strip()) - except Exception as e: + except Exception: pass -class CrtSearch(enumratorBaseThreaded): +class CrtSearch(EnumratorBaseThreaded): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True): subdomains = subdomains or [] base_url = 'https://crt.sh/?q=%25.{domain}' @@ -777,17 +782,17 @@ def extract_domains(self, resp): continue if '@' in subdomain: - subdomain = subdomain[subdomain.find('@')+1:] + subdomain = subdomain[subdomain.find('@') + 1:] if subdomain not in self.subdomains and subdomain != self.domain: if self.verbose: self.print_("%s%s: %s%s" % (R, self.engine_name, W, subdomain)) self.subdomains.append(subdomain.strip()) - except Exception as e: + except Exception: pass -class PassiveDNS(enumratorBaseThreaded): +class PassiveDNS(EnumratorBaseThreaded): def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True): subdomains = subdomains or [] base_url = 'https://api.sublist3r.com/search.php?domain={domain}' @@ -800,7 +805,7 @@ def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True): def req(self, url): try: resp = self.session.get(url, headers=self.headers, timeout=self.timeout) - except Exception as e: + except Exception: resp = None return self.get_response(resp) @@ -822,41 +827,72 @@ def extract_domains(self, resp): if self.verbose: self.print_("%s%s: %s%s" % (R, self.engine_name, W, subdomain)) self.subdomains.append(subdomain.strip()) - except Exception as e: + except Exception: pass -class portscan(): - def __init__(self, subdomains, ports): +class Portscan(): + def __init__(self, subdomains, ports, savefile, find_ip=False): self.subdomains = subdomains self.ports = ports + self.savefile = savefile + self.find_ip = find_ip self.threads = 20 - self.lock = threading.BoundedSemaphore(value=self.threads) + self.scan_lock = threading.BoundedSemaphore(value=self.threads) + self.write_lock = threading.Lock() - def port_scan(self, host, ports): - openports = [] - self.lock.acquire() - for port in ports: + def port_scan(self, subdomain): + open_ports = [] + + self.scan_lock.acquire() + for port in self.ports: try: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(2) - result = s.connect_ex((host, int(port))) + result = s.connect_ex((subdomain, int(port))) if result == 0: - openports.append(port) + open_ports.append(port) s.close() except Exception: pass - self.lock.release() - if len(openports) > 0: - print("%s%s%s - %sFound open ports:%s %s%s%s" % (G, host, W, R, W, Y, ', '.join(openports), W)) + self.scan_lock.release() + + # Lock to prevent incorrect output format with threading + self.write_lock.acquire() + if len(open_ports) > 0: + if self.find_ip: + ip = get_host_by_name(subdomain) + print("{}{} ({}){} - {}Found open ports:{} {}{}{}".format(G, subdomain, ip, W, R, + W, Y, ', '.join(open_ports), W)) + if self.savefile: + write_file(self.savefile, "{},{},{}".format(subdomain, + ip, ','.join(open_ports))) + else: + print("{}{}{} - {}Found open ports:{} {}{}{}".format(G, subdomain, W, R, W, Y, + ', '.join(open_ports), W)) + if self.savefile: + write_file(self.savefile, "{},{}".format(subdomain, + ','.join(open_ports))) + else: + if self.find_ip: + ip = get_host_by_name(subdomain) + print("{}{} ({}){}".format(G, subdomain, ip, W)) + if self.savefile: + write_file(self.savefile, "{},{}".format(subdomain, ip)) + else: + print("{}{}{}".format(G, subdomain, W)) + if self.savefile: + write_file(self.savefile, "{}".format(subdomain)) + + self.write_lock.release() def run(self): for subdomain in self.subdomains: - t = threading.Thread(target=self.port_scan, args=(subdomain, self.ports)) + t = threading.Thread(target=self.port_scan, args=(subdomain, )) t.start() -def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, engines): +def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, engines, find_ip): bruteforce_list = set() search_list = set() @@ -887,23 +923,34 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e if verbose and not silent: print(Y + "[-] verbosity is enabled, will show the subdomains results in realtime" + W) - supported_engines = {'baidu': BaiduEnum, - 'yahoo': YahooEnum, - 'google': GoogleEnum, - 'bing': BingEnum, - 'ask': AskEnum, - 'netcraft': NetcraftEnum, - 'dnsdumpster': DNSdumpster, - 'virustotal': Virustotal, - 'threatcrowd': ThreatCrowd, - 'ssl': CrtSearch, - 'passivedns': PassiveDNS - } - - chosenEnums = [] + if savefile: + if os.path.isfile(savefile): + user_input = raw_input("{}[-] File exists!{} Do you want to overwrite file {}{}{}? [y/n] ".format(R, W, Y, savefile, W)) + if user_input.lower().startswith('y'): + os.remove(savefile) + else: + print("{}[-] Please re-run the script with different output file name{}".format(R, W)) + sys.exit(1) + print("%s[-] Saving results to file: %s%s%s%s" % (Y, W, R, savefile, W)) + + supported_engines = { + 'baidu': BaiduEnum, + 'yahoo': YahooEnum, + 'google': GoogleEnum, + 'bing': BingEnum, + 'ask': AskEnum, + 'netcraft': NetcraftEnum, + 'dnsdumpster': DNSdumpster, + 'virustotal': Virustotal, + 'threatcrowd': ThreatCrowd, + 'ssl': CrtSearch, + 'passivedns': PassiveDNS + } + + chosen_enums = [] if engines is None: - chosenEnums = [ + chosen_enums = [ BaiduEnum, YahooEnum, GoogleEnum, BingEnum, AskEnum, NetcraftEnum, DNSdumpster, Virustotal, ThreatCrowd, CrtSearch, PassiveDNS @@ -912,10 +959,10 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e engines = engines.split(',') for engine in engines: if engine.lower() in supported_engines: - chosenEnums.append(supported_engines[engine.lower()]) + chosen_enums.append(supported_engines[engine.lower()]) # Start the engines enumeration - enums = [enum(domain, [], q=subdomains_queue, silent=silent, verbose=verbose) for enum in chosenEnums] + enums = [enum(domain, [], q=subdomains_queue, silent=silent, verbose=verbose) for enum in chosen_enums] for enum in enums: enum.start() for enum in enums: @@ -935,6 +982,7 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e process_count = threads output = False json_output = False + bruteforce_list = subbrute.print_target(parsed_domain.netloc, record_type, subs, resolvers, process_count, output, json_output, search_list, verbose) subdomains = search_list.union(bruteforce_list) @@ -942,9 +990,6 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e if subdomains: subdomains = sorted(subdomains, key=subdomain_sorting_key) - if savefile: - write_file(savefile, subdomains) - if not silent: print(Y + "[-] Total Unique Subdomains Found: %s" % len(subdomains) + W) @@ -952,12 +997,21 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e if not silent: print(G + "[-] Start port scan now for the following ports: %s%s" % (Y, ports) + W) ports = ports.split(',') - pscan = portscan(subdomains, ports) + pscan = Portscan(subdomains, ports, savefile, find_ip) pscan.run() elif not silent: for subdomain in subdomains: - print(G + subdomain + W) + if find_ip: + ip = get_host_by_name(subdomain) + print("{}{} ({}){}".format(G, subdomain, ip, W)) + if savefile: + write_file(savefile, "{},{}".format(subdomain, ip)) + else: + print("{}{}{}".format(G, subdomain, W)) + if savefile: + write_file(savefile, "{},{}".format(subdomain, ip)) + return subdomains @@ -970,7 +1024,8 @@ def main(domain, threads, savefile, ports, silent, verbose, enable_bruteforce, e enable_bruteforce = args.bruteforce verbose = args.verbose engines = args.engines + find_ip = args.findip if verbose or verbose is None: verbose = True banner() - res = main(domain, threads, savefile, ports, silent=False, verbose=verbose, enable_bruteforce=enable_bruteforce, engines=engines) + res = main(domain, threads, savefile, ports, silent=False, verbose=verbose, enable_bruteforce=enable_bruteforce, engines=engines, find_ip=find_ip)