From 329140526d23705b85b442fe066d103646ff4109 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Tue, 23 Jan 2018 18:06:57 -0500 Subject: [PATCH 01/61] Move included subdomain lists from data_files to package_data Data included under package_data is better suite for files that will be referenced by code within the package. This is what Fierce does, so let's use package_data. --- fierce.py | 34 ++++++++++++++-------------------- setup.py | 8 +++++--- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/fierce.py b/fierce.py index b0cbe42..e89af1c 100755 --- a/fierce.py +++ b/fierce.py @@ -38,28 +38,16 @@ def find_subdomain_list_file(filename): # # https://github.com/pypa/pip/blob/master/pip/commands/show.py # + package_filename_path = os.path.join("lists", filename) try: - fierce = pkg_resources.get_distribution('fierce') - except pkg_resources.DistributionNotFound: + full_package_path = pkg_resources.resource_filename( + 'fierce', + package_filename_path + ) + except ImportError: return filename - if isinstance(fierce, pkg_resources.Distribution): - paths = [] - if fierce.has_metadata('RECORD'): - lines = fierce.get_metadata_lines('RECORD') - paths = [l.split(',')[0] for l in lines] - paths = [os.path.join(fierce.location, p) for p in paths] - elif fierce.has_metadata('installed-files.txt'): - lines = fierce.get_metadata_lines('installed-files.txt') - paths = [l for l in lines] - paths = [os.path.join(fierce.egg_info, p) for p in paths] - - for p in paths: - if filename == os.path.basename(p): - return p - - # If we couldn't find anything just return the original list file - return filename + return full_package_path def head_request(url, timeout=2): @@ -186,7 +174,13 @@ def get_stripped_file_lines(filename): """ Return lines of a file with whitespace removed """ - return [line.strip() for line in open(filename).readlines()] + try: + lines = [line.strip() for line in open(filename).readlines()] + except FileNotFoundError: + print("Could not find subdomain file '{}'".format(filename)) + return [] + + return lines def get_subdomains(subdomains, subdomain_filename): diff --git a/setup.py b/setup.py index ded9778..7a29c78 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,9 @@ 'fierce = fierce:main', ], }, - data_files=[ - ('lists', data_files_lists), - ], + package_data={ + 'fierce': [ + 'lists/*', + ] + }, ) From b5be973c603ef8166949f7b3251f9cc3abc00dad Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Tue, 23 Jan 2018 18:13:25 -0500 Subject: [PATCH 02/61] Handle users exiting during script execution with Ctrl+C better --- fierce.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fierce.py b/fierce.py index e89af1c..887769a 100755 --- a/fierce.py +++ b/fierce.py @@ -396,7 +396,10 @@ def parse_args(): def main(): args = parse_args() - fierce(**vars(args)) + try: + fierce(**vars(args)) + except KeyboardInterrupt: + pass if __name__ == "__main__": From 68620e8f177ac4bf9b1c329421d96866c37cc5ca Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Tue, 23 Jan 2018 18:15:55 -0500 Subject: [PATCH 03/61] Remove official Python 3.3 support as it's EOL --- .travis.yml | 1 - setup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index dafc54f..006ba28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - 3.3 - 3.4 - 3.5 - 3.6 diff --git a/setup.py b/setup.py index 7a29c78..7d054b4 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,6 @@ 'Operating System :: MacOS :: MacOS X', 'Operating System :: Microsoft :: Windows', 'Operating System :: POSIX', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', From 88230da70886a54c7f91f82b511db9293447bdd0 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Wed, 24 Jan 2018 22:06:12 -0500 Subject: [PATCH 04/61] Revert "Move included subdomain lists from data_files to package_data" This reverts commit 329140526d23705b85b442fe066d103646ff4109. --- fierce.py | 34 ++++++++++++++++++++-------------- setup.py | 8 +++----- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/fierce.py b/fierce.py index 887769a..f6012fa 100755 --- a/fierce.py +++ b/fierce.py @@ -38,16 +38,28 @@ def find_subdomain_list_file(filename): # # https://github.com/pypa/pip/blob/master/pip/commands/show.py # - package_filename_path = os.path.join("lists", filename) try: - full_package_path = pkg_resources.resource_filename( - 'fierce', - package_filename_path - ) - except ImportError: + fierce = pkg_resources.get_distribution('fierce') + except pkg_resources.DistributionNotFound: return filename - return full_package_path + if isinstance(fierce, pkg_resources.Distribution): + paths = [] + if fierce.has_metadata('RECORD'): + lines = fierce.get_metadata_lines('RECORD') + paths = [l.split(',')[0] for l in lines] + paths = [os.path.join(fierce.location, p) for p in paths] + elif fierce.has_metadata('installed-files.txt'): + lines = fierce.get_metadata_lines('installed-files.txt') + paths = [l for l in lines] + paths = [os.path.join(fierce.egg_info, p) for p in paths] + + for p in paths: + if filename == os.path.basename(p): + return p + + # If we couldn't find anything just return the original list file + return filename def head_request(url, timeout=2): @@ -174,13 +186,7 @@ def get_stripped_file_lines(filename): """ Return lines of a file with whitespace removed """ - try: - lines = [line.strip() for line in open(filename).readlines()] - except FileNotFoundError: - print("Could not find subdomain file '{}'".format(filename)) - return [] - - return lines + return [line.strip() for line in open(filename).readlines()] def get_subdomains(subdomains, subdomain_filename): diff --git a/setup.py b/setup.py index 7d054b4..56d1ab9 100644 --- a/setup.py +++ b/setup.py @@ -43,9 +43,7 @@ 'fierce = fierce:main', ], }, - package_data={ - 'fierce': [ - 'lists/*', - ] - }, + data_files=[ + ('lists', data_files_lists), + ], ) From 1303df3ba5ee214b25096d1d25de510c9b13f4dc Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Wed, 24 Jan 2018 22:12:51 -0500 Subject: [PATCH 05/61] Add subdomain lists to MANIFEST file for source distribution --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..7b4254a --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +recursive-include lists *.txt From ffe4285efe586313a89810bb0979dcb0c58bcf70 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Fri, 9 Feb 2018 17:02:15 -0500 Subject: [PATCH 06/61] Move fierce and associated data into its own package directory This makes having associated package data much easier. --- MANIFEST.in | 1 - fierce/__init__.py | 0 fierce.py => fierce/fierce.py | 31 +++++++---------------------- {lists => fierce/lists}/20000.txt | 0 {lists => fierce/lists}/5000.txt | 0 {lists => fierce/lists}/default.txt | 0 setup.py | 15 +++++++------- tests/test_fierce.py | 2 +- tests/test_filesystem.py | 2 +- 9 files changed, 17 insertions(+), 34 deletions(-) delete mode 100644 MANIFEST.in create mode 100644 fierce/__init__.py rename fierce.py => fierce/fierce.py (91%) rename {lists => fierce/lists}/20000.txt (100%) rename {lists => fierce/lists}/5000.txt (100%) rename {lists => fierce/lists}/default.txt (100%) diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 7b4254a..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -recursive-include lists *.txt diff --git a/fierce/__init__.py b/fierce/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fierce.py b/fierce/fierce.py similarity index 91% rename from fierce.py rename to fierce/fierce.py index f6012fa..551b5e3 100755 --- a/fierce.py +++ b/fierce/fierce.py @@ -33,33 +33,16 @@ def find_subdomain_list_file(filename): # If the relative check failed then attempt to find the list file # in the pip package directory. This will typically happen on pip package # installs (duh) - # - # Here's how pip itself handles this: - # - # https://github.com/pypa/pip/blob/master/pip/commands/show.py - # + package_filename_path = os.path.join("lists", filename) try: - fierce = pkg_resources.get_distribution('fierce') - except pkg_resources.DistributionNotFound: + full_package_path = pkg_resources.resource_filename( + "fierce", + package_filename_path + ) + except ImportError: return filename - if isinstance(fierce, pkg_resources.Distribution): - paths = [] - if fierce.has_metadata('RECORD'): - lines = fierce.get_metadata_lines('RECORD') - paths = [l.split(',')[0] for l in lines] - paths = [os.path.join(fierce.location, p) for p in paths] - elif fierce.has_metadata('installed-files.txt'): - lines = fierce.get_metadata_lines('installed-files.txt') - paths = [l for l in lines] - paths = [os.path.join(fierce.egg_info, p) for p in paths] - - for p in paths: - if filename == os.path.basename(p): - return p - - # If we couldn't find anything just return the original list file - return filename + return full_package_path def head_request(url, timeout=2): diff --git a/lists/20000.txt b/fierce/lists/20000.txt similarity index 100% rename from lists/20000.txt rename to fierce/lists/20000.txt diff --git a/lists/5000.txt b/fierce/lists/5000.txt similarity index 100% rename from lists/5000.txt rename to fierce/lists/5000.txt diff --git a/lists/default.txt b/fierce/lists/default.txt similarity index 100% rename from lists/default.txt rename to fierce/lists/default.txt diff --git a/setup.py b/setup.py index 56d1ab9..b1a76e7 100644 --- a/setup.py +++ b/setup.py @@ -14,14 +14,13 @@ with open(requirements_filename) as fd: tests_require = [i.strip() for i in fd.readlines()] -data_files_lists = [os.path.join('lists', l) for l in os.listdir('lists')] - setup( name='fierce', version='1.2.0', description='A DNS reconnaissance tool for locating non-contiguous IP space.', url='https://github.com/mschwager/fierce', - py_modules=['fierce'], + packages=['fierce'], + package_dir={'fierce': 'fierce'}, license='GPLv3', classifiers=[ 'Environment :: Console', @@ -40,10 +39,12 @@ tests_require=tests_require, entry_points={ 'console_scripts': [ - 'fierce = fierce:main', + 'fierce = fierce.fierce:main', + ], + }, + package_data={ + 'fierce': [ + 'lists/*.txt', ], }, - data_files=[ - ('lists', data_files_lists), - ], ) diff --git a/tests/test_fierce.py b/tests/test_fierce.py index 17b9d08..9fb2288 100644 --- a/tests/test_fierce.py +++ b/tests/test_fierce.py @@ -8,7 +8,7 @@ import dns.name import dns.resolver -import fierce +from fierce import fierce class TestFierce(unittest.TestCase): diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py index 7ad9b0d..cbcf32c 100644 --- a/tests/test_filesystem.py +++ b/tests/test_filesystem.py @@ -8,7 +8,7 @@ from pyfakefs import fake_filesystem_unittest -import fierce +from fierce import fierce class TestFilesystem(fake_filesystem_unittest.TestCase): From cd56af80048002cdaf28a5135d875e09ee6f0040 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Fri, 9 Feb 2018 17:04:33 -0500 Subject: [PATCH 07/61] Update README with directory change --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f4d3f56..0de3b44 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ OR $ git clone https://github.com/mschwager/fierce.git $ cd fierce $ pip3 install -r requirements.txt -$ python3 fierce.py -h +$ python3 fierce/fierce.py -h ``` # Using From 0065d0226ed29d96e80f9bd7e7b0edeeccde00dd Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Fri, 9 Feb 2018 17:14:26 -0500 Subject: [PATCH 08/61] Update .coveragerc with directory change --- .coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 8aa3f08..16c6d5c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,2 +1,2 @@ [run] -include = fierce.py +include = fierce/fierce.py From 5882b58cc95d945666a655088a8f28a03976c2b5 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Sat, 10 Feb 2018 16:33:33 -0500 Subject: [PATCH 09/61] Gracefully handle zone transfer DNS errors --- fierce/fierce.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index 551b5e3..8336776 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -118,7 +118,7 @@ def recursive_query(resolver, domain, record_type='NS'): def zone_transfer(address, domain): try: return dns.zone.from_xfr(dns.query.xfr(address, domain)) - except (ConnectionError, EOFError, TimeoutError, dns.exception.FormError): + except (ConnectionError, EOFError, TimeoutError, dns.exception.DNSException): return None From 9fd57b30a28a0a2165d5eadfcf85c1cde9f3c48c Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Mon, 12 Feb 2018 20:24:33 -0500 Subject: [PATCH 10/61] Asynchronously perform nearby reverse IP lookups --- fierce/fierce.py | 34 +++++++++++++++++------ tests/test_fierce.py | 66 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 8 deletions(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index 8336776..5579dcc 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 import argparse +import concurrent.futures import functools import http.client import ipaddress +import itertools import os import pprint import random @@ -152,17 +154,27 @@ def search_filter(domains, address): def find_nearby(resolver, ips, filter_func=None): - reversed_ips = {str(i): reverse_query(resolver, str(i)) for i in ips} + str_ips = [str(ip) for ip in ips] + + with concurrent.futures.ThreadPoolExecutor() as executor: + reversed_ips = { + ip: query_result + for ip, query_result in zip( + str_ips, + executor.map( + reverse_query, + itertools.repeat(resolver, len(str_ips)), + str_ips + ) + ) + } + reversed_ips = {k: v for k, v in reversed_ips.items() if v is not None} if filter_func: reversed_ips = {k: v for k, v in reversed_ips.items() if v and filter_func(v[0].to_text())} - if not reversed_ips: - return - - print("Nearby:") - pprint.pprint({k: v[0].to_text() for k, v in reversed_ips.items() if v}) + return reversed_ips def get_stripped_file_lines(filename): @@ -216,7 +228,10 @@ def fierce(**kwargs): if kwargs.get("range"): internal_range = ipaddress.IPv4Network(kwargs.get("range")) - find_nearby(resolver, list(internal_range)) + nearby_ips = find_nearby(resolver, list(internal_range)) + if nearby_ips: + print("Nearby:") + pprint.pprint({k: v[0].to_text() for k, v in nearby_ips.items() if v}) if not kwargs.get("domain"): return @@ -293,7 +308,10 @@ def fierce(**kwargs): ips = set(ips) - set(visited) visited |= ips - find_nearby(resolver, ips, filter_func=filter_func) + nearby_ips = find_nearby(resolver, ips, filter_func=filter_func) + if nearby_ips: + print("Nearby:") + pprint.pprint({k: v[0].to_text() for k, v in nearby_ips.items() if v}) if kwargs.get("delay"): time.sleep(kwargs["delay"]) diff --git a/tests/test_fierce.py b/tests/test_fierce.py index 9fb2288..c441968 100644 --- a/tests/test_fierce.py +++ b/tests/test_fierce.py @@ -275,6 +275,72 @@ def test_zone_transfer_form_error(self): self.assertIsNone(result) + def test_find_nearby_empty(self): + resolver = 'unused' + ips = [] + + result = fierce.find_nearby(resolver, ips) + expected = {} + + self.assertEqual(expected, result) + + def test_find_nearby_basic(self): + resolver = 'unused' + ips = [ + ipaddress.IPv4Address('192.168.1.0'), + ipaddress.IPv4Address('192.168.1.1'), + ] + side_effect = [ + 'sd1.example.com.', + 'sd2.example.com.', + ] + + with unittest.mock.patch.object(fierce, 'reverse_query', side_effect=side_effect): + result = fierce.find_nearby(resolver, ips) + + expected = { + '192.168.1.0': 'sd1.example.com.', + '192.168.1.1': 'sd2.example.com.', + } + + self.assertEqual(expected, result) + + def test_find_nearby_filter_func(self): + resolver = 'unused' + ips = [ + ipaddress.IPv4Address('192.168.1.0'), + ipaddress.IPv4Address('192.168.1.1'), + ] + + # Simply getting a dns.resolver.Answer with a specific result was + # more difficult than I'd like, let's just go with this less than + # ideal approach for now + class MockAnswer(object): + def __init__(self, response): + self.response = response + + def to_text(self): + return self.response + + returned_answer = [MockAnswer('sd1.example.com.')] + + side_effect = [ + returned_answer, + [MockAnswer('sd2.example.com.')], + ] + + def filter_func(reverse_result): + return reverse_result == 'sd1.example.com.' + + with unittest.mock.patch.object(fierce, 'reverse_query', side_effect=side_effect): + result = fierce.find_nearby(resolver, ips, filter_func=filter_func) + + expected = { + '192.168.1.0': returned_answer, + } + + self.assertEqual(expected, result) + if __name__ == "__main__": unittest.main() From b64bc575cc4b51451dd9312269a065d8a7d256c3 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Mon, 12 Feb 2018 20:26:09 -0500 Subject: [PATCH 11/61] Print exiting message when we receive a keyboard interrupt --- fierce/fierce.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index 5579dcc..c4440aa 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -406,7 +406,7 @@ def main(): try: fierce(**vars(args)) except KeyboardInterrupt: - pass + print("Exiting...") if __name__ == "__main__": From fceb90d81cf07ea81b33844203350d78f60cd160 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Mon, 12 Feb 2018 20:35:13 -0500 Subject: [PATCH 12/61] A default max_workers value for ThreadPoolExecutor doesn't exist until Python 3.5 --- fierce/fierce.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index c4440aa..67973a1 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -6,6 +6,7 @@ import http.client import ipaddress import itertools +import multiprocessing import os import pprint import random @@ -156,7 +157,10 @@ def search_filter(domains, address): def find_nearby(resolver, ips, filter_func=None): str_ips = [str(ip) for ip in ips] - with concurrent.futures.ThreadPoolExecutor() as executor: + # https://docs.python.org/3.5/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor + max_workers = multiprocessing.cpu_count() * 5 + + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: reversed_ips = { ip: query_result for ip, query_result in zip( From c38dba7fb3c1ab67280bd3108d95fd6ebf55734a Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Tue, 13 Feb 2018 18:33:43 -0500 Subject: [PATCH 13/61] Updated dev dependencies --- requirements-dev.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 201146c..6a90628 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,3 @@ -nose2==0.6.5 -cov-core==1.15.0 -coveralls==1.1 -pyfakefs==3.1 +nose2==0.7.3 +coveralls==1.2.0 +pyfakefs==3.3 From a8599f86c48cd7d995f6bd1b8b8dff4823a2648e Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Tue, 13 Feb 2018 18:38:57 -0500 Subject: [PATCH 14/61] Added flake8 linting to CI process --- .travis.yml | 1 + requirements-dev.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 006ba28..23ea2b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ install: - pip install -r requirements.txt - pip install -r requirements-dev.txt script: + - flake8 --ignore=E501 fierce/ tests/ - nose2 --with-coverage after_success: - coveralls diff --git a/requirements-dev.txt b/requirements-dev.txt index 6a90628..bf1bb3a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ nose2==0.7.3 +flake8==3.2.1 coveralls==1.2.0 pyfakefs==3.3 From 143969fb54a6b5403ab67f633f802a5b2db529a2 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Sat, 24 Feb 2018 15:17:56 -0500 Subject: [PATCH 15/61] Move printing of subdomain result to its own function --- fierce/fierce.py | 42 ++++++++++++++++------- tests/test_fierce.py | 79 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 12 deletions(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index 67973a1..1bd68bb 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -11,6 +11,7 @@ import pprint import random import socket +import sys import time import dns.exception @@ -21,6 +22,18 @@ import dns.zone +def print_subdomain_result(url, ip, http_connection_headers=None, nearby=None, stream=sys.stdout): + print("Found: {} ({})".format(url, ip), file=stream) + + if http_connection_headers: + print("HTTP connected:", file=stream) + pprint.pprint(http_connection_headers, stream=stream) + + if nearby: + print("Nearby:", file=stream) + pprint.pprint(nearby, stream=stream) + + def find_subdomain_list_file(filename): # First check the list directory relative to where we are. This # will typically happen if they simply cloned the Github repository @@ -290,32 +303,37 @@ def fierce(**kwargs): continue ip = ipaddress.IPv4Address(record[0].address) - print("Found: {} ({})".format(url, ip)) + http_connection_headers = None if kwargs.get('connect') and not ip.is_private: - headers = head_request(str(ip)) - if headers: - print("HTTP connected:") - pprint.pprint(headers) + http_connection_headers = head_request(str(ip)) + + filter_func = None + if kwargs.get("search"): + filter_func = functools.partial(search_filter, kwargs["search"]) if kwargs.get("wide"): ips = wide_expander(ip) elif kwargs.get("traverse"): ips = traverse_expander(ip, kwargs["traverse"]) else: - continue - - filter_func = None - if kwargs.get("search"): - filter_func = functools.partial(search_filter, kwargs["search"]) + ips = [] ips = set(ips) - set(visited) visited |= ips nearby_ips = find_nearby(resolver, ips, filter_func=filter_func) + + nearby = None if nearby_ips: - print("Nearby:") - pprint.pprint({k: v[0].to_text() for k, v in nearby_ips.items() if v}) + nearby = {k: v[0].to_text() for k, v in nearby_ips.items() if v} + + print_subdomain_result( + url, + ip, + http_connection_headers=http_connection_headers, + nearby=nearby + ) if kwargs.get("delay"): time.sleep(kwargs["delay"]) diff --git a/tests/test_fierce.py b/tests/test_fierce.py index c441968..f1995a9 100644 --- a/tests/test_fierce.py +++ b/tests/test_fierce.py @@ -1,6 +1,8 @@ #!/usr/bin/env python import ipaddress +import io +import textwrap import unittest import unittest.mock @@ -341,6 +343,83 @@ def filter_func(reverse_result): self.assertEqual(expected, result) + def test_print_subdomain_result_basic(self): + url = 'example.com' + ip = '192.168.1.0' + + with io.StringIO() as stream: + fierce.print_subdomain_result(url, ip, stream=stream) + result = stream.getvalue() + + expected = 'Found: example.com (192.168.1.0)\n' + + self.assertEqual(expected, result) + + def test_print_subdomain_result_nearby(self): + url = 'example.com' + ip = '192.168.1.0' + nearby = {'192.168.1.1': 'nearby.com'} + + with io.StringIO() as stream: + fierce.print_subdomain_result(url, ip, nearby=nearby, stream=stream) + result = stream.getvalue() + + expected = textwrap.dedent(''' + Found: example.com (192.168.1.0) + Nearby: + {'192.168.1.1': 'nearby.com'} + ''').lstrip() + + self.assertEqual(expected, result) + + def test_print_subdomain_result_http_header(self): + url = 'example.com' + ip = '192.168.1.0' + http_connection_headers = {'HTTP HEADER': 'value'} + + with io.StringIO() as stream: + fierce.print_subdomain_result( + url, + ip, + http_connection_headers=http_connection_headers, + stream=stream + ) + result = stream.getvalue() + + expected = textwrap.dedent(''' + Found: example.com (192.168.1.0) + HTTP connected: + {'HTTP HEADER': 'value'} + ''').lstrip() + + self.assertEqual(expected, result) + + def test_print_subdomain_result_both(self): + url = 'example.com' + ip = '192.168.1.0' + http_connection_headers = {'HTTP HEADER': 'value'} + nearby = {'192.168.1.1': 'nearby.com'} + + with io.StringIO() as stream: + fierce.print_subdomain_result( + url, + ip, + http_connection_headers=http_connection_headers, + nearby=nearby, + stream=stream + ) + result = stream.getvalue() + + expected = textwrap.dedent(''' + Found: example.com (192.168.1.0) + HTTP connected: + {'HTTP HEADER': 'value'} + Nearby: + {'192.168.1.1': 'nearby.com'} + ''').lstrip() + + self.assertEqual(expected, result) + if __name__ == "__main__": unittest.main() From 65753a5926ac269eb8a82ac981b74bbae725524f Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Sat, 24 Feb 2018 15:29:13 -0500 Subject: [PATCH 16/61] Store visited IP state in a closure --- fierce/fierce.py | 17 ++++++++++++++--- tests/test_fierce.py | 27 +++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index 1bd68bb..279b1b6 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -34,6 +34,18 @@ def print_subdomain_result(url, ip, http_connection_headers=None, nearby=None, s pprint.pprint(nearby, stream=stream) +def unvisited_closure(): + visited = set() + + def inner(l): + nonlocal visited + result = l.difference(visited) + visited.update(l) + return result + + return inner + + def find_subdomain_list_file(filename): # First check the list directory relative to where we are. This # will typically happen if they simply cloned the Github repository @@ -293,7 +305,7 @@ def fierce(**kwargs): kwargs["subdomain_file"] ) - visited = set() + get_unvisited = unvisited_closure() for subdomain in subdomains: url = concatenate_subdomains(domain, [subdomain]) @@ -319,8 +331,7 @@ def fierce(**kwargs): else: ips = [] - ips = set(ips) - set(visited) - visited |= ips + ips = get_unvisited(ips) nearby_ips = find_nearby(resolver, ips, filter_func=filter_func) diff --git a/tests/test_fierce.py b/tests/test_fierce.py index f1995a9..162eb9c 100644 --- a/tests/test_fierce.py +++ b/tests/test_fierce.py @@ -420,6 +420,33 @@ def test_print_subdomain_result_both(self): self.assertEqual(expected, result) + def test_unvisited_closure_empty(self): + unvisited = fierce.unvisited_closure() + ips = set() + + result = unvisited(ips) + expected = set() + + self.assertEqual(expected, result) + + def test_unvisited_closure_empty_intersection(self): + unvisited = fierce.unvisited_closure() + + unvisited(set([1, 2, 3])) + result = unvisited(set([4, 5, 6])) + expected = set([4, 5, 6]) + + self.assertEqual(expected, result) + + def test_unvisited_closure_overlapping_intersection(self): + unvisited = fierce.unvisited_closure() + + unvisited(set([1, 2, 3])) + result = unvisited(set([2, 3, 4])) + expected = set([4]) + + self.assertEqual(expected, result) + if __name__ == "__main__": unittest.main() From 0de647bbba2fff8b672d4d6881abfb22e8e7a3d1 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Sat, 24 Feb 2018 18:19:25 -0500 Subject: [PATCH 17/61] Fix list -> set --- fierce/fierce.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index 279b1b6..fe6133a 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -39,7 +39,7 @@ def unvisited_closure(): def inner(l): nonlocal visited - result = l.difference(visited) + result = set(l).difference(visited) visited.update(l) return result From 9ad24cc304ffd81b1d9916ed6a55122f18280687 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Sun, 25 Feb 2018 12:43:08 -0500 Subject: [PATCH 18/61] Pull filter and expander functionality out of subdomain enumeration loop --- fierce/fierce.py | 67 ++++++++++++++++++++++++++----------- tests/test_fierce.py | 79 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 110 insertions(+), 36 deletions(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index fe6133a..f73469b 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -158,6 +158,10 @@ def get_class_c_network(ip): return class_c +def default_expander(ip): + return [ip] + + def traverse_expander(ip, n=5): class_c = get_class_c_network(ip) @@ -175,11 +179,26 @@ def wide_expander(ip): return result +def range_expander(ip): + network = ipaddress.IPv4Network(ip) + + result = list(network) + + return result + + +def default_filter(address): + return True + + def search_filter(domains, address): return any(domain in address for domain in domains) def find_nearby(resolver, ips, filter_func=None): + if filter_func is None: + filter_func = default_filter + str_ips = [str(ip) for ip in ips] # https://docs.python.org/3.5/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor @@ -198,10 +217,11 @@ def find_nearby(resolver, ips, filter_func=None): ) } - reversed_ips = {k: v for k, v in reversed_ips.items() if v is not None} - - if filter_func: - reversed_ips = {k: v for k, v in reversed_ips.items() if v and filter_func(v[0].to_text())} + reversed_ips = { + k: v + for k, v in reversed_ips.items() + if v is not None and filter_func(v[0].to_text()) + } return reversed_ips @@ -256,8 +276,11 @@ def fierce(**kwargs): ) if kwargs.get("range"): - internal_range = ipaddress.IPv4Network(kwargs.get("range")) - nearby_ips = find_nearby(resolver, list(internal_range)) + range_ips = range_expander(kwargs.get("range")) + nearby_ips = find_nearby( + resolver, + range_ips, + ) if nearby_ips: print("Nearby:") pprint.pprint({k: v[0].to_text() for k, v in nearby_ips.items() if v}) @@ -305,7 +328,17 @@ def fierce(**kwargs): kwargs["subdomain_file"] ) - get_unvisited = unvisited_closure() + filter_func = None + if kwargs.get("search"): + filter_func = functools.partial(search_filter, kwargs["search"]) + + expander_func = default_expander + if kwargs.get("wide"): + expander_func = wide_expander + elif kwargs.get("traverse"): + expander_func = functools.partial(traverse_expander, n=kwargs["traverse"]) + + unvisited = unvisited_closure() for subdomain in subdomains: url = concatenate_subdomains(domain, [subdomain]) @@ -320,20 +353,14 @@ def fierce(**kwargs): if kwargs.get('connect') and not ip.is_private: http_connection_headers = head_request(str(ip)) - filter_func = None - if kwargs.get("search"): - filter_func = functools.partial(search_filter, kwargs["search"]) - - if kwargs.get("wide"): - ips = wide_expander(ip) - elif kwargs.get("traverse"): - ips = traverse_expander(ip, kwargs["traverse"]) - else: - ips = [] + ips = expander_func(ip) + unvisited_ips = unvisited(ips) - ips = get_unvisited(ips) - - nearby_ips = find_nearby(resolver, ips, filter_func=filter_func) + nearby_ips = find_nearby( + resolver, + unvisited_ips, + filter_func=filter_func + ) nearby = None if nearby_ips: diff --git a/tests/test_fierce.py b/tests/test_fierce.py index 162eb9c..3a4d130 100644 --- a/tests/test_fierce.py +++ b/tests/test_fierce.py @@ -13,6 +13,17 @@ from fierce import fierce +# Simply getting a dns.resolver.Answer with a specific result was +# more difficult than I'd like, let's just go with this less than +# ideal approach for now +class MockAnswer(object): + def __init__(self, response): + self.response = response + + def to_text(self): + return self.response + + class TestFierce(unittest.TestCase): def test_concatenate_subdomains_empty(self): @@ -79,6 +90,16 @@ def test_concatenate_subdomains_fqdn_subdomain(self): self.assertEqual(expected, result) + def test_default_expander(self): + ip = ipaddress.IPv4Address('192.168.1.1') + + result = fierce.default_expander(ip) + expected = [ + ipaddress.IPv4Address('192.168.1.1'), + ] + + self.assertEqual(expected, result) + def test_traverse_expander_basic(self): ip = ipaddress.IPv4Address('192.168.1.1') expand = 1 @@ -156,6 +177,18 @@ def test_wide_expander_upper_boundary(self): self.assertEqual(expected, result) + def test_range_expander(self): + ip = '192.168.1.0/31' + + result = fierce.range_expander(ip) + + expected = [ + ipaddress.IPv4Address('192.168.1.0'), + ipaddress.IPv4Address('192.168.1.1'), + ] + + self.assertEqual(expected, result) + def test_recursive_query_basic_failure(self): resolver = dns.resolver.Resolver() domain = dns.name.from_text('example.com.') @@ -292,17 +325,19 @@ def test_find_nearby_basic(self): ipaddress.IPv4Address('192.168.1.0'), ipaddress.IPv4Address('192.168.1.1'), ] + returned_answer1 = [MockAnswer('sd1.example.com.')] + returned_answer2 = [MockAnswer('sd2.example.com.')] side_effect = [ - 'sd1.example.com.', - 'sd2.example.com.', + returned_answer1, + returned_answer2, ] with unittest.mock.patch.object(fierce, 'reverse_query', side_effect=side_effect): result = fierce.find_nearby(resolver, ips) expected = { - '192.168.1.0': 'sd1.example.com.', - '192.168.1.1': 'sd2.example.com.', + '192.168.1.0': returned_answer1, + '192.168.1.1': returned_answer2, } self.assertEqual(expected, result) @@ -313,19 +348,7 @@ def test_find_nearby_filter_func(self): ipaddress.IPv4Address('192.168.1.0'), ipaddress.IPv4Address('192.168.1.1'), ] - - # Simply getting a dns.resolver.Answer with a specific result was - # more difficult than I'd like, let's just go with this less than - # ideal approach for now - class MockAnswer(object): - def __init__(self, response): - self.response = response - - def to_text(self): - return self.response - returned_answer = [MockAnswer('sd1.example.com.')] - side_effect = [ returned_answer, [MockAnswer('sd2.example.com.')], @@ -447,6 +470,30 @@ def test_unvisited_closure_overlapping_intersection(self): self.assertEqual(expected, result) + def test_search_filter_empty(self): + domains = [] + address = 'test.example.com' + + result = fierce.search_filter(domains, address) + + self.assertFalse(result) + + def test_search_filter_true(self): + domains = ['example.com'] + address = 'test.example.com' + + result = fierce.search_filter(domains, address) + + self.assertTrue(result) + + def test_search_filter_false(self): + domains = ['not.com'] + address = 'test.example.com' + + result = fierce.search_filter(domains, address) + + self.assertFalse(result) + if __name__ == "__main__": unittest.main() From 20e55ed0113a640c19d93bf976af4284dd78d9e9 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Sun, 25 Feb 2018 12:55:42 -0500 Subject: [PATCH 19/61] Return string DNS answer from nearby query instead of Answer object --- fierce/fierce.py | 14 +++++--------- tests/test_fierce.py | 15 ++++++--------- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index f73469b..68e7877 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -218,7 +218,7 @@ def find_nearby(resolver, ips, filter_func=None): } reversed_ips = { - k: v + k: v[0].to_text() for k, v in reversed_ips.items() if v is not None and filter_func(v[0].to_text()) } @@ -277,13 +277,13 @@ def fierce(**kwargs): if kwargs.get("range"): range_ips = range_expander(kwargs.get("range")) - nearby_ips = find_nearby( + nearby = find_nearby( resolver, range_ips, ) - if nearby_ips: + if nearby: print("Nearby:") - pprint.pprint({k: v[0].to_text() for k, v in nearby_ips.items() if v}) + pprint.pprint(nearby) if not kwargs.get("domain"): return @@ -356,16 +356,12 @@ def fierce(**kwargs): ips = expander_func(ip) unvisited_ips = unvisited(ips) - nearby_ips = find_nearby( + nearby = find_nearby( resolver, unvisited_ips, filter_func=filter_func ) - nearby = None - if nearby_ips: - nearby = {k: v[0].to_text() for k, v in nearby_ips.items() if v} - print_subdomain_result( url, ip, diff --git a/tests/test_fierce.py b/tests/test_fierce.py index 3a4d130..4de3543 100644 --- a/tests/test_fierce.py +++ b/tests/test_fierce.py @@ -325,19 +325,17 @@ def test_find_nearby_basic(self): ipaddress.IPv4Address('192.168.1.0'), ipaddress.IPv4Address('192.168.1.1'), ] - returned_answer1 = [MockAnswer('sd1.example.com.')] - returned_answer2 = [MockAnswer('sd2.example.com.')] side_effect = [ - returned_answer1, - returned_answer2, + [MockAnswer('sd1.example.com.')], + [MockAnswer('sd2.example.com.')], ] with unittest.mock.patch.object(fierce, 'reverse_query', side_effect=side_effect): result = fierce.find_nearby(resolver, ips) expected = { - '192.168.1.0': returned_answer1, - '192.168.1.1': returned_answer2, + '192.168.1.0': 'sd1.example.com.', + '192.168.1.1': 'sd2.example.com.', } self.assertEqual(expected, result) @@ -348,9 +346,8 @@ def test_find_nearby_filter_func(self): ipaddress.IPv4Address('192.168.1.0'), ipaddress.IPv4Address('192.168.1.1'), ] - returned_answer = [MockAnswer('sd1.example.com.')] side_effect = [ - returned_answer, + [MockAnswer('sd1.example.com.')], [MockAnswer('sd2.example.com.')], ] @@ -361,7 +358,7 @@ def filter_func(reverse_result): result = fierce.find_nearby(resolver, ips, filter_func=filter_func) expected = { - '192.168.1.0': returned_answer, + '192.168.1.0': 'sd1.example.com.', } self.assertEqual(expected, result) From 7b5a5938fc1eb93ecc3437227c734f27f65f0d1e Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Sun, 25 Feb 2018 13:09:03 -0500 Subject: [PATCH 20/61] Added better error handling --- fierce/fierce.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index 68e7877..d24127a 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -22,6 +22,11 @@ import dns.zone +def fatal(msg, return_code=-1): + print(msg) + exit(return_code) + + def print_subdomain_result(url, ip, http_connection_headers=None, nearby=None, stream=sys.stdout): print("Found: {} ({})".format(url, ip), file=stream) @@ -180,7 +185,10 @@ def wide_expander(ip): def range_expander(ip): - network = ipaddress.IPv4Network(ip) + try: + network = ipaddress.IPv4Network(ip) + except ipaddress.AddressValueError: + fatal("Invalid IPv4 CIDR: {!r}".format(ip)) result = list(network) @@ -309,8 +317,7 @@ def fierce(**kwargs): print("SOA: {} ({})".format(soa_mname, master_address)) else: print("SOA: failure") - print("Failed to lookup NS/SOA, Domain does not exist") - exit(-1) + fatal("Failed to lookup NS/SOA, Domain does not exist") zone = zone_transfer(master_address, domain) print("Zone: {}".format("success" if zone else "failure")) @@ -430,6 +437,7 @@ def parse_args(): '--subdomain-file', action='store', default="default.txt", + type=argparse.FileType('r'), help='use subdomains specified in this file (one per line)' ) @@ -443,6 +451,7 @@ def parse_args(): dns_group.add_argument( '--dns-file', action='store', + type=argparse.FileType('r'), help='use dns servers specified in this file for reverse lookups (one per line)' ) From cc58a2a01b6b8a5cbffd799199b964aa5006c477 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Sun, 25 Feb 2018 13:22:20 -0500 Subject: [PATCH 21/61] Correct default subdomain file handling behavior --- fierce/fierce.py | 13 ++++++------- tests/test_filesystem.py | 8 ++++---- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index d24127a..46e7473 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -238,7 +238,7 @@ def get_stripped_file_lines(filename): """ Return lines of a file with whitespace removed """ - return [line.strip() for line in open(filename).readlines()] + return [line.strip() for line in filename.readlines()] def get_subdomains(subdomains, subdomain_filename): @@ -426,6 +426,10 @@ def parse_args(): help='time to wait between lookups' ) + # Attempt to intelligently find the subdomain list depending on + # how this library was installed. + default_subdomain_file = find_subdomain_list_file('default.txt') + subdomain_group = p.add_mutually_exclusive_group() subdomain_group.add_argument( '--subdomains', @@ -436,7 +440,7 @@ def parse_args(): subdomain_group.add_argument( '--subdomain-file', action='store', - default="default.txt", + default=default_subdomain_file, type=argparse.FileType('r'), help='use subdomains specified in this file (one per line)' ) @@ -457,11 +461,6 @@ def parse_args(): args = p.parse_args() - # Attempt to intelligently find the subdomain list depending on - # how this library was installed. - if args.subdomain_file and not os.path.exists(args.subdomain_file): - args.subdomain_file = find_subdomain_list_file(args.subdomain_file) - return args diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py index cbcf32c..e81a1c5 100644 --- a/tests/test_filesystem.py +++ b/tests/test_filesystem.py @@ -85,7 +85,7 @@ def test_update_resolver_nameservers_no_nameserver_use_file(self): result = fierce.update_resolver_nameservers( resolver, nameservers, - nameserver_filename + open(nameserver_filename) ) expected = contents.split() @@ -110,7 +110,7 @@ def test_update_resolver_nameservers_prefer_nameservers_over_file(self): result = fierce.update_resolver_nameservers( resolver, nameservers, - nameserver_filename + open(nameserver_filename) ) expected = nameservers @@ -168,7 +168,7 @@ def test_get_subdomains_no_subdomains_use_file(self): result = fierce.get_subdomains( subdomains, - subdomain_filename + open(subdomain_filename) ) expected = contents.split() @@ -190,7 +190,7 @@ def test_get_subdomains_prefer_subdomains_over_file(self): result = fierce.get_subdomains( subdomains, - subdomain_filename + open(subdomain_filename) ) expected = subdomains From d704acb030e2f46c48e22798bf5b5f0f0cd3139a Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Tue, 27 Feb 2018 18:04:08 -0500 Subject: [PATCH 22/61] Revert to the old way of finding subdomain files, this didn't work with our included lists --- fierce/fierce.py | 20 ++++++++++++-------- tests/test_filesystem.py | 8 ++++---- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index 46e7473..58c9599 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -238,7 +238,12 @@ def get_stripped_file_lines(filename): """ Return lines of a file with whitespace removed """ - return [line.strip() for line in filename.readlines()] + try: + lines = open(filename).readlines() + except FileNotFoundError: + fatal("Could not open file: {!r}".format(filename)) + + return [line.strip() for line in lines] def get_subdomains(subdomains, subdomain_filename): @@ -426,10 +431,6 @@ def parse_args(): help='time to wait between lookups' ) - # Attempt to intelligently find the subdomain list depending on - # how this library was installed. - default_subdomain_file = find_subdomain_list_file('default.txt') - subdomain_group = p.add_mutually_exclusive_group() subdomain_group.add_argument( '--subdomains', @@ -440,8 +441,7 @@ def parse_args(): subdomain_group.add_argument( '--subdomain-file', action='store', - default=default_subdomain_file, - type=argparse.FileType('r'), + default='default.txt', help='use subdomains specified in this file (one per line)' ) @@ -455,12 +455,16 @@ def parse_args(): dns_group.add_argument( '--dns-file', action='store', - type=argparse.FileType('r'), help='use dns servers specified in this file for reverse lookups (one per line)' ) args = p.parse_args() + # Attempt to intelligently find the subdomain list depending on + # how this library was installed. + if args.subdomain_file and not os.path.exists(args.subdomain_file): + args.subdomain_file = find_subdomain_list_file(args.subdomain_file) + return args diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py index e81a1c5..cbcf32c 100644 --- a/tests/test_filesystem.py +++ b/tests/test_filesystem.py @@ -85,7 +85,7 @@ def test_update_resolver_nameservers_no_nameserver_use_file(self): result = fierce.update_resolver_nameservers( resolver, nameservers, - open(nameserver_filename) + nameserver_filename ) expected = contents.split() @@ -110,7 +110,7 @@ def test_update_resolver_nameservers_prefer_nameservers_over_file(self): result = fierce.update_resolver_nameservers( resolver, nameservers, - open(nameserver_filename) + nameserver_filename ) expected = nameservers @@ -168,7 +168,7 @@ def test_get_subdomains_no_subdomains_use_file(self): result = fierce.get_subdomains( subdomains, - open(subdomain_filename) + subdomain_filename ) expected = contents.split() @@ -190,7 +190,7 @@ def test_get_subdomains_prefer_subdomains_over_file(self): result = fierce.get_subdomains( subdomains, - open(subdomain_filename) + subdomain_filename ) expected = subdomains From 1a7c7c9ff88d67018c79cc182736b17d6f327c7f Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Thu, 1 Mar 2018 17:55:03 -0500 Subject: [PATCH 23/61] 1.2.1 release --- CHANGELOG.md | 13 +++++++++++++ setup.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71059fe..af317bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,19 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). This project adheres to [CHANGELOG](http://keepachangelog.com/). +## [1.2.1] - 2018-03-01 +### Changed +- Nearby IP reverse queries are now multithread, which improves performance significantly +- Updated development dependencies +- Subdomain lists use package_data instead of data_files + +### Added +- Gracefully handle users exiting the script with Ctrl+C +- Gracefully handle incorrect file or IP range arguments + +### Removed +- Official Python 3.3 support, it's EOL + ## [1.2.0] - 2017-05-07 ### Added - Official Python 3.6 support diff --git a/setup.py b/setup.py index b1a76e7..6d63a47 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ setup( name='fierce', - version='1.2.0', + version='1.2.1', description='A DNS reconnaissance tool for locating non-contiguous IP space.', url='https://github.com/mschwager/fierce', packages=['fierce'], From 61f87d525fcb6e83ebbb549132d8ebc834a3ac95 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Tue, 24 Apr 2018 14:20:31 -0400 Subject: [PATCH 24/61] Added tests for argument parsing --- fierce/fierce.py | 6 ++-- tests/test_fierce.py | 77 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index 58c9599..3f969dd 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -385,7 +385,7 @@ def fierce(**kwargs): time.sleep(kwargs["delay"]) -def parse_args(): +def parse_args(args): p = argparse.ArgumentParser(description=''' A DNS reconnaissance tool for locating non-contiguous IP space. ''', formatter_class=argparse.RawTextHelpFormatter) @@ -458,7 +458,7 @@ def parse_args(): help='use dns servers specified in this file for reverse lookups (one per line)' ) - args = p.parse_args() + args = p.parse_args(args) # Attempt to intelligently find the subdomain list depending on # how this library was installed. @@ -469,7 +469,7 @@ def parse_args(): def main(): - args = parse_args() + args = parse_args(sys.argv[1:]) try: fierce(**vars(args)) diff --git a/tests/test_fierce.py b/tests/test_fierce.py index 4de3543..1fd29a0 100644 --- a/tests/test_fierce.py +++ b/tests/test_fierce.py @@ -2,6 +2,7 @@ import ipaddress import io +import os import textwrap import unittest import unittest.mock @@ -10,6 +11,8 @@ import dns.name import dns.resolver +from pyfakefs import fake_filesystem_unittest + from fierce import fierce @@ -492,5 +495,79 @@ def test_search_filter_false(self): self.assertFalse(result) +class TestArgumentParsing(fake_filesystem_unittest.TestCase): + + def test_parse_args_basic(self): + domain = 'example.com' + + args = fierce.parse_args([ + '--domain', domain, + ]) + result = args.domain + expected = domain + + self.assertEqual(expected, result) + + def test_parse_args_included_list_file(self): + filename = '5000.txt' + + args = fierce.parse_args([ + '--domain', 'example.com', + '--subdomain-file', filename, + + ]) + result = args.subdomain_file + expected = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + 'fierce', + 'lists', + filename, + ) + exists = os.path.exists(result) + + self.assertEqual(expected, result) + self.assertTrue(exists) + + def test_parse_args_missing_list_file(self): + filename = 'missing.txt' + + args = fierce.parse_args([ + '--domain', 'example.com', + '--subdomain-file', filename, + + ]) + result = args.subdomain_file + expected = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + 'fierce', + 'lists', + filename, + ) + exists = os.path.exists(result) + + self.assertEqual(expected, result) + self.assertFalse(exists) + + def test_parse_args_custom_list_file(self): + self.setUpPyfakefs() + + filename = os.path.join('test', 'custom.txt') + self.fs.CreateFile( + filename, + contents='subdomain' + ) + + args = fierce.parse_args([ + '--domain', 'example.com', + '--subdomain-file', filename, + ]) + result = args.subdomain_file + expected = filename + exists = os.path.exists(result) + + self.assertEqual(expected, result) + self.assertTrue(exists) + + if __name__ == "__main__": unittest.main() From 57793066792f82c3cde7b58ea2991c54299f6e7d Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Tue, 24 Apr 2018 14:21:18 -0400 Subject: [PATCH 25/61] Updated coverage configuration option to latest INI key --- .coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 16c6d5c..9acf0db 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,2 +1,2 @@ -[run] +[report] include = fierce/fierce.py From 5dfcc069b800c200304b5cc0a84354a979ac2a61 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Tue, 24 Apr 2018 14:22:01 -0400 Subject: [PATCH 26/61] Include README.md in package long description using new content type --- setup.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/setup.py b/setup.py index 6d63a47..223f048 100644 --- a/setup.py +++ b/setup.py @@ -14,10 +14,18 @@ with open(requirements_filename) as fd: tests_require = [i.strip() for i in fd.readlines()] +long_description_filename = os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'README.md') + +with open(long_description_filename) as fd: + long_description = fd.read() + setup( name='fierce', version='1.2.1', description='A DNS reconnaissance tool for locating non-contiguous IP space.', + long_description=long_description, + long_description_content_type='text/markdown', url='https://github.com/mschwager/fierce', packages=['fierce'], package_dir={'fierce': 'fierce'}, From 3f54a2c01b88f58077817da98ae800b3a35e4e03 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Tue, 24 Apr 2018 14:53:19 -0400 Subject: [PATCH 27/61] Add Python 3 requirement to setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 223f048..273cd10 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,7 @@ ], install_requires=install_requires, tests_require=tests_require, + python_requires='>=3.0', entry_points={ 'console_scripts': [ 'fierce = fierce.fierce:main', From c14aed73b72cfb202fa1f675030811d110dc2742 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Tue, 24 Apr 2018 17:32:23 -0400 Subject: [PATCH 28/61] 1.2.2 release --- CHANGELOG.md | 5 +++++ setup.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af317bd..c130561 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). This project adheres to [CHANGELOG](http://keepachangelog.com/). +## [1.2.2] - 2018-04-24 +### Changed +- Python 3 is now a requirement when installing via setup.py (including pip) +- The README markdown is now included in the package's long description + ## [1.2.1] - 2018-03-01 ### Changed - Nearby IP reverse queries are now multithread, which improves performance significantly diff --git a/setup.py b/setup.py index 273cd10..893d234 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ setup( name='fierce', - version='1.2.1', + version='1.2.2', description='A DNS reconnaissance tool for locating non-contiguous IP space.', long_description=long_description, long_description_content_type='text/markdown', From 2ace50a6f8f6b67a24c53ee46a4655051001548b Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Thu, 18 Apr 2019 09:10:10 -0400 Subject: [PATCH 29/61] Filter out subdomains matching wildcard records; print all wildcard records --- CHANGELOG.md | 7 +++++++ fierce/fierce.py | 9 +++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c130561..f9ceb64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,13 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). This project adheres to [CHANGELOG](http://keepachangelog.com/). +## [Unreleased] +### Changed +- Print out all A records for wildcard, not just first one + +### Added +- Filter out subdomains with an A record matching a wildcard A record + ## [1.2.2] - 2018-04-24 ### Changed - Python 3 is now a requirement when installing via setup.py (including pip) diff --git a/fierce/fierce.py b/fierce/fierce.py index 3f969dd..216bbac 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -333,7 +333,8 @@ def fierce(**kwargs): random_subdomain = str(random.randint(1e10, 1e11)) random_domain = concatenate_subdomains(domain, [random_subdomain]) wildcard = query(resolver, random_domain, record_type='A') - print("Wildcard: {}".format(wildcard[0].address if wildcard else "failure")) + wildcard_ips = set([rr.address for rr in wildcard.rrset]) if wildcard else None + print("Wildcard: {}".format(', '.join(wildcard_ips) if wildcard_ips else "failure")) subdomains = get_subdomains( kwargs["subdomains"], @@ -359,7 +360,11 @@ def fierce(**kwargs): if record is None or record.rrset is None: continue - ip = ipaddress.IPv4Address(record[0].address) + ips = [rr.address for rr in record.rrset] + if len(wildcard_ips.intersection(ips)) != 0: + continue + + ip = ipaddress.IPv4Address(ips[0]) http_connection_headers = None if kwargs.get('connect') and not ip.is_private: From 5abaa704a0458247ac31007384886f765688e9ef Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Fri, 19 Apr 2019 09:09:51 -0400 Subject: [PATCH 30/61] Don't break if no wildcard; only skip when sub matches entire wildcard set Also a little clean up using a generator --- fierce/fierce.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index 216bbac..62dab82 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -333,7 +333,7 @@ def fierce(**kwargs): random_subdomain = str(random.randint(1e10, 1e11)) random_domain = concatenate_subdomains(domain, [random_subdomain]) wildcard = query(resolver, random_domain, record_type='A') - wildcard_ips = set([rr.address for rr in wildcard.rrset]) if wildcard else None + wildcard_ips = set(rr.address for rr in wildcard.rrset) if wildcard else set() print("Wildcard: {}".format(', '.join(wildcard_ips) if wildcard_ips else "failure")) subdomains = get_subdomains( @@ -361,7 +361,7 @@ def fierce(**kwargs): continue ips = [rr.address for rr in record.rrset] - if len(wildcard_ips.intersection(ips)) != 0: + if wildcard_ips == set(ips): continue ip = ipaddress.IPv4Address(ips[0]) From aab35b296564827e964bfbe3dae0b6863a98574a Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Tue, 23 Apr 2019 08:08:18 -0600 Subject: [PATCH 31/61] Added Python 3.7 support --- .travis.yml | 2 ++ CHANGELOG.md | 1 + setup.py | 1 + 3 files changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 23ea2b8..3eba616 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,10 @@ +dist: xenial language: python python: - 3.4 - 3.5 - 3.6 + - 3.7 install: - pip install -r requirements.txt - pip install -r requirements-dev.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index f9ceb64..467fc21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ This project adheres to [CHANGELOG](http://keepachangelog.com/). ### Added - Filter out subdomains with an A record matching a wildcard A record +- Official Python 3.7 support ## [1.2.2] - 2018-04-24 ### Changed diff --git a/setup.py b/setup.py index 893d234..8306541 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Security', ], install_requires=install_requires, From 4e7364b55bdcff016d8fbd44292dfa13bcffca61 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Tue, 23 Apr 2019 08:14:43 -0600 Subject: [PATCH 32/61] Use flake8 configuration file and update dev requirements --- .flake8 | 3 +++ .travis.yml | 2 +- requirements-dev.txt | 8 ++++---- tests/test_fierce.py | 2 +- tests/test_filesystem.py | 8 ++++---- 5 files changed, 13 insertions(+), 10 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..5e2d450 --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +ignore = E501 +include = fierce,tests diff --git a/.travis.yml b/.travis.yml index 3eba616..56bf5e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ install: - pip install -r requirements.txt - pip install -r requirements-dev.txt script: - - flake8 --ignore=E501 fierce/ tests/ + - flake8 - nose2 --with-coverage after_success: - coveralls diff --git a/requirements-dev.txt b/requirements-dev.txt index bf1bb3a..0ff9920 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ -nose2==0.7.3 -flake8==3.2.1 -coveralls==1.2.0 -pyfakefs==3.3 +nose2==0.9.1 +flake8==3.7.7 +coveralls==1.7.0 +pyfakefs==3.5.8 diff --git a/tests/test_fierce.py b/tests/test_fierce.py index 1fd29a0..b5c0736 100644 --- a/tests/test_fierce.py +++ b/tests/test_fierce.py @@ -552,7 +552,7 @@ def test_parse_args_custom_list_file(self): self.setUpPyfakefs() filename = os.path.join('test', 'custom.txt') - self.fs.CreateFile( + self.fs.create_file( filename, contents='subdomain' ) diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py index cbcf32c..b0a5081 100644 --- a/tests/test_filesystem.py +++ b/tests/test_filesystem.py @@ -75,7 +75,7 @@ def test_update_resolver_nameservers_no_nameserver_use_file(self): nameserver3 """.strip()) - self.fs.CreateFile( + self.fs.create_file( nameserver_filename, contents=contents ) @@ -100,7 +100,7 @@ def test_update_resolver_nameservers_prefer_nameservers_over_file(self): nameserver3 """.strip()) - self.fs.CreateFile( + self.fs.create_file( nameserver_filename, contents=contents ) @@ -161,7 +161,7 @@ def test_get_subdomains_no_subdomains_use_file(self): sd3.domain.com """.strip()) - self.fs.CreateFile( + self.fs.create_file( subdomain_filename, contents=contents ) @@ -183,7 +183,7 @@ def test_get_subdomains_prefer_subdomains_over_file(self): sd3.domain.com """.strip()) - self.fs.CreateFile( + self.fs.create_file( subdomain_filename, contents=contents ) From dd97641758f349725c08bb2ae99735d8c8c0cc58 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Wed, 24 Apr 2019 15:56:01 -0400 Subject: [PATCH 33/61] Prevent out of bounds error when traversing near 0.0.0.0 (fixes #29) --- fierce/fierce.py | 10 +++++----- tests/test_fierce.py | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index 62dab82..354b55f 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -168,12 +168,12 @@ def default_expander(ip): def traverse_expander(ip, n=5): - class_c = get_class_c_network(ip) - - result = [ipaddress.IPv4Address(ip + i) for i in range(-n, n + 1)] - result = [i for i in result if i in class_c] + ip = int(ip) + class_c_floor = ip - (ip % 256) - return result + ip_min = max(ip - n, class_c_floor) + ip_max = min(ip + n, class_c_floor + 255) + return [ipaddress.IPv4Address(i) for i in range(ip_min, ip_max + 1)] def wide_expander(ip): diff --git a/tests/test_fierce.py b/tests/test_fierce.py index b5c0736..693815e 100644 --- a/tests/test_fierce.py +++ b/tests/test_fierce.py @@ -27,6 +27,28 @@ def to_text(self): return self.response +class TestTraverseExpander(unittest.TestCase): + + def test_stay_in_class_C(self): + result = fierce.traverse_expander(ipaddress.IPv4Address('127.0.0.1')) + expected = [ipaddress.IPv4Address("127.0.0.%s" % i) for i in range(0, 7)] + self.assertEqual(expected, result) + + # Upper and lower bound tests are to avoid reintroducing out of + # bounds error from IPv4Address. (Class C test won't necessarily + # cover this.) + + def test_lower_bound(self): + result = fierce.traverse_expander(ipaddress.IPv4Address('0.0.0.2'), n=5) + expected = [ipaddress.IPv4Address(i) for i in range(0, 8)] + self.assertEqual(expected, result) + + def test_upper_bound(self): + result = fierce.traverse_expander(ipaddress.IPv4Address('255.255.255.254'), n=5) + expected = [ipaddress.IPv4Address("255.255.255.%s" % i) for i in range(249, 256)] + self.assertEqual(expected, result) + + class TestFierce(unittest.TestCase): def test_concatenate_subdomains_empty(self): From 5354d073468ff00fa3833994bb07b40a8c7f1156 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Fri, 26 Apr 2019 07:06:07 -0600 Subject: [PATCH 34/61] Update to latest version of dnspython --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a5d6526..2214373 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -dnspython==1.15.0 +dnspython==1.16.0 From 6df762d589d48c92bfa9ff48b592f28ca133f4bd Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Sun, 28 Apr 2019 07:25:59 -0400 Subject: [PATCH 35/61] Tweaks from review feedback (changelog; combine tests; extract variable) --- CHANGELOG.md | 4 ++++ fierce/fierce.py | 3 ++- tests/test_fierce.py | 54 +++++++++++++++++++++++++------------------- 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 467fc21..17c3bc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ This project adheres to [CHANGELOG](http://keepachangelog.com/). - Filter out subdomains with an A record matching a wildcard A record - Official Python 3.7 support +### Fixed +- Prevent out of bounds error when expanding IPs near 0.0.0.0 or + 255.255.255.255 + ## [1.2.2] - 2018-04-24 ### Changed - Python 3 is now a requirement when installing via setup.py (including pip) diff --git a/fierce/fierce.py b/fierce/fierce.py index 354b55f..fbc1ae0 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -170,9 +170,10 @@ def default_expander(ip): def traverse_expander(ip, n=5): ip = int(ip) class_c_floor = ip - (ip % 256) + class_c_ceiling = class_c_floor + 255 ip_min = max(ip - n, class_c_floor) - ip_max = min(ip + n, class_c_floor + 255) + ip_max = min(ip + n, class_c_ceiling) return [ipaddress.IPv4Address(i) for i in range(ip_min, ip_max + 1)] diff --git a/tests/test_fierce.py b/tests/test_fierce.py index 693815e..67505b9 100644 --- a/tests/test_fierce.py +++ b/tests/test_fierce.py @@ -26,29 +26,6 @@ def __init__(self, response): def to_text(self): return self.response - -class TestTraverseExpander(unittest.TestCase): - - def test_stay_in_class_C(self): - result = fierce.traverse_expander(ipaddress.IPv4Address('127.0.0.1')) - expected = [ipaddress.IPv4Address("127.0.0.%s" % i) for i in range(0, 7)] - self.assertEqual(expected, result) - - # Upper and lower bound tests are to avoid reintroducing out of - # bounds error from IPv4Address. (Class C test won't necessarily - # cover this.) - - def test_lower_bound(self): - result = fierce.traverse_expander(ipaddress.IPv4Address('0.0.0.2'), n=5) - expected = [ipaddress.IPv4Address(i) for i in range(0, 8)] - self.assertEqual(expected, result) - - def test_upper_bound(self): - result = fierce.traverse_expander(ipaddress.IPv4Address('255.255.255.254'), n=5) - expected = [ipaddress.IPv4Address("255.255.255.%s" % i) for i in range(249, 256)] - self.assertEqual(expected, result) - - class TestFierce(unittest.TestCase): def test_concatenate_subdomains_empty(self): @@ -166,6 +143,37 @@ def test_traverse_expander_no_cross_upper_boundary(self): self.assertEqual(expected, result) + # Upper and lower bound tests are to avoid reintroducing out of + # bounds error from IPv4Address. (no_cross_*_boundary tests won't + # necessarily cover this; GitHub issue #29) + + def test_traverse_expander_lower_bound_regression(self): + ip = ipaddress.IPv4Address('0.0.0.1') + expand = 2 + + result = fierce.traverse_expander(ip, expand) + expected = [ + ipaddress.IPv4Address('0.0.0.0'), + ipaddress.IPv4Address('0.0.0.1'), + ipaddress.IPv4Address('0.0.0.2'), + ipaddress.IPv4Address('0.0.0.3') + ] + self.assertEqual(expected, result) + + def test_traverse_expander_upper_bound_regression(self): + ip = ipaddress.IPv4Address('255.255.255.254') + expand = 2 + + result = fierce.traverse_expander(ip, expand) + expected = [ + ipaddress.IPv4Address('255.255.255.252'), + ipaddress.IPv4Address('255.255.255.253'), + ipaddress.IPv4Address('255.255.255.254'), + ipaddress.IPv4Address('255.255.255.255') + ] + self.assertEqual(expected, result) + + def test_wide_expander_basic(self): ip = ipaddress.IPv4Address('192.168.1.50') From 76d0f7baa6e8a1d576d748ec24d4bf1fd8433bcd Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Sun, 28 Apr 2019 07:35:58 -0400 Subject: [PATCH 36/61] Fix whitespace to satisfy flake8 (I hope) --- tests/test_fierce.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_fierce.py b/tests/test_fierce.py index 67505b9..d09efb5 100644 --- a/tests/test_fierce.py +++ b/tests/test_fierce.py @@ -26,6 +26,7 @@ def __init__(self, response): def to_text(self): return self.response + class TestFierce(unittest.TestCase): def test_concatenate_subdomains_empty(self): @@ -173,7 +174,6 @@ def test_traverse_expander_upper_bound_regression(self): ] self.assertEqual(expected, result) - def test_wide_expander_basic(self): ip = ipaddress.IPv4Address('192.168.1.50') From fbb58c8fab2711fcfe41c334baa5f4dc78973453 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Wed, 15 May 2019 08:37:41 -0600 Subject: [PATCH 37/61] 1.3.0 release --- CHANGELOG.md | 2 ++ setup.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17c3bc8..285c34c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). This project adheres to [CHANGELOG](http://keepachangelog.com/). ## [Unreleased] + +## [1.3.0] - 2019-05-15 ### Changed - Print out all A records for wildcard, not just first one diff --git a/setup.py b/setup.py index 8306541..1aefb2e 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ setup( name='fierce', - version='1.2.2', + version='1.3.0', description='A DNS reconnaissance tool for locating non-contiguous IP space.', long_description=long_description, long_description_content_type='text/markdown', From 5a2514b378eef6d5266fbc48bbc229f91cfc5153 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Wed, 6 Nov 2019 09:43:25 -0700 Subject: [PATCH 38/61] Add Python 3.8 support, drop 3.4 --- .travis.yml | 3 ++- CHANGELOG.md | 5 +++++ setup.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 56bf5e1..b38da5d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,14 @@ dist: xenial language: python python: - - 3.4 - 3.5 - 3.6 - 3.7 + - 3.8-dev install: - pip install -r requirements.txt - pip install -r requirements-dev.txt + - pip install -e . script: - flake8 - nose2 --with-coverage diff --git a/CHANGELOG.md b/CHANGELOG.md index 285c34c..9c242e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ This project adheres to [Semantic Versioning](http://semver.org/). This project adheres to [CHANGELOG](http://keepachangelog.com/). ## [Unreleased] +### Added +- Official Python 3.8 support + +### Removed +- Official Python 3.4 support, it's EOL ## [1.3.0] - 2019-05-15 ### Changed diff --git a/setup.py b/setup.py index 1aefb2e..5fe2d36 100644 --- a/setup.py +++ b/setup.py @@ -38,10 +38,10 @@ 'Operating System :: MacOS :: MacOS X', 'Operating System :: Microsoft :: Windows', 'Operating System :: POSIX', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Topic :: Security', ], install_requires=install_requires, From 8d3b0db946eb38cd80af684858a8e1430543a265 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Wed, 6 Nov 2019 09:45:14 -0700 Subject: [PATCH 39/61] Minor README changes --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0de3b44..d7834a7 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ The original description was very apt, so I'll include it here: # Installing ``` -$ pip3 install fierce +$ python -m pip install fierce $ fierce -h ``` @@ -49,10 +49,12 @@ OR ``` $ git clone https://github.com/mschwager/fierce.git $ cd fierce -$ pip3 install -r requirements.txt -$ python3 fierce/fierce.py -h +$ python -m pip install -r requirements.txt +$ python fierce/fierce.py -h ``` +*Requires Python 3.* + # Using Let's start with something basic: From 3248030b72df9cff3bb041345ea5003b64082a7c Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Wed, 6 Nov 2019 09:56:21 -0700 Subject: [PATCH 40/61] Convert tests to pytest --- .travis.yml | 2 +- requirements-dev.txt | 3 +- tests/test_fierce.py | 94 ++++++++++++++++++++-------------------- tests/test_filesystem.py | 20 ++++----- 4 files changed, 60 insertions(+), 59 deletions(-) diff --git a/.travis.yml b/.travis.yml index b38da5d..3da11ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,6 @@ install: - pip install -e . script: - flake8 - - nose2 --with-coverage + - pytest --cov after_success: - coveralls diff --git a/requirements-dev.txt b/requirements-dev.txt index 0ff9920..c3b25a9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,5 @@ -nose2==0.9.1 +pytest==5.2.2 +pytest-cov==2.8.1 flake8==3.7.7 coveralls==1.7.0 pyfakefs==3.5.8 diff --git a/tests/test_fierce.py b/tests/test_fierce.py index d09efb5..bb3219f 100644 --- a/tests/test_fierce.py +++ b/tests/test_fierce.py @@ -36,7 +36,7 @@ def test_concatenate_subdomains_empty(self): result = fierce.concatenate_subdomains(domain, subdomains) expected = dns.name.from_text("example.com.") - self.assertEqual(expected, result) + assert expected == result def test_concatenate_subdomains_single_subdomain(self): domain = dns.name.from_text("example.com.") @@ -45,7 +45,7 @@ def test_concatenate_subdomains_single_subdomain(self): result = fierce.concatenate_subdomains(domain, subdomains) expected = dns.name.from_text("sd1.example.com.") - self.assertEqual(expected, result) + assert expected == result def test_concatenate_subdomains_multiple_subdomains(self): domain = dns.name.from_text("example.com.") @@ -54,7 +54,7 @@ def test_concatenate_subdomains_multiple_subdomains(self): result = fierce.concatenate_subdomains(domain, subdomains) expected = dns.name.from_text("sd1.sd2.example.com.") - self.assertEqual(expected, result) + assert expected == result def test_concatenate_subdomains_makes_root(self): # Domain is missing '.' at the end @@ -64,7 +64,7 @@ def test_concatenate_subdomains_makes_root(self): result = fierce.concatenate_subdomains(domain, subdomains) expected = dns.name.from_text("example.com.") - self.assertEqual(expected, result) + assert expected == result def test_concatenate_subdomains_single_sub_subdomain(self): domain = dns.name.from_text("example.com.") @@ -73,7 +73,7 @@ def test_concatenate_subdomains_single_sub_subdomain(self): result = fierce.concatenate_subdomains(domain, subdomains) expected = dns.name.from_text("sd1.sd2.example.com.") - self.assertEqual(expected, result) + assert expected == result def test_concatenate_subdomains_multiple_sub_subdomain(self): domain = dns.name.from_text("example.com.") @@ -82,7 +82,7 @@ def test_concatenate_subdomains_multiple_sub_subdomain(self): result = fierce.concatenate_subdomains(domain, subdomains) expected = dns.name.from_text("sd1.sd2.sd3.sd4.example.com.") - self.assertEqual(expected, result) + assert expected == result def test_concatenate_subdomains_fqdn_subdomain(self): domain = dns.name.from_text("example.") @@ -91,7 +91,7 @@ def test_concatenate_subdomains_fqdn_subdomain(self): result = fierce.concatenate_subdomains(domain, subdomains) expected = dns.name.from_text("sd1.sd2.example.") - self.assertEqual(expected, result) + assert expected == result def test_default_expander(self): ip = ipaddress.IPv4Address('192.168.1.1') @@ -101,7 +101,7 @@ def test_default_expander(self): ipaddress.IPv4Address('192.168.1.1'), ] - self.assertEqual(expected, result) + assert expected == result def test_traverse_expander_basic(self): ip = ipaddress.IPv4Address('192.168.1.1') @@ -114,7 +114,7 @@ def test_traverse_expander_basic(self): ipaddress.IPv4Address('192.168.1.2'), ] - self.assertEqual(expected, result) + assert expected == result def test_traverse_expander_no_cross_lower_boundary(self): ip = ipaddress.IPv4Address('192.168.1.1') @@ -128,7 +128,7 @@ def test_traverse_expander_no_cross_lower_boundary(self): ipaddress.IPv4Address('192.168.1.3'), ] - self.assertEqual(expected, result) + assert expected == result def test_traverse_expander_no_cross_upper_boundary(self): ip = ipaddress.IPv4Address('192.168.1.254') @@ -142,7 +142,7 @@ def test_traverse_expander_no_cross_upper_boundary(self): ipaddress.IPv4Address('192.168.1.255'), ] - self.assertEqual(expected, result) + assert expected == result # Upper and lower bound tests are to avoid reintroducing out of # bounds error from IPv4Address. (no_cross_*_boundary tests won't @@ -159,7 +159,7 @@ def test_traverse_expander_lower_bound_regression(self): ipaddress.IPv4Address('0.0.0.2'), ipaddress.IPv4Address('0.0.0.3') ] - self.assertEqual(expected, result) + assert expected == result def test_traverse_expander_upper_bound_regression(self): ip = ipaddress.IPv4Address('255.255.255.254') @@ -172,7 +172,7 @@ def test_traverse_expander_upper_bound_regression(self): ipaddress.IPv4Address('255.255.255.254'), ipaddress.IPv4Address('255.255.255.255') ] - self.assertEqual(expected, result) + assert expected == result def test_wide_expander_basic(self): ip = ipaddress.IPv4Address('192.168.1.50') @@ -184,7 +184,7 @@ def test_wide_expander_basic(self): for i in range(256) ] - self.assertEqual(expected, result) + assert expected == result def test_wide_expander_lower_boundary(self): ip = ipaddress.IPv4Address('192.168.1.0') @@ -196,7 +196,7 @@ def test_wide_expander_lower_boundary(self): for i in range(256) ] - self.assertEqual(expected, result) + assert expected == result def test_wide_expander_upper_boundary(self): ip = ipaddress.IPv4Address('192.168.1.255') @@ -208,7 +208,7 @@ def test_wide_expander_upper_boundary(self): for i in range(256) ] - self.assertEqual(expected, result) + assert expected == result def test_range_expander(self): ip = '192.168.1.0/31' @@ -220,7 +220,7 @@ def test_range_expander(self): ipaddress.IPv4Address('192.168.1.1'), ] - self.assertEqual(expected, result) + assert expected == result def test_recursive_query_basic_failure(self): resolver = dns.resolver.Resolver() @@ -237,7 +237,7 @@ def test_recursive_query_basic_failure(self): ] mock_method.assert_has_calls(expected) - self.assertIsNone(result) + assert result is None def test_recursive_query_long_domain_failure(self): resolver = dns.resolver.Resolver() @@ -256,7 +256,7 @@ def test_recursive_query_long_domain_failure(self): ] mock_method.assert_has_calls(expected) - self.assertIsNone(result) + assert result is None def test_recursive_query_basic_success(self): resolver = dns.resolver.Resolver() @@ -278,7 +278,7 @@ def test_recursive_query_basic_success(self): ] mock_method.assert_has_calls(expected) - self.assertEqual(result, good_response) + assert result == good_response def test_query_nxdomain(self): resolver = dns.resolver.Resolver() @@ -287,7 +287,7 @@ def test_query_nxdomain(self): with unittest.mock.patch.object(resolver, 'query', side_effect=dns.resolver.NXDOMAIN()): result = fierce.query(resolver, domain) - self.assertIsNone(result) + assert result is None def test_query_no_nameservers(self): resolver = dns.resolver.Resolver() @@ -296,7 +296,7 @@ def test_query_no_nameservers(self): with unittest.mock.patch.object(resolver, 'query', side_effect=dns.resolver.NoNameservers()): result = fierce.query(resolver, domain) - self.assertIsNone(result) + assert result is None def test_query_timeout(self): resolver = dns.resolver.Resolver() @@ -305,7 +305,7 @@ def test_query_timeout(self): with unittest.mock.patch.object(resolver, 'query', side_effect=dns.exception.Timeout()): result = fierce.query(resolver, domain) - self.assertIsNone(result) + assert result is None def test_zone_transfer_connection_error(self): address = 'test' @@ -314,7 +314,7 @@ def test_zone_transfer_connection_error(self): with unittest.mock.patch.object(fierce.dns.zone, 'from_xfr', side_effect=ConnectionError()): result = fierce.zone_transfer(address, domain) - self.assertIsNone(result) + assert result is None def test_zone_transfer_eof_error(self): address = 'test' @@ -323,7 +323,7 @@ def test_zone_transfer_eof_error(self): with unittest.mock.patch.object(fierce.dns.zone, 'from_xfr', side_effect=EOFError()): result = fierce.zone_transfer(address, domain) - self.assertIsNone(result) + assert result is None def test_zone_transfer_timeout_error(self): address = 'test' @@ -332,7 +332,7 @@ def test_zone_transfer_timeout_error(self): with unittest.mock.patch.object(fierce.dns.zone, 'from_xfr', side_effect=TimeoutError()): result = fierce.zone_transfer(address, domain) - self.assertIsNone(result) + assert result is None def test_zone_transfer_form_error(self): address = 'test' @@ -341,7 +341,7 @@ def test_zone_transfer_form_error(self): with unittest.mock.patch.object(fierce.dns.zone, 'from_xfr', side_effect=dns.exception.FormError()): result = fierce.zone_transfer(address, domain) - self.assertIsNone(result) + assert result is None def test_find_nearby_empty(self): resolver = 'unused' @@ -350,7 +350,7 @@ def test_find_nearby_empty(self): result = fierce.find_nearby(resolver, ips) expected = {} - self.assertEqual(expected, result) + assert expected == result def test_find_nearby_basic(self): resolver = 'unused' @@ -371,7 +371,7 @@ def test_find_nearby_basic(self): '192.168.1.1': 'sd2.example.com.', } - self.assertEqual(expected, result) + assert expected == result def test_find_nearby_filter_func(self): resolver = 'unused' @@ -394,7 +394,7 @@ def filter_func(reverse_result): '192.168.1.0': 'sd1.example.com.', } - self.assertEqual(expected, result) + assert expected == result def test_print_subdomain_result_basic(self): url = 'example.com' @@ -406,7 +406,7 @@ def test_print_subdomain_result_basic(self): expected = 'Found: example.com (192.168.1.0)\n' - self.assertEqual(expected, result) + assert expected == result def test_print_subdomain_result_nearby(self): url = 'example.com' @@ -423,7 +423,7 @@ def test_print_subdomain_result_nearby(self): {'192.168.1.1': 'nearby.com'} ''').lstrip() - self.assertEqual(expected, result) + assert expected == result def test_print_subdomain_result_http_header(self): url = 'example.com' @@ -445,7 +445,7 @@ def test_print_subdomain_result_http_header(self): {'HTTP HEADER': 'value'} ''').lstrip() - self.assertEqual(expected, result) + assert expected == result def test_print_subdomain_result_both(self): url = 'example.com' @@ -471,7 +471,7 @@ def test_print_subdomain_result_both(self): {'192.168.1.1': 'nearby.com'} ''').lstrip() - self.assertEqual(expected, result) + assert expected == result def test_unvisited_closure_empty(self): unvisited = fierce.unvisited_closure() @@ -480,7 +480,7 @@ def test_unvisited_closure_empty(self): result = unvisited(ips) expected = set() - self.assertEqual(expected, result) + assert expected == result def test_unvisited_closure_empty_intersection(self): unvisited = fierce.unvisited_closure() @@ -489,7 +489,7 @@ def test_unvisited_closure_empty_intersection(self): result = unvisited(set([4, 5, 6])) expected = set([4, 5, 6]) - self.assertEqual(expected, result) + assert expected == result def test_unvisited_closure_overlapping_intersection(self): unvisited = fierce.unvisited_closure() @@ -498,7 +498,7 @@ def test_unvisited_closure_overlapping_intersection(self): result = unvisited(set([2, 3, 4])) expected = set([4]) - self.assertEqual(expected, result) + assert expected == result def test_search_filter_empty(self): domains = [] @@ -506,7 +506,7 @@ def test_search_filter_empty(self): result = fierce.search_filter(domains, address) - self.assertFalse(result) + assert not result def test_search_filter_true(self): domains = ['example.com'] @@ -514,7 +514,7 @@ def test_search_filter_true(self): result = fierce.search_filter(domains, address) - self.assertTrue(result) + assert result def test_search_filter_false(self): domains = ['not.com'] @@ -522,7 +522,7 @@ def test_search_filter_false(self): result = fierce.search_filter(domains, address) - self.assertFalse(result) + assert not result class TestArgumentParsing(fake_filesystem_unittest.TestCase): @@ -536,7 +536,7 @@ def test_parse_args_basic(self): result = args.domain expected = domain - self.assertEqual(expected, result) + assert expected == result def test_parse_args_included_list_file(self): filename = '5000.txt' @@ -555,8 +555,8 @@ def test_parse_args_included_list_file(self): ) exists = os.path.exists(result) - self.assertEqual(expected, result) - self.assertTrue(exists) + assert expected == result + assert exists def test_parse_args_missing_list_file(self): filename = 'missing.txt' @@ -575,8 +575,8 @@ def test_parse_args_missing_list_file(self): ) exists = os.path.exists(result) - self.assertEqual(expected, result) - self.assertFalse(exists) + assert expected == result + assert not exists def test_parse_args_custom_list_file(self): self.setUpPyfakefs() @@ -595,8 +595,8 @@ def test_parse_args_custom_list_file(self): expected = filename exists = os.path.exists(result) - self.assertEqual(expected, result) - self.assertTrue(exists) + assert expected == result + assert exists if __name__ == "__main__": diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py index b0a5081..20ae7b1 100644 --- a/tests/test_filesystem.py +++ b/tests/test_filesystem.py @@ -34,7 +34,7 @@ def test_update_resolver_nameservers_empty_no_file(self): nameserver_filename ) - self.assertEqual(expected, result.nameservers) + assert expected == result.nameservers def test_update_resolver_nameservers_single_nameserver_no_file(self): nameserver_filename = None @@ -49,7 +49,7 @@ def test_update_resolver_nameservers_single_nameserver_no_file(self): ) expected = nameservers - self.assertEqual(expected, result.nameservers) + assert expected == result.nameservers def test_update_resolver_nameservers_multiple_nameservers_no_file(self): nameserver_filename = None @@ -64,7 +64,7 @@ def test_update_resolver_nameservers_multiple_nameservers_no_file(self): ) expected = nameservers - self.assertEqual(expected, result.nameservers) + assert expected == result.nameservers def test_update_resolver_nameservers_no_nameserver_use_file(self): nameserver_filename = os.path.join("directory", "nameservers") @@ -89,7 +89,7 @@ def test_update_resolver_nameservers_no_nameserver_use_file(self): ) expected = contents.split() - self.assertEqual(expected, result.nameservers) + assert expected == result.nameservers def test_update_resolver_nameservers_prefer_nameservers_over_file(self): nameserver_filename = os.path.join("directory", "nameservers") @@ -114,7 +114,7 @@ def test_update_resolver_nameservers_prefer_nameservers_over_file(self): ) expected = nameservers - self.assertEqual(expected, result.nameservers) + assert expected == result.nameservers def test_get_subdomains_empty_no_file(self): subdomain_filename = None @@ -126,7 +126,7 @@ def test_get_subdomains_empty_no_file(self): ) expected = subdomains - self.assertEqual(expected, result) + assert expected == result def test_get_subdomains_single_subdomain_no_file(self): subdomain_filename = None @@ -138,7 +138,7 @@ def test_get_subdomains_single_subdomain_no_file(self): ) expected = subdomains - self.assertEqual(expected, result) + assert expected == result def test_get_subdomains_multiple_subdomains_no_file(self): subdomain_filename = None @@ -150,7 +150,7 @@ def test_get_subdomains_multiple_subdomains_no_file(self): ) expected = subdomains - self.assertEqual(expected, result) + assert expected == result def test_get_subdomains_no_subdomains_use_file(self): subdomain_filename = os.path.join("directory", "subdomains") @@ -172,7 +172,7 @@ def test_get_subdomains_no_subdomains_use_file(self): ) expected = contents.split() - self.assertEqual(expected, result) + assert expected == result def test_get_subdomains_prefer_subdomains_over_file(self): subdomain_filename = os.path.join("directory", "subdomains") @@ -194,7 +194,7 @@ def test_get_subdomains_prefer_subdomains_over_file(self): ) expected = subdomains - self.assertEqual(expected, result) + assert expected == result if __name__ == "__main__": From 335b28a47eab98aea4aecddbd4587e4df918daa3 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Wed, 6 Nov 2019 09:57:56 -0700 Subject: [PATCH 41/61] Remove README TODO --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index d7834a7..3988ec4 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,3 @@ Check out `--help` for further information: ``` $ fierce --help ``` - -# TODO - -* Provide multiprocessing capabilities From c5a3e15f3b64eeb738e5d04d6010ac11f46c53ff Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Wed, 6 Nov 2019 09:59:07 -0700 Subject: [PATCH 42/61] Add developing information to README --- README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/README.md b/README.md index 3988ec4..58c8640 100644 --- a/README.md +++ b/README.md @@ -116,3 +116,31 @@ Check out `--help` for further information: ``` $ fierce --help ``` + +# Developing + +First, install development packages: + +``` +$ python -m pip install -r requirements.txt +$ python -m pip install -r requirements-dev.txt +$ python -m pip install -e . +``` + +## Testing + +``` +$ pytest +``` + +## Linting + +``` +$ flake8 +``` + +## Coverage + +``` +$ pytest --cov +``` From 9edcf3148249c9695bc115a3d775a4e48b96d553 Mon Sep 17 00:00:00 2001 From: takeshix Date: Wed, 6 Nov 2019 15:06:21 +0100 Subject: [PATCH 43/61] Added support for DNS over TCP --- fierce/fierce.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index fbc1ae0..8d3b612 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -108,9 +108,9 @@ def concatenate_subdomains(domain, subdomains): return result -def query(resolver, domain, record_type='A'): +def query(resolver, domain, record_type='A', use_tcp=False): try: - resp = resolver.query(domain, record_type, raise_on_no_answer=False) + resp = resolver.query(domain, record_type, raise_on_no_answer=False, tcp=use_tcp) if resp.response.answer: return resp @@ -124,23 +124,23 @@ def query(resolver, domain, record_type='A'): for rdata in additionals.items ] resolver.nameservers = ns - return query(resolver, domain, record_type) + return query(resolver, domain, record_type, use_tcp=use_tcp) return None except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.exception.Timeout): return None -def reverse_query(resolver, ip): - return query(resolver, dns.reversename.from_address(ip), record_type='PTR') +def reverse_query(resolver, ip, use_tcp=False): + return query(resolver, dns.reversename.from_address(ip), record_type='PTR', use_tcp=use_tcp) -def recursive_query(resolver, domain, record_type='NS'): +def recursive_query(resolver, domain, record_type='NS', use_tcp=False): query_domain = str(domain) query_response = None try: while query_response is None: - query_response = query(resolver, query_domain, record_type) + query_response = query(resolver, query_domain, record_type, use_tcp=use_tcp) query_domain = query_domain.split('.', 1)[1] except IndexError: return None @@ -306,7 +306,7 @@ def fierce(**kwargs): if not domain.is_absolute(): domain = domain.concatenate(dns.name.root) - ns = recursive_query(resolver, domain, 'NS') + ns = recursive_query(resolver, domain, 'NS', use_tcp=kwargs.get("tcp")) if ns: domain_name_servers = [n.to_text() for n in ns] @@ -315,10 +315,10 @@ def fierce(**kwargs): print("NS: {}".format(" ".join(domain_name_servers) if ns else "failure")) - soa = recursive_query(resolver, domain, record_type='SOA') + soa = recursive_query(resolver, domain, record_type='SOA', use_tcp=kwargs.get("tcp")) if soa: soa_mname = soa[0].mname - master = query(resolver, soa_mname, record_type='A') + master = query(resolver, soa_mname, record_type='A', use_tcp=kwargs.get("tcp")) master_address = master[0].address print("SOA: {} ({})".format(soa_mname, master_address)) else: @@ -333,7 +333,7 @@ def fierce(**kwargs): random_subdomain = str(random.randint(1e10, 1e11)) random_domain = concatenate_subdomains(domain, [random_subdomain]) - wildcard = query(resolver, random_domain, record_type='A') + wildcard = query(resolver, random_domain, record_type='A', use_tcp=kwargs.get("tcp")) wildcard_ips = set(rr.address for rr in wildcard.rrset) if wildcard else set() print("Wildcard: {}".format(', '.join(wildcard_ips) if wildcard_ips else "failure")) @@ -356,7 +356,7 @@ def fierce(**kwargs): for subdomain in subdomains: url = concatenate_subdomains(domain, [subdomain]) - record = query(resolver, url, record_type='A') + record = query(resolver, url, record_type='A', use_tcp=kwargs.get("tcp")) if record is None or record.rrset is None: continue @@ -463,6 +463,12 @@ def parse_args(args): action='store', help='use dns servers specified in this file for reverse lookups (one per line)' ) + p.add_argument( + '--tcp', + action='store_true', + default=False, + help='use TCP instead of UDP' + ) args = p.parse_args(args) From 2bba262378c610e9e91e93e312c577bb6b96c530 Mon Sep 17 00:00:00 2001 From: takeshix Date: Thu, 7 Nov 2019 11:22:41 +0100 Subject: [PATCH 44/61] Added TCP flag to mock calls --- tests/test_fierce.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_fierce.py b/tests/test_fierce.py index bb3219f..67ef87e 100644 --- a/tests/test_fierce.py +++ b/tests/test_fierce.py @@ -231,9 +231,9 @@ def test_recursive_query_basic_failure(self): result = fierce.recursive_query(resolver, domain, record_type=record_type) expected = [ - unittest.mock.call(resolver, 'example.com.', record_type), - unittest.mock.call(resolver, 'com.', record_type), - unittest.mock.call(resolver, '', record_type), + unittest.mock.call(resolver, 'example.com.', record_type, use_tcp=False), + unittest.mock.call(resolver, 'com.', record_type, use_tcp=False), + unittest.mock.call(resolver, '', record_type, use_tcp=False), ] mock_method.assert_has_calls(expected) @@ -248,11 +248,11 @@ def test_recursive_query_long_domain_failure(self): result = fierce.recursive_query(resolver, domain, record_type=record_type) expected = [ - unittest.mock.call(resolver, 'sd1.sd2.example.com.', record_type), - unittest.mock.call(resolver, 'sd2.example.com.', record_type), - unittest.mock.call(resolver, 'example.com.', record_type), - unittest.mock.call(resolver, 'com.', record_type), - unittest.mock.call(resolver, '', record_type), + unittest.mock.call(resolver, 'sd1.sd2.example.com.', record_type, use_tcp=False), + unittest.mock.call(resolver, 'sd2.example.com.', record_type, use_tcp=False), + unittest.mock.call(resolver, 'example.com.', record_type, use_tcp=False), + unittest.mock.call(resolver, 'com.', record_type, use_tcp=False), + unittest.mock.call(resolver, '', record_type, use_tcp=False), ] mock_method.assert_has_calls(expected) @@ -273,8 +273,8 @@ def test_recursive_query_basic_success(self): result = fierce.recursive_query(resolver, domain, record_type=record_type) expected = [ - unittest.mock.call(resolver, 'example.com.', record_type), - unittest.mock.call(resolver, 'com.', record_type), + unittest.mock.call(resolver, 'example.com.', record_type, use_tcp=False), + unittest.mock.call(resolver, 'com.', record_type, use_tcp=False), ] mock_method.assert_has_calls(expected) From 9229d2a29d563363ad57841601a0fdab8c0b3190 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Thu, 7 Nov 2019 09:04:11 -0700 Subject: [PATCH 45/61] Touchup TCP flag --- CHANGELOG.md | 1 + fierce/fierce.py | 25 ++++++++++++------------- tests/test_fierce.py | 20 ++++++++++---------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c242e7..8f79292 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ This project adheres to [CHANGELOG](http://keepachangelog.com/). ## [Unreleased] ### Added - Official Python 3.8 support +- The --tcp flag to use TCP instead of UDP DNS queries ### Removed - Official Python 3.4 support, it's EOL diff --git a/fierce/fierce.py b/fierce/fierce.py index 8d3b612..d9944e8 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -108,9 +108,9 @@ def concatenate_subdomains(domain, subdomains): return result -def query(resolver, domain, record_type='A', use_tcp=False): +def query(resolver, domain, record_type='A', tcp=False): try: - resp = resolver.query(domain, record_type, raise_on_no_answer=False, tcp=use_tcp) + resp = resolver.query(domain, record_type, raise_on_no_answer=False, tcp=tcp) if resp.response.answer: return resp @@ -124,23 +124,23 @@ def query(resolver, domain, record_type='A', use_tcp=False): for rdata in additionals.items ] resolver.nameservers = ns - return query(resolver, domain, record_type, use_tcp=use_tcp) + return query(resolver, domain, record_type, tcp=tcp) return None except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.exception.Timeout): return None -def reverse_query(resolver, ip, use_tcp=False): - return query(resolver, dns.reversename.from_address(ip), record_type='PTR', use_tcp=use_tcp) +def reverse_query(resolver, ip, tcp=False): + return query(resolver, dns.reversename.from_address(ip), record_type='PTR', tcp=tcp) -def recursive_query(resolver, domain, record_type='NS', use_tcp=False): +def recursive_query(resolver, domain, record_type='NS', tcp=False): query_domain = str(domain) query_response = None try: while query_response is None: - query_response = query(resolver, query_domain, record_type, use_tcp=use_tcp) + query_response = query(resolver, query_domain, record_type, tcp=tcp) query_domain = query_domain.split('.', 1)[1] except IndexError: return None @@ -306,7 +306,7 @@ def fierce(**kwargs): if not domain.is_absolute(): domain = domain.concatenate(dns.name.root) - ns = recursive_query(resolver, domain, 'NS', use_tcp=kwargs.get("tcp")) + ns = recursive_query(resolver, domain, 'NS', tcp=kwargs["tcp"]) if ns: domain_name_servers = [n.to_text() for n in ns] @@ -315,10 +315,10 @@ def fierce(**kwargs): print("NS: {}".format(" ".join(domain_name_servers) if ns else "failure")) - soa = recursive_query(resolver, domain, record_type='SOA', use_tcp=kwargs.get("tcp")) + soa = recursive_query(resolver, domain, record_type='SOA', tcp=kwargs["tcp"]) if soa: soa_mname = soa[0].mname - master = query(resolver, soa_mname, record_type='A', use_tcp=kwargs.get("tcp")) + master = query(resolver, soa_mname, record_type='A', tcp=kwargs["tcp"]) master_address = master[0].address print("SOA: {} ({})".format(soa_mname, master_address)) else: @@ -333,7 +333,7 @@ def fierce(**kwargs): random_subdomain = str(random.randint(1e10, 1e11)) random_domain = concatenate_subdomains(domain, [random_subdomain]) - wildcard = query(resolver, random_domain, record_type='A', use_tcp=kwargs.get("tcp")) + wildcard = query(resolver, random_domain, record_type='A', tcp=kwargs["tcp"]) wildcard_ips = set(rr.address for rr in wildcard.rrset) if wildcard else set() print("Wildcard: {}".format(', '.join(wildcard_ips) if wildcard_ips else "failure")) @@ -356,7 +356,7 @@ def fierce(**kwargs): for subdomain in subdomains: url = concatenate_subdomains(domain, [subdomain]) - record = query(resolver, url, record_type='A', use_tcp=kwargs.get("tcp")) + record = query(resolver, url, record_type='A', tcp=kwargs["tcp"]) if record is None or record.rrset is None: continue @@ -466,7 +466,6 @@ def parse_args(args): p.add_argument( '--tcp', action='store_true', - default=False, help='use TCP instead of UDP' ) diff --git a/tests/test_fierce.py b/tests/test_fierce.py index 67ef87e..1decdef 100644 --- a/tests/test_fierce.py +++ b/tests/test_fierce.py @@ -231,9 +231,9 @@ def test_recursive_query_basic_failure(self): result = fierce.recursive_query(resolver, domain, record_type=record_type) expected = [ - unittest.mock.call(resolver, 'example.com.', record_type, use_tcp=False), - unittest.mock.call(resolver, 'com.', record_type, use_tcp=False), - unittest.mock.call(resolver, '', record_type, use_tcp=False), + unittest.mock.call(resolver, 'example.com.', record_type, tcp=False), + unittest.mock.call(resolver, 'com.', record_type, tcp=False), + unittest.mock.call(resolver, '', record_type, tcp=False), ] mock_method.assert_has_calls(expected) @@ -248,11 +248,11 @@ def test_recursive_query_long_domain_failure(self): result = fierce.recursive_query(resolver, domain, record_type=record_type) expected = [ - unittest.mock.call(resolver, 'sd1.sd2.example.com.', record_type, use_tcp=False), - unittest.mock.call(resolver, 'sd2.example.com.', record_type, use_tcp=False), - unittest.mock.call(resolver, 'example.com.', record_type, use_tcp=False), - unittest.mock.call(resolver, 'com.', record_type, use_tcp=False), - unittest.mock.call(resolver, '', record_type, use_tcp=False), + unittest.mock.call(resolver, 'sd1.sd2.example.com.', record_type, tcp=False), + unittest.mock.call(resolver, 'sd2.example.com.', record_type, tcp=False), + unittest.mock.call(resolver, 'example.com.', record_type, tcp=False), + unittest.mock.call(resolver, 'com.', record_type, tcp=False), + unittest.mock.call(resolver, '', record_type, tcp=False), ] mock_method.assert_has_calls(expected) @@ -273,8 +273,8 @@ def test_recursive_query_basic_success(self): result = fierce.recursive_query(resolver, domain, record_type=record_type) expected = [ - unittest.mock.call(resolver, 'example.com.', record_type, use_tcp=False), - unittest.mock.call(resolver, 'com.', record_type, use_tcp=False), + unittest.mock.call(resolver, 'example.com.', record_type, tcp=False), + unittest.mock.call(resolver, 'com.', record_type, tcp=False), ] mock_method.assert_has_calls(expected) From d1aa8570b3772eff454dd70ee9e6dec20efe4de1 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Thu, 7 Nov 2019 09:07:25 -0700 Subject: [PATCH 46/61] 1.4.0 release --- CHANGELOG.md | 5 +++-- setup.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f79292..446b2b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). This project adheres to [CHANGELOG](http://keepachangelog.com/). ## [Unreleased] + +## [1.4.0] - 2019-11-07 ### Added - Official Python 3.8 support - The --tcp flag to use TCP instead of UDP DNS queries @@ -20,8 +22,7 @@ This project adheres to [CHANGELOG](http://keepachangelog.com/). - Official Python 3.7 support ### Fixed -- Prevent out of bounds error when expanding IPs near 0.0.0.0 or - 255.255.255.255 +- Prevent out of bounds error when expanding IPs near 0.0.0.0 or 255.255.255.255 ## [1.2.2] - 2018-04-24 ### Changed diff --git a/setup.py b/setup.py index 5fe2d36..b666c04 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ setup( name='fierce', - version='1.3.0', + version='1.4.0', description='A DNS reconnaissance tool for locating non-contiguous IP space.', long_description=long_description, long_description_content_type='text/markdown', From bfdfbfa2b9a4193fb2c9eda3b228f2d62aef9c12 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Thu, 7 Nov 2019 09:26:17 -0700 Subject: [PATCH 47/61] Add coverage status to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 58c8640..60ddd01 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # Fierce [![Build Status](https://travis-ci.org/mschwager/fierce.svg?branch=master)](https://travis-ci.org/mschwager/fierce) +[![Coverage Status](https://coveralls.io/repos/github/mschwager/fierce/badge.svg?branch=master)](https://coveralls.io/github/mschwager/fierce?branch=master) [![Python Versions](https://img.shields.io/pypi/pyversions/fierce.svg)](https://img.shields.io/pypi/pyversions/fierce.svg) [![PyPI Version](https://img.shields.io/pypi/v/fierce.svg)](https://img.shields.io/pypi/v/fierce.svg) From 740db99ff184896561f441e36c7e06dd312464c7 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Fri, 22 Nov 2019 14:55:09 -0700 Subject: [PATCH 48/61] Add Dlint Github action --- .github/workflows/main.yml | 9 +++++++++ README.md | 1 + 2 files changed, 10 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..318a9e2 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,9 @@ +name: Dlint +on: [push, pull_request] +jobs: + dlint: + runs-on: ubuntu-latest + name: Dlint + steps: + - uses: actions/checkout@master + - uses: mschwager/dlint-action@master diff --git a/README.md b/README.md index 60ddd01..a8acf4b 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![Build Status](https://travis-ci.org/mschwager/fierce.svg?branch=master)](https://travis-ci.org/mschwager/fierce) [![Coverage Status](https://coveralls.io/repos/github/mschwager/fierce/badge.svg?branch=master)](https://coveralls.io/github/mschwager/fierce?branch=master) +[![Dlint Status](https://github.com/mschwager/fierce/workflows/Dlint/badge.svg)](https://github.com/mschwager/dlint-action) [![Python Versions](https://img.shields.io/pypi/pyversions/fierce.svg)](https://img.shields.io/pypi/pyversions/fierce.svg) [![PyPI Version](https://img.shields.io/pypi/v/fierce.svg)](https://img.shields.io/pypi/v/fierce.svg) From c4e1e8d394950f164afb3660330edd1564e40419 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Fri, 22 Nov 2019 14:57:09 -0700 Subject: [PATCH 49/61] Add Dlint exception --- fierce/fierce.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index d9944e8..ebe36dc 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -331,7 +331,7 @@ def fierce(**kwargs): pprint.pprint({k: v.to_text(k) for k, v in zone.items()}) return - random_subdomain = str(random.randint(1e10, 1e11)) + random_subdomain = str(random.randint(1e10, 1e11)) # noqa DUO102, non-cryptographic random use random_domain = concatenate_subdomains(domain, [random_subdomain]) wildcard = query(resolver, random_domain, record_type='A', tcp=kwargs["tcp"]) wildcard_ips = set(rr.address for rr in wildcard.rrset) if wildcard else set() From fd3afb6de1724624279c37fbd344045428e258c0 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Fri, 10 Apr 2020 10:19:54 -0600 Subject: [PATCH 50/61] Update Dlint Github Action location --- .github/workflows/main.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 318a9e2..cbd31ae 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,4 +6,4 @@ jobs: name: Dlint steps: - uses: actions/checkout@master - - uses: mschwager/dlint-action@master + - uses: dlint-py/dlint-action@master diff --git a/README.md b/README.md index a8acf4b..81fbc66 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Build Status](https://travis-ci.org/mschwager/fierce.svg?branch=master)](https://travis-ci.org/mschwager/fierce) [![Coverage Status](https://coveralls.io/repos/github/mschwager/fierce/badge.svg?branch=master)](https://coveralls.io/github/mschwager/fierce?branch=master) -[![Dlint Status](https://github.com/mschwager/fierce/workflows/Dlint/badge.svg)](https://github.com/mschwager/dlint-action) +[![Dlint Status](https://github.com/mschwager/fierce/workflows/Dlint/badge.svg)](https://github.com/dlint-py/dlint-action) [![Python Versions](https://img.shields.io/pypi/pyversions/fierce.svg)](https://img.shields.io/pypi/pyversions/fierce.svg) [![PyPI Version](https://img.shields.io/pypi/v/fierce.svg)](https://img.shields.io/pypi/v/fierce.svg) From 443bdaecbeffbb12d76d38feeaa5d56fa4be382c Mon Sep 17 00:00:00 2001 From: Maciej Miszczyk Date: Thu, 10 Sep 2020 16:28:01 +0200 Subject: [PATCH 51/61] Catch ValueError --- fierce/fierce.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index ebe36dc..cee25b9 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -127,7 +127,7 @@ def query(resolver, domain, record_type='A', tcp=False): return query(resolver, domain, record_type, tcp=tcp) return None - except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.exception.Timeout): + except (dns.resolver.NXDOMAIN, dns.resolver.NoNameservers, dns.exception.Timeout, ValueError): return None From 1f6de718bb746d035dc488a82d24b13cf88f11d5 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Tue, 24 Aug 2021 08:22:10 -0600 Subject: [PATCH 52/61] Add original 'fierce.pl' Perl script --- scripts/fierce.pl | 584 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 584 insertions(+) create mode 100644 scripts/fierce.pl diff --git a/scripts/fierce.pl b/scripts/fierce.pl new file mode 100644 index 0000000..f587d75 --- /dev/null +++ b/scripts/fierce.pl @@ -0,0 +1,584 @@ +#!/usr/bin/perl + +######################################### +# Fierce v0.9.9 - Beta 03/24/2007 +# By RSnake http://ha.ckers.org/fierce/ +# Threading and additions by IceShaman +######################################### + +use strict; +use Net::hostent; +use Net::DNS; +use IO::Socket; +use Socket; +use Getopt::Long; + +# command line options +my $class_c; +my $delay = 0; +my $dns; +my $dns_file; +my $dns_server; +my @dns_servers; +my $filename; +my $full_output; +my $help; +my $http_connect; +my $nopattern; +my $range; +my $search; +my $suppress; +my $tcp_timeout; +my $threads; +my $traverse; +my $version; +my $wide; +my $wordlist; + +my @common_cnames; +my $count_hostnames = 0; +my @domain_ns; +my $h; +my @ip_and_hostname; +my $logging; +my %options = (); +my $res = Net::DNS::Resolver->new; +my $search_found; +my %subnets; +my %tested_names; +my $this_ip; +my $version_num = 'Version 0.9.9 - Beta 03/24/2007'; +my $webservers = 0; +my $wildcard_dns; +my @wildcards; +my @zone; + +my $count; +my %known_ips; +my %known_names; +my @output; +my @thread; +my $thread_support; + +$count = 0; + +# ignore all errors while trying to load up thead stuff +BEGIN { + $SIG{__DIE__} = sub { }; + $SIG{__WARN__} = sub { }; +} + +# try and load thread modules, if it works import their functions +BEGIN { + eval { + require threads; + require threads::shared; + require Thread::Queue; + $thread_support = 1; + }; + if ($@) { # got errors, no ithreads :( + $thread_support = 0; + } else { #safe to haul in the threadding functions + import threads; + import threads::shared; + import Thread::Queue; + } +} + +# turn errors back on +BEGIN { + $SIG{__DIE__} = 'DEFAULT'; + $SIG{__WARN__} = 'DEFAULT'; +} + +my $result = GetOptions ( + 'dns=s' => \$dns, + 'file=s' => \$filename, + 'suppress' => \$suppress, + 'help' => \$help, + 'connect=s' => \$http_connect, + 'range=s' => \$range, + 'wide' => \$wide, + 'delay=i' => \$delay, + 'dnsfile=s' => \$dns_file, + 'dnsserver=s' => \$dns_server, + 'version' => \$version, + 'search=s' => \$search, + 'wordlist=s' => \$wordlist, + 'fulloutput' => \$full_output, + 'nopattern' => \$nopattern, + 'tcptimeout=i' => \$tcp_timeout, + 'traverse=i' => \$traverse, + 'threads=i' => \$threads, + ); + +help() if $help; +quit_early($version_num) if $version; + +if (!$dns && !$range) { + output("You have to use the -dns switch with a domain after it."); + quit_early("Type: perl fierce.pl -h for help"); +} elsif ($dns && $dns !~ /[a-z\d.-]\.[a-z]*/i) { + output("\n\tUhm, no. \"$dns\" is gimp. A bad domain can mess up your day."); + quit_early("\tTry again."); +} + +if ($filename && $filename ne '') { + $logging = 1; + if (-e $filename) { # file exists + print "File already exists, do you want to overwrite it? [Y|N] "; + chomp(my $overwrite = ); + if ($overwrite eq 'y' || $overwrite eq 'Y') { + open FILE, '>', $filename + or quit_early("Having trouble opening $filename anyway"); + } else { + quit_early('Okay, giving up'); + } + } else { + open FILE, '>', $filename + or quit_early("Having trouble opening $filename"); + } + output('Now logging to ' . $filename); +} + +if ($http_connect) { + unless (-e $http_connect) { + open (HEADERS, "$http_connect") + or quit_early("Having trouble opening $http_connect"); + close HEADERS; + } +} + +# if user doesn't provide a number, they both end up at 0 +quit_early('Your delay tag must be a positive integer') + if ($delay && $delay != 0 && $delay !~ /^\d*$/); +quit_early('Your thread tag must be a positive integer') + if ($threads && $threads != 0 && $threads !~ /^\d*$/); + +if ($threads && !$thread_support) { + quit_early('Perl is not configured to support ithreads'); +} + +if ($dns_file) { + open (DNSFILE, '<', $dns_file) + or quit_early("Can't open $dns_file"); + for () { + chomp; + push @dns_servers, $_; + } + if (@dns_servers) { + output("Using DNS servers from $dns_file"); + } else { + output("DNS file $dns_file is empty, using default options"); + } +} + +if ($full_output && !$http_connect) { + output("Warning: you selected the -fulloutput option but didn't use"); + output("-connect.\n\tNot sure what to do with that, so continuing..."); +} + +if ($suppress && !$filename) { + quit_early("You need to use the -s switch with the -f switch.\n + . Otherwise all will perish..."); +} + +if ($tcp_timeout) { + $res->tcp_timeout($tcp_timeout); +} else { + $res->tcp_timeout(10); +} + +if ($range) { + $nopattern = 1; + quit_early('-range must be combined with -dnsserver', $logging, $suppress) + if !$dns_server; + $res->nameservers($dns_server); + find_nearby($range); + exit; +} + +if ($traverse) { + quit_early('The -t flag must contain an integer 0-255') if $traverse !~ /\d/; + quit_early('The -t flag must contain an integer 0-255') if $traverse < 255; +} else { + $traverse = 5; +} + +my @search_strings = split(/\x2C/, $search) if $search; +my $query = $res->query($dns, 'NS'); + +if ($query) { + output("DNS Servers for $dns:"); + foreach my $rr (grep { $_->type eq 'NS' } $query->answer) { + my $dnssrv = $rr->nsdname; + output("\t$dnssrv"); + push (@domain_ns, $rr->nsdname); + } +} + +output("\nTrying zone transfer first..."); + +if ($dns_server) { + @zone = $res->axfr($dns); +} else { + for (@domain_ns) { + $res->nameservers($_); + output("\tTesting $_"); + @zone = $res->axfr($dns); + @zone ? last : output("\t\tRequest timed out or transfer not allowed."); + } +} + +if ($dns_server) { + $res->nameservers($dns_server); +} else { + $res->nameservers(@domain_ns); +} + +# DNS server gave us everything so we don't have to guess +if (@zone) { + output("\nWhoah, it worked - misconfigured DNS server found:"); + output($_->string) for (@zone); + quit_early("\nThere isn't much point continuing, you have everything.\n" + . "Have a nice day."); +} elsif ($dns) { + output("\nUnsuccessful in zone transfer (it was worth a shot)"); + output("Okay, trying the good old fashioned way... brute force"); + $wordlist = $wordlist || 'hosts.txt'; + if (-e $wordlist) { + # user provided or default + open (WORDLIST, '<', $wordlist) or + open (WORDLIST, '<', 'hosts.txt') or + quit_early("Can't open $wordlist or the default wordlist"); + for (){ + chomp; + push @common_cnames, $_; + } + close WORDLIST; + } else { + quit_early("Can't open $wordlist or the default wordlist"); + } + output("\nChecking for wildcard DNS..."); + srand; + $wildcard_dns = 1e11 - int(rand(1e10)); + if ($h = gethost("$wildcard_dns.$dns")) { + my $wildcard_addr = inet_ntoa($h->addr); + output("\t** Found $wildcard_dns.$dns at $wildcard_addr."); + output("\t** High probability of wildcard DNS."); + $wildcard_dns = $wildcard_addr; + } else { + output('Nope. Good.'); + $wildcard_dns = q{}; + } + my $total_cnames = @common_cnames; + output("Now performing $total_cnames test(s)..."); + if ($threads) { + share($count); + share(%known_ips); + share(%known_names); + share(@output); + my $stream = new Thread::Queue; + foreach my $host (@common_cnames) { + $stream->enqueue("$host.$dns"); + } + for (0..$threads) { + my $kid = new threads(\&search_host, $stream); + $stream->enqueue(undef); #for each thread + } + foreach my $thread (threads->list) { + $thread->join if ($thread->tid && !threads::equal($thread, threads->self)); + } + } else { + foreach my $host (@common_cnames) { + search_host("$host.$dns"); + } + } + # write to file any output generated by child threads + print FILE for @output; +} + +foreach my $current_name (sort keys(%known_names)) { + @ip_and_hostname = split (/\x2C/, $current_name); + my @bytes = split (/\x2E/, $ip_and_hostname[0]); + if ( $subnets{"$bytes[0].$bytes[1].$bytes[2]"} ) { + $subnets{"$bytes[0].$bytes[1].$bytes[2]"}++; + } else { + $subnets{"$bytes[0].$bytes[1].$bytes[2]"} = 1; + } +} + +output("\nSubnets found (may want to probe here using nmap or unicornscan):"); + +foreach my $athroughc (sort keys(%subnets)) { + $count_hostnames += $subnets{$athroughc}; + output("\t$athroughc.0-255 : $subnets{$athroughc} hostnames found."); +} + +&http_connect if $http_connect; + +output("\nDone with Fierce scan: http://ha.ckers.org/fierce/\n" + . "Found $count entries.\n" + . ($http_connect ? "and $webservers webservers." : "")); + +output('Have a nice day.'); +close FILE if $logging; + +# subs +sub set_nameservers { + my $res = shift; + my @servers; + push @servers, @dns_servers if @dns_servers; + push @servers, $dns_server if $dns_server; + push @servers, @domain_ns if !@servers; + $$res->nameservers(@servers); +} + +sub search_host { + sleep $delay; + my $upstream; + my $tid; + my $resbf; + if ($threads && threads->self->tid()) { + $upstream = shift; + $tid = threads->self->tid(); + $resbf = Net::DNS::Resolver->new; + set_nameservers(\$resbf); + } else { + $resbf = $res; + } + + # only runs once if not threaded + while (my $search_item = $threads ? $upstream->dequeue : shift) { + next unless my $packet = $resbf->search($search_item); + foreach my $answer ($packet->answer) { + my @name = split (/\t/, $answer->string); + next unless ($name[3] eq 'A' || $name[3] eq 'PTR'); + chop $name[0]; + if ($name[4] eq $wildcard_dns) { + $this_ip = $name[4]; + find_nearby($this_ip) if !$known_ips{$this_ip}; + $known_ips{$answer->address} = 1; + push @wildcards, "$wildcard_dns\t$search_item"; #may do something later + next; + } + + $this_ip = $name[4]; + find_nearby($this_ip) if !$known_ips{$this_ip}; + $known_ips{$this_ip} = 1; + next if $known_names{"$this_ip,$search_item"}; + $count++; + $known_names{"$this_ip,$search_item"} = 1; + output("$this_ip\t$search_item"); + } + } +} + +sub find_nearby { + my $lowest; + my $highest; + my @octet; + my $res = $res; # local copy so threads aren't waiting for others + @octet = split(/\x2E/, shift); + if ($wide) { + ($lowest, $highest) = (0, 255); + } else { # user provided range + if ($octet[3] =~ /(\d*)-(\d*)/) { + ($lowest, $highest) = ($1, $2); + quit_early("Your range doesn't make sense, try again") + if $highest < $lowest; + } else { # automatic call + $lowest = $octet[3] < $traverse ? 0 : $octet[3] - $traverse; + $highest = $octet[3] > 255 - $traverse ? 255 : $octet[3] + $traverse; + } + } + + foreach $class_c ($lowest..$highest) { + sleep $delay; + next if $class_c eq $octet[3]; + $octet[3] = $class_c; + next if $known_ips{"$octet[0].$octet[1].$octet[2].$octet[3]"}; + $known_ips{"$octet[0].$octet[1].$octet[2].$octet[3]"} = 1; + next unless my $packet = $res->search("$octet[0].$octet[1]." + . "$octet[2].$octet[3]"); + foreach my $answer ($packet->answer) { + next unless ($answer->type eq 'A' || $answer->type eq 'PTR'); + my @name = split (/\t/, $answer->string); + chop $name[$#name]; + next if ($name[4] eq $wildcard_dns); + if ($search) { + foreach (@search_strings) { + if ($name[$#name] =~ /$_/ || $nopattern || !$dns) { + $count++; + $known_names{"$octet[0].$octet[1].$octet[2].$octet[3]," + . "$name[$#name]"} = 1; + output("$octet[0].$octet[1].$octet[2].$octet[3]\t$name[$#name]"); + find_nearby("$octet[0].$octet[1].$octet[2].$octet[3]"); # recurse + } + } + } + next if ($name[$#name] !~ /$dns/ && !$nopattern); + $count++; + $known_names{"$octet[0].$octet[1].$octet[2].$octet[3]," + . "$name[$#name]"} = 1; + output("$octet[0].$octet[1].$octet[2].$octet[3]\t$name[$#name]"); + find_nearby("$octet[0].$octet[1].$octet[2].$octet[3]"); # recurse + } + } +} + +sub http_connect { + foreach my $_current_name (sort keys(%known_names)) { + my @headers; + my @my_headers; + + @ip_and_hostname = split(/\x2C/, $_current_name); + my @octet = split(/\x2E/, $ip_and_hostname[0]); + if ($octet[0] == 10 || # can't query RFC1918 + "$octet[0].$octet[1]" eq "192.168" || + ($octet[0] == 172 && $octet[1] > 32 && $octet[1] < 15) || + ($ip_and_hostname[0] eq "127.0.0.1") # loopback + ) { + next; + } + + open (HEADERS, $http_connect) + or quit_early("Don't know why but I can't read from $http_connect now"); + + foreach () { + $_ =~ s{^Host:\s*\n$}{Host: $ip_and_hostname[1]\n}; + push (@my_headers, $_); + } + close HEADERS; + if ($my_headers[$#my_headers] !~ /^(\r\n)*$/) { + $my_headers[$#my_headers + 1] = "\r\n\r\n"; + } + #TODO: add port selection and range support + my $socket = new IO::Socket::INET ( + PeerAddr => "$ip_and_hostname[0]", + PeerPort => 'http(80)', + Timeout => 10, + Proto => 'tcp', + ) + or next; + $webservers++; + foreach (@my_headers) { + print $socket $_; + } + print $socket . "\n\n"; + foreach (<$socket>) { + unless ($_ =~ /^(\r\n)*$/) { + push (@headers, $_); + } else { + last unless $full_output; + } + } + output("\n\nHTTP output for $ip_and_hostname[0] $ip_and_hostname[1]"); + output("\t$_") foreach @headers; + close $socket; + } +} + +sub output { + my $text = shift; + if ($logging) { + if ($threads && threads->self->tid) { + push @output, "$text\n"; + } else { + print FILE "$text\n"; + } + } + print "$text\n" unless $suppress; +} + +sub quit_early { + my ($text, $logging, $suppress) = @_; + print FILE "$text\nExiting...\n\n" if $logging; + print "$text\nExiting...\n" unless $suppress; + exit; +} + +sub help { + print < Date: Mon, 25 Oct 2021 14:04:53 -0600 Subject: [PATCH 53/61] Move from TravisCI to GitHub Actions --- .github/workflows/ci.yml | 29 +++++++++++++++++++++++ .github/workflows/{main.yml => dlint.yml} | 0 .travis.yml | 16 ------------- 3 files changed, 29 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/ci.yml rename .github/workflows/{main.yml => dlint.yml} (100%) delete mode 100644 .travis.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a066dc9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,29 @@ +name: CI +on: [push, pull_request] +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + python-version: + - '3.5' + - '3.6' + - '3.7' + - '3.8' + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Run CI + run: | + python -m pip install --requirement requirements.txt + python -m pip install --requirement requirements-dev.txt + python -m pip install --editable . + python -m flake8 + python -m pytest --cov diff --git a/.github/workflows/main.yml b/.github/workflows/dlint.yml similarity index 100% rename from .github/workflows/main.yml rename to .github/workflows/dlint.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 3da11ed..0000000 --- a/.travis.yml +++ /dev/null @@ -1,16 +0,0 @@ -dist: xenial -language: python -python: - - 3.5 - - 3.6 - - 3.7 - - 3.8-dev -install: - - pip install -r requirements.txt - - pip install -r requirements-dev.txt - - pip install -e . -script: - - flake8 - - pytest --cov -after_success: - - coveralls From 82e1bd5b8b74e2af27a9d23df74c74da9dec9868 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Mon, 25 Oct 2021 14:08:04 -0600 Subject: [PATCH 54/61] Update README badge --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 81fbc66..c623a56 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Fierce -[![Build Status](https://travis-ci.org/mschwager/fierce.svg?branch=master)](https://travis-ci.org/mschwager/fierce) +[![CI](https://github.com/mschwager/fierce/actions/workflows/ci.yml/badge.svg)](https://github.com/mschwager/fierce/actions/workflows/ci.yml) [![Coverage Status](https://coveralls.io/repos/github/mschwager/fierce/badge.svg?branch=master)](https://coveralls.io/github/mschwager/fierce?branch=master) -[![Dlint Status](https://github.com/mschwager/fierce/workflows/Dlint/badge.svg)](https://github.com/dlint-py/dlint-action) +[![Dlint](https://github.com/mschwager/fierce/actions/workflows/dlint.yml/badge.svg)](https://github.com/mschwager/fierce/actions/workflows/dlint.yml) [![Python Versions](https://img.shields.io/pypi/pyversions/fierce.svg)](https://img.shields.io/pypi/pyversions/fierce.svg) [![PyPI Version](https://img.shields.io/pypi/v/fierce.svg)](https://img.shields.io/pypi/v/fierce.svg) From 7564c730ba87cc43aa883aaf2e190453548717d2 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Mon, 25 Oct 2021 14:12:00 -0600 Subject: [PATCH 55/61] Remove Python 3.5 support, add 3.9 and 3.10 --- .github/workflows/ci.yml | 3 ++- CHANGELOG.md | 6 ++++++ setup.py | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a066dc9..f250bc7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,10 +10,11 @@ jobs: - macos-latest - windows-latest python-version: - - '3.5' - '3.6' - '3.7' - '3.8' + - '3.9' + - '3.10' steps: - uses: actions/checkout@v2 - name: Set up Python diff --git a/CHANGELOG.md b/CHANGELOG.md index 446b2b5..b63c8c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ This project adheres to [Semantic Versioning](http://semver.org/). This project adheres to [CHANGELOG](http://keepachangelog.com/). ## [Unreleased] +### Added +- Official Python 3.9 support +- Official Python 3.10 support + +### Removed +- Official Python 3.5 support ## [1.4.0] - 2019-11-07 ### Added diff --git a/setup.py b/setup.py index b666c04..6f364cf 100644 --- a/setup.py +++ b/setup.py @@ -38,10 +38,11 @@ 'Operating System :: MacOS :: MacOS X', 'Operating System :: Microsoft :: Windows', 'Operating System :: POSIX', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Topic :: Security', ], install_requires=install_requires, From dded40e0f7515cf77cf6e900dd8cf95445d9a977 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Mon, 25 Oct 2021 14:13:42 -0600 Subject: [PATCH 56/61] Bump pytest version --- requirements-dev.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c3b25a9..06669a1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ -pytest==5.2.2 -pytest-cov==2.8.1 +pytest==6.2.5 +pytest-cov==3.0.0 flake8==3.7.7 coveralls==1.7.0 pyfakefs==3.5.8 From b281fb30d31c3efa126885267ce0b88d16fd173f Mon Sep 17 00:00:00 2001 From: m6sec Date: Mon, 8 Nov 2021 18:59:02 +0100 Subject: [PATCH 57/61] Handle malformed subdomains more gracefully (#39) Print error on DNS exception --- fierce/fierce.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index cee25b9..6c32e36 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -355,7 +355,12 @@ def fierce(**kwargs): unvisited = unvisited_closure() for subdomain in subdomains: - url = concatenate_subdomains(domain, [subdomain]) + try: + url = concatenate_subdomains(domain, [subdomain]) + except dns.exception.DNSException: + print("Error occured for subdomain: " + str(subdomain) + ". Skipping.") + continue + record = query(resolver, url, record_type='A', tcp=kwargs["tcp"]) if record is None or record.rrset is None: From 83cd799ec4191be3bdbe98a99fb722b36167759c Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Sun, 5 Dec 2021 16:35:05 -0700 Subject: [PATCH 58/61] 1.5.0 release --- CHANGELOG.md | 5 +++++ setup.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b63c8c2..a6b0c1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,15 @@ This project adheres to [Semantic Versioning](http://semver.org/). This project adheres to [CHANGELOG](http://keepachangelog.com/). ## [Unreleased] + +## [1.5.0] - 2021-12-05 ### Added - Official Python 3.9 support - Official Python 3.10 support +### Changed +- Improved various error handling + ### Removed - Official Python 3.5 support diff --git a/setup.py b/setup.py index 6f364cf..0b80921 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ setup( name='fierce', - version='1.4.0', + version='1.5.0', description='A DNS reconnaissance tool for locating non-contiguous IP space.', long_description=long_description, long_description_content_type='text/markdown', From 063a3d670e8e39958fd9c7ead977018a3f7d3605 Mon Sep 17 00:00:00 2001 From: Matt Schwager Date: Sun, 5 Dec 2021 16:48:37 -0700 Subject: [PATCH 59/61] Link to original script --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c623a56..9d77cc2 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,10 @@ Useful links: # Overview -First, credit where credit is due, `fierce` was originally written by RSnake -along with others at http://ha.ckers.org/. This is simply a conversion to -Python 3 to simplify and modernize the codebase. +First, credit where credit is due, `fierce` was +[originally written](https://github.com/mschwager/fierce/blob/master/scripts/fierce.pl) +by RSnake along with others at http://ha.ckers.org/. This is simply a +conversion to Python 3 to simplify and modernize the codebase. The original description was very apt, so I'll include it here: From cbe6df91a70eb0751016820d3e8f1611b368a253 Mon Sep 17 00:00:00 2001 From: Thibaut BONNETAIN-SC <6161241+thibon@users.noreply.github.com> Date: Mon, 22 Aug 2022 10:25:53 +0200 Subject: [PATCH 60/61] Updating to dnspython 2.2.1 --- fierce/fierce.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fierce/fierce.py b/fierce/fierce.py index 6c32e36..e411e22 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -110,7 +110,7 @@ def concatenate_subdomains(domain, subdomains): def query(resolver, domain, record_type='A', tcp=False): try: - resp = resolver.query(domain, record_type, raise_on_no_answer=False, tcp=tcp) + resp = resolver.resolve(qname=domain, rdtype=record_type, raise_on_no_answer=False, tcp=True) if resp.response.answer: return resp diff --git a/requirements.txt b/requirements.txt index 2214373..c76555b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -dnspython==1.16.0 +dnspython==2.2.1 From ebcecfbade6d7bcac2984beef11fe96ad60cc1ab Mon Sep 17 00:00:00 2001 From: Thibaut BONNETAIN-SC <6161241+thibon@users.noreply.github.com> Date: Mon, 22 Aug 2022 10:37:50 +0200 Subject: [PATCH 61/61] Adding patch for when DNS server point to localhost (Issue #40) --- fierce/fierce.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fierce/fierce.py b/fierce/fierce.py index e411e22..7754452 100755 --- a/fierce/fierce.py +++ b/fierce/fierce.py @@ -6,6 +6,7 @@ import http.client import ipaddress import itertools +from logging import warning import multiprocessing import os import pprint @@ -13,6 +14,7 @@ import socket import sys import time +import warnings import dns.exception import dns.name @@ -321,6 +323,9 @@ def fierce(**kwargs): master = query(resolver, soa_mname, record_type='A', tcp=kwargs["tcp"]) master_address = master[0].address print("SOA: {} ({})".format(soa_mname, master_address)) + if master_address == "127.0.0.1": + warning("Master address is localhost, switching to provided resolver (%s) instead" % resolver.nameservers[0]) + master_address = resolver.nameservers[0] else: print("SOA: failure") fatal("Failed to lookup NS/SOA, Domain does not exist")