diff --git a/.gitignore b/.gitignore index 580e444..f4f8fef 100755 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ mazerunner_sdk.egg-info/ dist .coverage .cache -sdk-venv +.sdk htmlcov test_deployments +venv +.pytest_cache diff --git a/PKG-INFO b/PKG-INFO new file mode 100644 index 0000000..93776a9 --- /dev/null +++ b/PKG-INFO @@ -0,0 +1,32 @@ +Metadata-Version: 1.2.3 +Name: mazerunner_sdk_python +Version: 1.2.3 +Summary: MazeRunner python SDK +Home-page: https://github.com/Cymmetria/mazerunner_sdk_python +Author: Cymmetria +License: BSD-3 +Download-URL: https://github.com/Cymmetria/mazerunner_sdk_python/archive/1.2.3.tar.gz +Description: This library implements a convenient client for MazeRunner™ API for Python. + Using this library, you will be able to easily configure and manipulate the key features + of MazeRunner, such as the creation of a deception campaign, turning decoys on or off, deployment on + remote endpoints, and inspecting alerts with their attached evidence. + + See the documentation at https://community.cymmetria.com/api + + Fork us at https://github.com/Cymmetria/mazerunner_sdk_python + +Keywords: mazerunner, sdk, python, deception +Requires: argparse +Requires: mohawk +Requires: requests +Requires: requests-hawk +Requires: six +Requires: wsgiref +Platform: any +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.7 +Classifier: Topic :: Software Development :: Libraries diff --git a/README.md b/README.md index bb3c467..9705c06 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,8 @@ Structure of the json_credentials file: ###Generate documentation files: ~~~~ -export PYTHONPATH=`pwd` -make html +make dev-env +make docs ~~~~ ###See documentation at [https://community.cymmetria.com/api](https://community.cymmetria.com/api) diff --git a/conftest.py b/conftest.py index 3af2105..93d7e8b 100644 --- a/conftest.py +++ b/conftest.py @@ -1,4 +1,5 @@ def pytest_addoption(parser): + parser.addoption("--initial_clean", action="store_true", default=False) parser.addoption("--json_credentials", action="store", default=None, help="Json file with the relevant credentials") parser.addoption("--runslow", action="store_true", default=False, help="Run slow tests") diff --git a/makefile b/makefile new file mode 100644 index 0000000..f04a687 --- /dev/null +++ b/makefile @@ -0,0 +1,16 @@ +SHELL := /bin/bash + +.PHONY: dev-env +dev-env: + sudo apt-get install -y texlive-latex-extra texlive-fonts-recommended python-sphinx; \ + virtualenv .sdk; \ + source .sdk/bin/activate; \ + pip install -r requirements.txt; \ + pip install -r testing_requirements.txt; + +.PHONY: docs +docs: export PYTHONPATH = $(shell pwd):$(shell pwd)/mazerunner:$(shell pwd)/mazerunner/samples +docs: + source .sdk/bin/activate; \ + make -f sphinx_makefile html latexpdf + cp build/latex/MazeRunnerSDK.pdf build/html/sdk.pdf diff --git a/mazerunner/api_client.py b/mazerunner/api_client.py index 6cee850..c81997c 100644 --- a/mazerunner/api_client.py +++ b/mazerunner/api_client.py @@ -1,3 +1,5 @@ +import json +import urlparse from httplib import NO_CONTENT import shutil from numbers import Number @@ -7,13 +9,16 @@ from mazerunner.exceptions import ValidationError, ServerError, BadParamError, \ InvalidInstallMethodError +ENTRIES_PER_PAGE = 500 +ISO_TIME_FORMAT = "%Y-%m-%d" + class BaseCollection(object): MODEL_CLASS = None def __init__(self, api_client, obj_class=None): """ - :param api_client: The connection instance. + :param api_client: The connection instance. :param obj_class: The class, instance of which all the members should be. """ self._api_client = api_client @@ -24,7 +29,7 @@ class Collection(BaseCollection): def __len__(self): query_params = self._get_query_params() response = self._api_client.api_request(self._get_url(), query_params=query_params) - return response['count'] + return response["count"] def __iter__(self): for chunk in self._iter_chunks(): @@ -34,14 +39,14 @@ def __iter__(self): def _iter_chunks(self): query_params = self._get_query_params() response = self._api_client.api_request(self._get_url(), query_params=query_params) - results = response['results'] + results = response["results"] while True: yield [self._obj_class(self._api_client, obj) for obj in results] # Get the next batch of objects if possible - if response.get('next'): - response = self._api_client.api_request(response['next'], query_params=query_params) - results = response['results'] + if response.get("next"): + response = self._api_client.api_request(response["next"], query_params=query_params) + results = response["results"] else: return @@ -81,8 +86,8 @@ def create_item(self, data, files=None): :param data: Element data. :param files: Relevant file paths to upload for the element. """ - response = self._api_client.api_request(self._get_url(), 'post', data=data, files=files) - return self._obj_class(self._api_client, response) + response = self._api_client.api_request(self._get_url(), "post", data=data, files=files) + return self._obj_class(self._api_client, response).load() class UnpaginatedEditableCollection(EditableCollection): @@ -131,10 +136,10 @@ def __init__(self, api_client, param_dict): self._update_related_fields() def __repr__(self): - properties = ' '.join('%s=%s' % (key, repr(value)) + properties = " ".join("%s=%s" % (key, repr(value)) for key, value in self._param_dict.items()) - return '<%s: %s>' % (self.__class__.__name__, properties) + return "<%s: %s>" % (self.__class__.__name__, properties) def _update_related_fields(self): for key, field_type in self.RELATED_COLLECTIONS.iteritems(): @@ -143,8 +148,9 @@ def _update_related_fields(self): self._param_dict.get(key, []))) for key, field_type in self.RELATED_FIELDS.iteritems(): - if key in self._param_dict: - setattr(self, key, field_type(self._api_client, self._param_dict[key])) + value = self._param_dict.get(key, None) + if value: + setattr(self, key, field_type(self._api_client, value)) def __getattr__(self, item): if item not in self._param_dict: @@ -171,14 +177,14 @@ def load(self): class Entity(BaseEntity): def _update_item(self, data, files=None): - response = self._api_client.api_request(self.url, 'put', data=data, files=files) + response = self._api_client.api_request(self.url, "put", data=data, files=files) self._update_entity_data(response) def _partial_update_item(self, data, files=None): non_empty_data = {key: value for key, value in data.iteritems() if value} if non_empty_data: response = self._api_client.api_request(url=self.url, - method='patch', + method="patch", data=non_empty_data, files=files) self._update_entity_data(response) @@ -187,7 +193,7 @@ def delete(self): """ Delete this element. """ - self._api_client.api_request(self.url, 'delete') + self._api_client.api_request(self.url, "delete") class Decoy(Entity): @@ -198,10 +204,10 @@ class Decoy(Entity): or an external machine downloaded as an OVA and manually deployed on an ESX machine. """ - NAME = 'decoy' + NAME = "decoy" def update(self, name, chosen_static_ip=None, chosen_subnet=None, chosen_gateway=None, - chosen_dns=None, dns_address=''): + chosen_dns=None, dns_address=""): """ Change decoy configuration. @@ -222,13 +228,14 @@ def update(self, name, chosen_static_ip=None, chosen_subnet=None, chosen_gateway ec2_region=self.ec2_region, ec2_subnet_id=self.ec2_subnet_id, vlan=self.vlan, - chosen_interface=self.chosen_interface, + interface=self.interface, name=name, chosen_static_ip=chosen_static_ip, chosen_subnet=chosen_subnet, chosen_gateway=chosen_gateway, chosen_dns=chosen_dns, - dns_address=dns_address + dns_address=dns_address, + network_type=self.network_type ) non_empty_data = {key: value for key, value in data.iteritems() if value} self._update_item(non_empty_data) @@ -237,28 +244,30 @@ def power_on(self): """ Start the decoy machine. """ - self._api_client.api_request("{}{}".format(self.url, 'power_on/'), 'post') + self._api_client.api_request("{}{}".format(self.url, "power_on/"), "post") def recreate(self): """ Recreate the decoy machine. """ - self._api_client.api_request("{}{}".format(self.url, 'recreate/'), 'post') + self._api_client.api_request("{}{}".format(self.url, "recreate/"), "post") def power_off(self): """ Shut down the decoy machine. """ - self._api_client.api_request("{}{}".format(self.url, 'power_off/'), 'post') + self._api_client.api_request("{}{}".format(self.url, "power_off/"), "post") def test_dns(self): """ Check whether the decoy is properly registered in the DNS server. """ try: - return self._api_client.api_request("{}{}".format(self.url, 'test_dns/'), 'post') + return self._api_client.api_request("{}{}".format(self.url, "test_dns/"), "post") except ValidationError as e: - if str(e) == '{"non_field_errors":["Failed to resolve address for decoy"]}': + data = json.loads(e.message) + errors = data.get("non_field_errors", []) + if len(errors) == 1 and errors[0].startswith("Failed to resolve address for decoy"): return False raise @@ -268,12 +277,8 @@ def download(self, location_with_name): :param location_with_name: Destination path. """ - response = self._api_client.api_request("{}{}".format(self.url, 'download/'), stream=True) - - file_path = "{}.{}".format(location_with_name, "ova") - with open(file_path, 'wb') as f: - response.raw.decode_content = True - shutil.copyfileobj(response.raw, f) + self._api_client.request_and_download(url="{}{}".format(self.url, "download/"), + destination_path="{}.{}".format(location_with_name, "ova")) class DecoyCollection(EditableCollection): @@ -286,11 +291,13 @@ class DecoyCollection(EditableCollection): MODEL_CLASS = Decoy def create(self, os, vm_type, name, hostname, chosen_static_ip=None, chosen_subnet=None, - chosen_gateway=None, chosen_dns=None, chosen_interface=None, vlan=None, - ec2_region=None, ec2_subnet_id=None, account=None, dns_address=''): + chosen_gateway=None, chosen_dns=None, interface=1, vlan=None, + ec2_region=None, ec2_subnet_id=None, account=None, dns_address="", network_type="PROMISC"): """ Create a decoy. + :param network_type: Network type of the decoy. Options : \ + PROMISC, NON_PROMISC, VLAN_TRUNK :param os: OS installed on the server. Options: \ Ubuntu_1404, Windows_7, Windows_Server_2012, Windows_Server_2008. :param vm_type: Server type. KVM for nested (recommended) or OVA for standalone. @@ -301,8 +308,7 @@ def create(self, os, vm_type, name, hostname, chosen_static_ip=None, chosen_subn :param chosen_subnet: Decoy subnet mask. :param chosen_gateway: Decoy default gateway address. :param chosen_dns: The DNS server address (This is NOT the name of the decoy). - :param chosen_interface: The physical interface to which the decoy should be connected. \ - Applicable for non-promiscuous mode only. + :param interface: The physical interface to which the decoy should be connected. \ :param vlan: VLAN to which the decoy will be connected (if applicable). :param ec2_region: EC2 region (e.g., eu-west-1), if applicable. :param ec2_subnet_id: EC2 subnet ID, if applicable. @@ -319,12 +325,13 @@ def create(self, os, vm_type, name, hostname, chosen_static_ip=None, chosen_subn chosen_subnet=chosen_subnet, chosen_gateway=chosen_gateway, chosen_dns=chosen_dns, - chosen_interface=chosen_interface, + interface=interface, vlan=vlan, ec2_region=ec2_region, ec2_subnet_id=ec2_subnet_id, account=account, - dns_address=dns_address + dns_address=dns_address, + network_type=network_type ) non_empty_data = {key: value for key, value in data.iteritems() if value} return self.create_item(non_empty_data) @@ -343,11 +350,11 @@ class Service(Entity): * Remote desktop """ - NAME = 'service' + NAME = "service" RELATED_COLLECTIONS = { - 'attached_decoys': Decoy, - 'available_decoys': Decoy + "attached_decoys": Decoy, + "available_decoys": Decoy } def update(self, name, zip_file_path=None, **kwargs): @@ -358,7 +365,7 @@ def update(self, name, zip_file_path=None, **kwargs): :param zip_file_path: A file to upload, if applicable. :param kwargs: Additional relevant parameters. """ - files = {"zip_file": open(zip_file_path, 'rb')} if zip_file_path else None + files = {"zip_file": open(zip_file_path, "rb")} if zip_file_path else None data = dict( name=name, service_type=self.service_type @@ -373,8 +380,8 @@ def connect_to_decoy(self, decoy_id): :param decoy_id: The ID of the decoy to which the service should be attached. """ data = dict(decoy_id=decoy_id) - self._api_client.api_request(url="{}{}".format(self.url, 'connect_to_decoy/'), - method='post', + self._api_client.api_request(url="{}{}".format(self.url, "connect_to_decoy/"), + method="post", data=data) self.load() @@ -385,8 +392,8 @@ def detach_from_decoy(self, decoy_id): :param decoy_id: Decoy ID from which the service should be detached. """ data = dict(decoy_id=decoy_id) - self._api_client.api_request(url="{}{}".format(self.url, 'detach_from_decoy/'), - method='post', + self._api_client.api_request(url="{}{}".format(self.url, "detach_from_decoy/"), + method="post", data=data) self.load() @@ -410,7 +417,7 @@ def create(self, name, service_type, zip_file_path=None, **kwargs): :param zip_file_path: The path of a ZIP file to upload, if applicable. :param kwargs: Additional relevant parameters. """ - files = {"zip_file": open(zip_file_path, 'rb')} if zip_file_path else None + files = {"zip_file": open(zip_file_path, "rb")} if zip_file_path else None data = dict( name=name, service_type=service_type @@ -433,7 +440,7 @@ class DeploymentGroup(Entity): the deployment group's associated breadcrumbs on the deployment group's associated endpoints. """ - NAME = 'deployment-group' + NAME = "deployment-group" def update(self, name, description): """ @@ -471,7 +478,7 @@ def check_conflicts(self, os): :param os: OS type (Windows/Linux). """ query_dict = dict(os=os) - return self._api_client.api_request(url="{}{}".format(self.url, 'check_conflicts/'), + return self._api_client.api_request(url="{}{}".format(self.url, "check_conflicts/"), query_params=query_dict) def deploy(self, location_with_name, os, download_type, download_format="ZIP"): @@ -484,16 +491,12 @@ def deploy(self, location_with_name, os, download_type, download_format="ZIP"): :param download_format: Installer format (ZIP/MSI/EXE). """ query_dict = dict(os=os, download_type=download_type, download_format=download_format) - response = self._api_client.api_request( - "{}{}".format(self.url, 'deploy/'), - query_params=query_dict, stream=True) - file_path = "{}.{}".format(location_with_name, download_format.lower()) - with open(file_path, 'wb') as f: - response.raw.decode_content = True - shutil.copyfileobj(response.raw, f) + self._api_client.request_and_download(url="{}{}".format(self.url, "deploy/"), + destination_path=file_path, + query_params=query_dict) - def auto_deploy(self, install_method, run_method, username, password, domain='', + def auto_deploy(self, install_method, run_method, username, password, domain="", deploy_on="all"): """ Deploy all the breadcrumbs that are members of this deployment group on all the endpoints @@ -517,7 +520,7 @@ def auto_deploy(self, install_method, run_method, username, password, domain='', domain=domain, deploy_on=deploy_on ) - self._api_client.api_request("{}{}".format(self.url, 'auto_deploy/'), 'post', data=data) + self._api_client.api_request("{}{}".format(self.url, "auto_deploy/"), "post", data=data) class DeploymentGroupCollection(EditableCollection): @@ -528,6 +531,7 @@ class DeploymentGroupCollection(EditableCollection): """ MODEL_CLASS = DeploymentGroup + ALL_BREADCRUMBS_DEPLOYMENT_GROUP_ID = 1 def create(self, name, description=None): """ @@ -566,7 +570,7 @@ def test_deployment_credentials(self, addr, install_method, username, password, ) return self._api_client.api_request(url="{}{}".format(self._get_url(), "test_deployment_credentials/"), - method='post', data=data) + method="post", data=data) def auto_deploy_groups(self, deployment_groups_ids, install_method, run_method, username, password, domain=None, deploy_on="all"): @@ -594,9 +598,10 @@ def auto_deploy_groups(self, deployment_groups_ids, install_method, run_method, domain=domain, deploy_on=deploy_on ) - self._api_client.api_request(url="{}{}".format(self._get_url(), "auto_deploy_groups/"), - method='post', - data=data) + url = "{}{}/".format(self._get_url(), "deploy") + return self._api_client.api_request(url=url, + method="post", + data=data) def deploy_all(self, location_with_name, os, download_format="ZIP"): """ @@ -606,16 +611,10 @@ def deploy_all(self, location_with_name, os, download_format="ZIP"): :param os: OS for which the installation is intended. :param download_format: Installer format (ZIP/MSI/EXE). """ - query_dict = dict(os=os, download_format=download_format) - response = self._api_client.api_request( - url="{}{}".format(self._get_url(), 'deploy_all/'), - query_params=query_dict, - stream=True) - file_path = "{}.{}".format(location_with_name, download_format.lower()) - with open(file_path, 'wb') as f: - response.raw.decode_content = True - shutil.copyfileobj(response.raw, f) + self._api_client.request_and_download(url="{}{}".format(self._get_url(), "deploy_all/"), + destination_path=file_path, + query_params=dict(os=os, download_format=download_format)) def params(self): raise NotImplementedError @@ -645,12 +644,12 @@ class Breadcrumb(Entity): - A cookie with a session token, stored in the endpoint's browser. """ - NAME = 'breadcrumb' + NAME = "breadcrumb" RELATED_COLLECTIONS = { - 'attached_services': Service, - 'available_services': Service, - 'deployment_groups': DeploymentGroup + "attached_services": Service, + "available_services": Service, + "deployment_groups": DeploymentGroup } def update(self, name, **kwargs): @@ -675,8 +674,8 @@ def connect_to_service(self, service_id): :param service_id: The service ID to which the breadcrumb should be attached. """ data = dict(service_id=service_id) - self._api_client.api_request(url="{}{}".format(self.url, 'connect_to_service/'), - method='post', + self._api_client.api_request(url="{}{}".format(self.url, "connect_to_service/"), + method="post", data=data) self.load() @@ -687,8 +686,8 @@ def detach_from_service(self, service_id): :param service_id: Service ID from which the breadcrumbs should be detached. """ data = dict(service_id=service_id) - self._api_client.api_request(url="{}{}".format(self.url, 'detach_from_service/'), - method='post', + self._api_client.api_request(url="{}{}".format(self.url, "detach_from_service/"), + method="post", data=data) self.load() @@ -702,14 +701,10 @@ def deploy(self, location_with_name, os, download_type, download_format="ZIP"): :param download_format: Installer format (ZIP/EXE/MSI). """ query_dict = dict(os=os, download_type=download_type, download_format=download_format) - response = self._api_client.api_request( - "{}{}".format(self.url, 'deploy/'), - query_params=query_dict, stream=True) - file_path = "{}.{}".format(location_with_name, download_format.lower()) - with open(file_path, 'wb') as f: - response.raw.decode_content = True - shutil.copyfileobj(response.raw, f) + self._api_client.request_and_download(url="{}{}".format(self.url, "deploy/"), + destination_path=file_path, + query_params=query_dict) def add_to_group(self, deployment_group_id): """ @@ -719,10 +714,10 @@ def add_to_group(self, deployment_group_id): """ if not isinstance(deployment_group_id, Number): - raise BadParamError('deployment_group_id must be a number') + raise BadParamError("deployment_group_id must be a number") data = dict(deployment_group_id=deployment_group_id) - self._api_client.api_request("{}{}".format(self.url, 'add_to_group/'), 'post', data=data) + self._api_client.api_request("{}{}".format(self.url, "add_to_group/"), "post", data=data) self.load() def remove_from_group(self, deployment_group_id): @@ -732,11 +727,20 @@ def remove_from_group(self, deployment_group_id): :param deployment_group_id: Deployment group ID from which the breadcrumb should be removed. """ data = dict(deployment_group_id=deployment_group_id) - self._api_client.api_request(url="{}{}".format(self.url, 'remove_from_group/'), - method='post', + self._api_client.api_request(url="{}{}".format(self.url, "remove_from_group/"), + method="post", data=data) self.load() + def download_breadcrumb_honeydoc(self, location_with_name): + """ + Download the HoneyDoc file for this breadcrumb (for HoneyDoc breadcrumbs only). + + :param location_with_name: Destination path. + """ + self._api_client.request_and_download(url="{}{}".format(self.url, "download_breadcrumb_honeydoc/"), + destination_path=location_with_name) + class BreadcrumbCollection(EditableCollection): """ @@ -747,22 +751,114 @@ class BreadcrumbCollection(EditableCollection): MODEL_CLASS = Breadcrumb - def create(self, name, breadcrumb_type, **kwargs): + def create(self, name, breadcrumb_type, file_field_name=None, file_path=None, **kwargs): """ Create a new breadcrumb. :param name: An internal name for the breadcrumb. :param breadcrumb_type: The type of breadcrumb. See options for a list \ of available breadcrumb types. + :param file_field_name: For breadcrumbs requiring a file, the name of the + breadcrumb field expected to contain the file content. + :param file_path: For breadcrumbs requiring a file, the path to the file to upload. :param kwargs: Other parameters relevant for the desired breadcrumb type. See options \ for more information. """ + files = {file_field_name: open(file_path, "rb")} if file_field_name else None data = dict( name=name, breadcrumb_type=breadcrumb_type ) data.update(kwargs) - return self.create_item(data) + return self.create_item(data, files=files) + + +class AlertProcessDLL(BaseEntity): + """ + A DLL file that was used by an attacker's process. + """ + def download_file(self, destination_path): + """ + Download the DLL file to the local disk + + :param destination_path: Location on the disk where you want to save the file. + """ + self._api_client.request_and_download( + url="{url}{download_file_suffix}/".format(url=self.url, + download_file_suffix="download_file"), + destination_path=destination_path) + + +class AlertProcessDLLCollection(Collection): + """ + DLL files associated with a specific binary attack tool, fetched by the Forensic Puller. + + This entity will be returned by calling :py:meth:`AlertProcess.get_dlls` + """ + + MODEL_CLASS = AlertProcessDLL + URL_SUFFIX = "dll" + + def __init__(self, api_client, alert_process, obj_class=None): + super(AlertProcessDLLCollection, self).__init__(api_client, obj_class) + self.alert_process = alert_process + + def _get_url(self): + return "{process_url}{suffix}/".format(process_url=self.alert_process.url, + suffix=self.URL_SUFFIX) + + +class AlertProcess(BaseEntity): + """ + A suspicious process that has been detected on an attacker's endpoint by the Forensic Puller. + + For more information about the Forensic Puller, navigate to User Menu > User Manual. + """ + + def get_dlls(self): + """ + Get a generator of the DLL files that were used by the process. + """ + return AlertProcessDLLCollection(self._api_client, alert_process=self) + + def download_file(self, destination_path): + """ + Download the attacker's tool to your local disk + + :param destination_path: Location on the disk where you want to save the file. + """ + self._api_client.request_and_download( + url="{url}{download_file_suffix}/".format(url=self.url, + download_file_suffix="download_file"), + destination_path=destination_path) + + def download_minidump(self, destination_path): + """ + Download minidump of the attacker's process + + :param destination_path: Location on the disk where you want to save the file. + """ + self._api_client.request_and_download( + url="{url}{download_file_suffix}/".format(url=self.url, + download_file_suffix="download_minidump"), + destination_path="{}.dump".format(destination_path)) + + +class AlertProcessCollection(Collection): + """ + A subset of the processes associated with a specific alert. + + This entity will be returned by calling :py:meth:`api_client.Alert.get_processes` + """ + MODEL_CLASS = AlertProcess + URL_SUFFIX = "process" + + def __init__(self, api_client, alert, obj_class=None): + super(AlertProcessCollection, self).__init__(api_client, obj_class) + self.alert = alert + + def _get_url(self): + return "{alert_url}{suffix}/".format(alert_url=self.alert.url, suffix=self.URL_SUFFIX) class Alert(BaseEntity): @@ -774,13 +870,14 @@ class Alert(BaseEntity): which query was run on the DB, which SMB shares were accessed, etc. """ - NAME = 'alert' + NAME = "alert" + PROCESSES_URL_SUFFIX = "processes" def delete(self): """ Delete the alert """ - self._api_client.api_request(self.url, 'delete') + self._api_client.api_request(self.url, "delete") def download_image_file(self, location_with_name): """ @@ -788,13 +885,8 @@ def download_image_file(self, location_with_name): :param location_with_name: Download destination path. """ - response = self._api_client.api_request(url="{}{}".format(self.url, 'download_image_file/'), - stream=True) - - file_path = "{}.bin".format(location_with_name) - with open(file_path, 'wb') as f: - response.raw.decode_content = True - shutil.copyfileobj(response.raw, f) + self._api_client.request_and_download(url="{}{}".format(self.url, "download_image_file/"), + destination_path="{}.bin".format(location_with_name)) def download_memory_dump_file(self, location_with_name): """ @@ -802,14 +894,8 @@ def download_memory_dump_file(self, location_with_name): :param location_with_name: Download destination path. """ - response = self._api_client.api_request(url="{}{}".format(self.url, - 'download_memory_dump_file/'), - stream=True) - - file_path = "{}.bin".format(location_with_name) - with open(file_path, 'wb') as f: - response.raw.decode_content = True - shutil.copyfileobj(response.raw, f) + self._api_client.request_and_download(url="{}{}".format(self.url, "download_memory_dump_file/"), + destination_path="{}.bin".format(location_with_name)) def download_network_capture_file(self, location_with_name): """ @@ -817,14 +903,8 @@ def download_network_capture_file(self, location_with_name): :param location_with_name: Download destination path. """ - response = self._api_client.api_request(url="{}{}".format(self.url, - 'download_network_capture_file/'), - stream=True) - - file_path = "{}.pcap".format(location_with_name) - with open(file_path, 'wb') as f: - response.raw.decode_content = True - shutil.copyfileobj(response.raw, f) + self._api_client.request_and_download(url="{}{}".format(self.url, "download_network_capture_file/"), + destination_path="{}.pcap".format(location_with_name)) def download_stix_file(self, location_with_name): """ @@ -832,13 +912,16 @@ def download_stix_file(self, location_with_name): :param location_with_name: Download destination path. """ - response = self._api_client.api_request(url="{}{}".format(self.url, 'download_stix_file/'), - stream=True) + self._api_client.request_and_download(url="{}{}".format(self.url, "download_stix_file/"), + destination_path="{}.xml".format(location_with_name)) - file_path = "{}.xml".format(location_with_name) - with open(file_path, 'wb') as f: - response.raw.decode_content = True - shutil.copyfileobj(response.raw, f) + def get_processes(self): + """ + Get a generator of all the processes associated with the alert. + + Supported versions: MazeRunner 1.7.0 and above. + """ + return AlertProcessCollection(api_client=self._api_client, alert=self) class AlertCollection(Collection): @@ -850,36 +933,78 @@ class AlertCollection(Collection): MODEL_CLASS = Alert - def __init__(self, api_client, filter_enabled=False, only_alerts=False, alert_types=None): + def __init__(self, api_client, filter_enabled=False, only_alerts=False, alert_types=None, + start_date=None, end_date=None, id_greater_than=None, username=None, source=None, + keywords=None, decoy_name=None): """ :param api_client: The connection instance. :param filter_enabled: Whether this is a filtered collection. :param only_alerts: Filter out alerts in status 'Ignore' and 'Mute'. :param alert_types: E.g., code, HTTP, etc. See params() for the full list. + :param start_date: The beginning of the date range, formatted dd/mm/yyyy. + :param end_date: The end of the date range, formatted dd/mm/yyyy. + :param id_greater_than: Filter alerts to see only alerts that occur after this ID. + :param username: The breadcrumb's username, which the attacker used to log in. + :param source: The IP or hostname of the attacker's endpoint. + :param keywords: Search main fields for these keywords. + :param decoy_name: The name of the decoy that was attacked. """ super(AlertCollection, self).__init__(api_client) self.filter_enabled = filter_enabled self.only_alerts = only_alerts self.alert_types = alert_types + self.start_date = start_date + self.end_date = end_date + self.id_greater_than = id_greater_than + self.username = username + self.source = source + self.keywords = keywords + self.decoy_name = decoy_name def _get_query_params(self): return dict(filter_enabled=self.filter_enabled, only_alerts=self.only_alerts, - alert_types=self.alert_types) - - def filter(self, filter_enabled=False, only_alerts=False, alert_types=None): + alert_types=self.alert_types, + per_page=ENTRIES_PER_PAGE, + start_date=self.start_date, + end_date=self.end_date, + id_gt=self.id_greater_than, + username=self.username, + source=self.source, + keywords=self.keywords, + decoy_name=self.decoy_name) + + def filter(self, filter_enabled=False, only_alerts=False, alert_types=None, + start_date=None, end_date=None, id_greater_than=None, username=None, source=None, + keywords=None, decoy_name=None): """ Get alerts by query. - :param filter_enabled: When False, the only_alerts and alert_types params will be ignored. + :param filter_enabled: When False, all the filtering params will be ignored. :param only_alerts: Only take alerts in 'Alert' status (exclude those in 'Mute' and \ 'Ignore' status). :param alert_types: A list of alert types. + :param start_date: The beginning of the date range, formatted dd/mm/yyyy. + :param end_date: The end of the date range, formatted dd/mm/yyyy. + :param id_greater_than: Filter alerts to see only alerts that occur after this ID. + :param username: The breadcrumb's username, which the attacker used to log in. + :param source: The IP or hostname of the attacker's endpoint. + :param keywords: Search main fields for these keywords. + :param decoy_name: The name of the decoy that was attacked. :return: A filtered :class:`api_client.AlertCollection`. """ formatted_alert_types = " ".join(alert_types) if alert_types else "" - return AlertCollection(self._api_client, filter_enabled=filter_enabled, - only_alerts=only_alerts, alert_types=formatted_alert_types) + return AlertCollection(self._api_client, + filter_enabled=filter_enabled, + only_alerts=only_alerts, + alert_types=formatted_alert_types, + start_date=start_date, + end_date=end_date, + id_greater_than=id_greater_than, + username=username, + source=source, + keywords=keywords, + decoy_name=decoy_name) def export(self, location_with_name): """ @@ -887,15 +1012,9 @@ def export(self, location_with_name): :param location_with_name: Download destination file. """ - query_params = self._get_query_params() - response = self._api_client.api_request(url="{}{}".format(self._get_url(), 'export/'), - stream=True, - query_params=query_params) - - file_path = "{}.csv".format(location_with_name) - with open(file_path, 'wb') as f: - response.raw.decode_content = True - shutil.copyfileobj(response.raw, f) + self._api_client.request_and_download(url="{}{}".format(self._get_url(), "export/"), + destination_path="{}.csv".format(location_with_name), + query_params=self._get_query_params()) def delete(self, selected_alert_ids=None, delete_all_filtered=False): """ @@ -914,37 +1033,88 @@ def delete(self, selected_alert_ids=None, delete_all_filtered=False): Example 2: Delete alerts by filter:: client = mazerunner.connect(...) - filtered_alerts = client.alerts.filter(alert_types=['share', 'http']) + filtered_alerts = client.alerts.filter(alert_types=["share", "http"]) filtered_alerts.delete(delete_all_filtered=True) """ data = dict(selected_alert_ids=selected_alert_ids, delete_all_filtered=delete_all_filtered) query_params = self._get_query_params() - self._api_client.api_request(url="{}{}".format(self._get_url(), 'delete_selected/'), - method='post', + self._api_client.api_request(url="{}{}".format(self._get_url(), "delete_selected/"), + method="post", data=data, query_params=query_params) +class ForensicPullerOnDemand(BaseCollection): + """ + Forensic Puller on demand. + + This entity will be returned by :py:attr:`api_client.APIClient.forensic_puller_on_demand`. + """ + URL_EXTENSION = "forensic-puller-on-demand-run" + + def run_on_ip_list(self, ip_list): + """ + Runs Forensic Puller on a list of IPs. + + :param ip_list: List of IPs. + """ + data = dict(ip_list=ip_list) + self._api_client.api_request( + url=self._get_url(), + method="post", + data=data) + + +class StorageUsageData(BaseCollection): + """ + Storage usage data. + This entity will be returned by :py:attr:`api_client.APIClient.storage_usage_data`. + """ + URL_EXTENSION = "storage-usage" + + def __unicode__(self): + return unicode(self.details()) + + def __str__(self): + return str(self.details()) + + def details(self): + return self._api_client.api_request(url=self._get_url(), method="get") + + def _get_url(self): + return self._api_client.api_urls[self.URL_EXTENSION] + + class Endpoint(Entity): """ An endpoint represents a single workstation in the organization, and the status - of the breadcrumbs' deployment on it. + of the breadcrumbs' deployment to it. """ - NAME = 'endpoint' + NAME = "endpoint" + + RELATED_FIELDS = { + "deployment_group": DeploymentGroup, + } def delete(self): """ Delete the endpoint. """ base_url = self._api_client.api_urls[self.NAME] - url = '%sdelete_selected/?filter_enabled=true' % base_url + url = "%sdelete_selected/?filter_enabled=true" % base_url data = {"selected_endpoints_ids": [self.id]} - self._api_client.api_request(url, 'post', data=data) + self._api_client.api_request(url, "post", data=data) + def reassign_to_group(self, deployment_group): + self._api_client.endpoints.reassign_to_group(deployment_group, [self]) -class EndpointCollection(Collection): + def clear_deployment_group(self): + self._api_client.endpoints.clear_deployment_group([self]) + + +class EndpointCollection(EditableCollection): """ A subset of the endpoints in the system. @@ -952,17 +1122,18 @@ class EndpointCollection(Collection): """ MODEL_CLASS = Endpoint + UNASSIGN_FROM_DEPLOYMENT_GROUP = "unassigned" RUN_METHOD_FOR_INSTALL_METHOD = { - 'ZIP': 'CMD_DEPLOY', - 'EXE': 'EXE_DEPLOY', - 'MSI': 'EXE_DEPLOY' + "ZIP": "CMD_DEPLOY", + "EXE": "EXE_DEPLOY", + "MSI": "EXE_DEPLOY" } def __init__(self, api_client, filter_enabled=False, - keywords='', + keywords="", statuses=None, deploy_groups=None): """ @@ -978,12 +1149,26 @@ def __init__(self, self.statuses = statuses self.deploy_groups = deploy_groups + def create(self, ip_address=None, dns=None, hostname=None, deployment_group_id=None): + """ + Create an endpoint. + + Pass at least one of the following parameters: ip_address, dns, or hostname. + + :param deployment_group_id: Id of the deployment group. + :param ip_address: Address of the endpoint. + :param dns: FQDN of the endpoint. + :param hostname: Hostname of the endpoint. + """ + data = dict(ip_address=ip_address, dns=dns, hostname=hostname, deployment_group=deployment_group_id) + return self.create_item(data=data) + def _get_query_params(self): return { - 'filter_enabled': self.filter_enabled, - 'keywords': self.keywords, - 'statuses': self.statuses, - 'deploy_groups': self.deploy_groups + "filter_enabled": self.filter_enabled, + "keywords": self.keywords, + "statuses": self.statuses, + "deploy_groups": self.deploy_groups } def filter(self, keywords=""): @@ -1009,20 +1194,35 @@ def reassign_to_group(self, deployment_group, endpoints): selected_endpoints_ids=[e.id for e in endpoints]) self._api_client.api_request( - "{}{}".format(self._get_url(), 'reassign_selected/'), - method='POST', + "{}{}".format(self._get_url(), "reassign_selected/"), + method="POST", + data=data) + + def clear_deployment_group(self, endpoints): + """ + Unassign specified endpoints from all deployment groups. + + :param endpoints: A list of endpoints that should be unassigned. + """ + data = dict( + to_group=self.UNASSIGN_FROM_DEPLOYMENT_GROUP, + selected_endpoints_ids=[ep.id for ep in endpoints]) + + self._api_client.api_request( + "{}{}".format(self._get_url(), "reassign_selected/"), + method="POST", data=data) def _get_run_method(self, install_method): if install_method not in self.RUN_METHOD_FOR_INSTALL_METHOD: - raise InvalidInstallMethodError('Invalid install method: %s' % install_method) + raise InvalidInstallMethodError("Invalid install method: %s" % install_method) return self.RUN_METHOD_FOR_INSTALL_METHOD[install_method.upper()] def clean_filtered(self, install_method, username, password, - domain=''): + domain=""): """ Uninstall breadcrumbs from all of the endpoints matching the filter. @@ -1032,16 +1232,18 @@ def clean_filtered(self, :param domain: The domain where that user is registered. Leave blank for local user. """ - self._api_client.api_request(url='{}{}'.format(self._get_url(), 'clean_selected/'), - method='POST', + self._api_client.api_request(url="{}{}".format(self._get_url(), "clean_selected/"), + method="POST", query_params=self._get_query_params(), data={ - 'clean_all_filtered': True, - 'username': username, - 'password': password, - 'domain': domain, - 'run_method': self._get_run_method(install_method), - 'install_method': install_method + "clean_all_filtered": True, + "username": username, + "password": { + "value": password + }, + "domain": domain, + "run_method": self._get_run_method(install_method), + "install_method": install_method }) def clean_by_endpoints_ids(self, @@ -1049,7 +1251,7 @@ def clean_by_endpoints_ids(self, install_method, username, password, - domain=''): + domain=""): """ Uninstall breadcrumbs from all of the specified endpoint IDs. @@ -1059,26 +1261,28 @@ def clean_by_endpoints_ids(self, :param password: Password for that user. :param domain: The domain where that user is registered. Leave blank for local user. """ - self._api_client.api_request(url='{}{}'.format(self._get_url(), 'clean_selected/'), - method='POST', + self._api_client.api_request(url="{}{}".format(self._get_url(), "clean_selected/"), + method="POST", data={ - 'selected_endpoints_ids': endpoints_ids, - 'username': username, - 'password': password, - 'domain': domain, - 'run_method': self._get_run_method(install_method), - 'install_method': install_method + "selected_endpoints_ids": endpoints_ids, + "username": username, + "password": { + "value": password + }, + "domain": domain, + "run_method": self._get_run_method(install_method), + "install_method": install_method }) def delete_filtered(self): """ Delete all the endpoints matching the filter. """ - self._api_client.api_request(url='{}{}'.format(self._get_url(), 'delete_selected/'), - method='POST', + self._api_client.api_request(url="{}{}".format(self._get_url(), "delete_selected/"), + method="POST", query_params=self._get_query_params(), data={ - 'delete_all_filtered': True + "delete_all_filtered": True }) def delete_by_endpoints_ids(self, endpoints_ids): @@ -1087,17 +1291,17 @@ def delete_by_endpoints_ids(self, endpoints_ids): :param endpoints_ids: List of the endpoint IDs to be deleted. """ - self._api_client.api_request(url='{}{}'.format(self._get_url(), 'delete_selected/'), - method='POST', + self._api_client.api_request(url="{}{}".format(self._get_url(), "delete_selected/"), + method="POST", data={ - 'selected_endpoints_ids': endpoints_ids, + "selected_endpoints_ids": endpoints_ids, }) def export_filtered(self): """ Export all filtered endpoints to CSV. """ - return self._api_client.api_request(url='{}{}'.format(self._get_url(), 'export/'), + return self._api_client.api_request(url="{}{}".format(self._get_url(), "export/"), query_params=self._get_query_params(), expect_json_response=False) @@ -1105,13 +1309,7 @@ def filter_data(self): """ Get the available values for the endpoint filters. """ - return self._api_client.api_request(url='{}{}'.format(self._get_url(), 'filter_data/')) - - def status_dashboard(self): - """ - Get a list of all endpoints and their statuses. - """ - return self._api_client.api_request(url='{}{}'.format(self._get_url(), 'status_dashboard/')) + return self._api_client.api_request(url="{}{}".format(self._get_url(), "filter_data/")) def params(self): raise NotImplementedError @@ -1124,13 +1322,13 @@ class BackgroundTask(BaseEntity): Examples of requests that create background tasks include deployment on endpoints, and importing the organization structure from Active Directory. """ - NAME = 'background-task' + NAME = "background-task" def stop(self): """ Stop task. """ - self._api_client.api_request("{}{}".format(self.url, 'stop/'), 'post') + self._api_client.api_request("{}{}".format(self.url, "stop/"), "post") class BackgroundTaskCollection(Collection): @@ -1167,7 +1365,7 @@ def acknowledge_all_complete(self): """ Acknowledge all tasks with the status 'stopped' or 'complete'. """ - self._api_client.api_request("{}{}".format(self._get_url(), 'acknowledge_all/'), 'post') + self._api_client.api_request("{}{}".format(self._get_url(), "acknowledge_all/"), "post") def params(self): raise NotImplementedError @@ -1182,7 +1380,7 @@ class AlertPolicy(BaseEntity): - 1 = Mute - 2 = Alert """ - NAME = 'alert-policy' + NAME = "alert-policy" def update_to_status(self, to_status): """ @@ -1191,7 +1389,7 @@ def update_to_status(self, to_status): :param to_status: The name of the new 'to_status' of the policy. """ data = dict(to_status=to_status) - response = self._api_client.api_request(self.url, 'put', data=data) + response = self._api_client.api_request(self.url, "put", data=data) self._update_entity_data(response) @@ -1218,7 +1416,7 @@ def reset_all_to_default(self): """ Reset the 'to_status' of all alert policies to their original system default. """ - self._api_client.api_request("{}{}".format(self._get_url(), 'reset_all/'), 'post') + self._api_client.api_request("{}{}".format(self._get_url(), "reset_all/"), "post") class CIDRMapping(BaseEntity): @@ -1230,23 +1428,23 @@ class CIDRMapping(BaseEntity): mapping, the daily CIDR block importer will also assign that deployment group to endpoints that were just imported or did not have one configured. """ - NAME = 'cidr-mapping' + NAME = "cidr-mapping" def generate_endpoints(self): """ Scan the CIDR block and import the endpoints. """ - return self._api_client.api_request('{}{}'.format(self.url, 'generate_endpoints/'), - method='post', + return self._api_client.api_request("{}{}".format(self.url, "generate_endpoints/"), + method="post", data={ - 'reassign': False + "reassign": False }) def delete(self): """ Delete this record. """ - self._api_client.api_request(self.url, 'delete') + self._api_client.api_request(self.url, "delete") class CIDRMappingCollection(UnpaginatedEditableCollection): @@ -1269,21 +1467,21 @@ def create(self, cidr_block, deployment_group, comments, active): :param active: Whether this block should be included in the import. """ return self.create_item({ - 'cidr_block': cidr_block, - 'deployment_group': deployment_group, - 'comments': comments, - 'active': active + "cidr_block": cidr_block, + "deployment_group": deployment_group, + "comments": comments, + "active": active }) def generate_all_endpoints(self): """ Scan all the active CIDR blocks in the system and import all of their endpoints. """ - return self._api_client.api_request('{}{}'.format(self._get_url(), - 'generate_all_endpoints/'), - method='post', + return self._api_client.api_request("{}{}".format(self._get_url(), + "generate_all_endpoints/"), + method="post", data={ - 'reassign': False + "reassign": False }) def params(self): @@ -1295,7 +1493,7 @@ class ActiveSOCEvent(Entity): A message to be sent to a SOC interface. """ - NAME = 'api-soc' + NAME = "api-soc" class ActiveSOCEventCollection(EditableCollection): @@ -1331,7 +1529,7 @@ def create_multiple_events(self, soc_name, events_dicts): source=soc_name, data=events_dicts ) - self._api_client.api_request(self._get_url(), 'post', data=data) + self._api_client.api_request(self._get_url(), "post", data=data) def params(self): raise NotImplementedError @@ -1343,6 +1541,67 @@ def get_item(self, id): raise NotImplementedError +class AuditLogLine(Entity): + """ + A message from the server's audit log + """ + NAME = "audit-log" + + +class AuditLogLineCollection(Collection): + """ + Use this to access MazeRunner's audit log. + + This entity will be returned by :py:attr:`api_client.APIClient.audit_log`. + """ + MODEL_CLASS = AuditLogLine + + def __init__(self, api_client, filter_enabled=False, item=None, username=None, event_type=None, keywords=None, + start_date=None, end_date=None, object_ids=None, category=None): + super(AuditLogLineCollection, self).__init__(api_client) + self.filter_enabled = filter_enabled + self.end_date = end_date + self.start_date = start_date + self.username = username + self.category = category + self.event_type = event_type + self.object_ids = object_ids + self.item = item + self.keywords = keywords + + def _get_query_params(self): + return {'filter_enabled': self.filter_enabled, + 'end_date': self.end_date, + 'start_date': self.start_date, + 'per_page': ENTRIES_PER_PAGE, + 'username[]': self.username, + 'category[]': self.category, + 'event_type[]': self.event_type, + 'object_ids': self.object_ids, + 'item': self.item} + + def filter(self, filter_enabled=True, item=None, username=None, event_type=None, + start_date=None, end_date=None, object_ids=None, category=None): + + for param_name, param in dict(username=username, + event_type=event_type, + category=category).iteritems(): + if param and type(param) != list: + raise BadParamError("{} has to be a list!".format(param_name)) + return AuditLogLineCollection(self._api_client, + filter_enabled=filter_enabled, + end_date=end_date, + start_date=start_date, + username=username, + category=category, + event_type=event_type, + object_ids=object_ids, + item=item) + + def delete(self): + self._api_client.api_request("".join([self._get_url(), "delete_audit_log/"]), method="post") + + class APIClient(object): """ This is the starting point for any interaction with MazeRunner. @@ -1364,10 +1623,10 @@ class APIClient(object): Example:: - client = mazerunner.connect(ip_address='1.2.3.4', - api_key='my-api-key', - api_secret='my-api-secret', - certificate='/path/to/MazeRunner.crt') + client = mazerunner.connect(ip_address="1.2.3.4", + api_key="my-api-key", + api_secret="my-api-secret", + certificate="/path/to/MazeRunner.crt") my_service = client.services.get_item(id=8) """ @@ -1385,13 +1644,13 @@ def __init__(self, host, api_key, api_secret, certificate): self._certificate = False else: self._certificate = certificate - self._base_url = 'https://%(host)s' % dict(host=host) + self._base_url = "https://%(host)s" % dict(host=host) self._session = requests.Session() - self.api_urls = self.api_request('/api/v1.0/') + self.api_urls = self.api_request("/api/v1.0/") def api_request(self, url, - method='get', + method="get", query_params=None, data=None, files=None, @@ -1414,6 +1673,22 @@ def api_request(self, if not url.startswith("http"): url = self._base_url + url + parsed = urlparse.urlparse(url) + parsed_no_query = urlparse.ParseResult( + scheme=parsed.scheme, + netloc=parsed.netloc, + path=parsed.path, + params=parsed.params, + query="", + fragment=parsed.fragment) + url = urlparse.urlunparse(parsed_no_query) + query = {query_param_name: set(query_param_value) + for query_param_name, query_param_value + in urlparse.parse_qs(parsed.query).items()} + if query_params: + query.update(query_params) + query_params = query + request_args = dict( method=method, url=url, @@ -1421,10 +1696,10 @@ def api_request(self, auth=self._auth ) if not files: - request_args['json'] = data + request_args["json"] = data else: - request_args['data'] = data - request_args['files'] = files + request_args["data"] = data + request_args["files"] = files req = requests.Request(**request_args) resp = self._session.send(req.prepare(), verify=self._certificate, stream=stream) @@ -1439,7 +1714,7 @@ def api_request(self, return resp if method == "delete" or resp.status_code == NO_CONTENT: - return + return None content_type = resp.headers.get("Content-Type", None) @@ -1461,6 +1736,12 @@ def api_request(self, "Bad response: Not json.\nContent:\n{content}".format(content=resp.content) ) + def request_and_download(self, url, destination_path, query_params=None): + data = self.api_request(url, stream=True, query_params=query_params) + with open(destination_path, "wb") as f: + data.raw.decode_content = True + shutil.copyfileobj(data.raw, f) + @property def decoys(self): """ @@ -1471,10 +1752,10 @@ def decoys(self): client = mazerunner.connect(...) backup_server_story_decoy = client.decoys.create( - name='backup_server_decoy', - os='Windows_Server_2012', - hostname: 'backup_server', - vm_type: "KVM") + name="backup_server_decoy", + os="Windows_Server_2012", + hostname="backupserver", + vm_type="KVM") old_decoy = client.decoys.get_item(id=5) old_decoy.delete() @@ -1491,8 +1772,8 @@ def services(self): client = mazerunner.connect(...) app_db_service = client.services.create( - name='app_db_service', - type='mysql') + name="app_db_service", + service_type="mysql") """ return ServiceCollection(self) @@ -1506,7 +1787,7 @@ def deployment_groups(self): client = mazerunner.connect(...) hr_deployment_group = client.deployment_groups.create( - name='breadcrumbs_for_hr_machines') + name="breadcrumbs_for_hr_machines") """ return DeploymentGroupCollection(self) @@ -1520,10 +1801,10 @@ def breadcrumbs(self): client = mazerunner.connect(...) mysql_breadcrumb = client.breadcrumbs.create( - breadcrumb_type='mysql', - name='mysql_breadcrumb', - deploy_for='root', - installation_type='mysql_history') + breadcrumb_type="mysql", + name="mysql_breadcrumb", + deploy_for="root", + installation_type="mysql_history") """ return BreadcrumbCollection(self) @@ -1536,10 +1817,30 @@ def alerts(self): Example:: client = mazerunner.connect(...) - code_alerts = client.alerts.filter(alert_types=['code']) + code_alerts = client.alerts.filter(alert_types=["code"]) """ return AlertCollection(self) + @property + def forensic_puller_on_demand(self): + """ + Get an :class:`api_client.ForensicPullerOnDemand` instance, on which you can + perform read and delete operations. + + Example:: + + client = mazerunner.connect(...) + code_alerts = client.forensic_puller_on_demand.run_on_ip_list(ip_list=["192.168.1.1"]) + """ + return ForensicPullerOnDemand(self) + + @property + def storage_usage(self): + """ + Get an :class:`api_client.StorageUsageData` + """ + return StorageUsageData(self) + @property def endpoints(self): """ @@ -1588,12 +1889,12 @@ def active_soc_events(self): Example:: client = mazerunner.connect(...) - self.active_soc_events.create_multiple_events('my-soc-interface-name', [{ - 'ComputerName': 'TEST_ENDPOINT1', - 'EventCode': 4625 + self.active_soc_events.create_multiple_events("my-soc-interface-name", [{ + "ComputerName": "TEST_ENDPOINT1", + "EventCode": 4625 },{ - 'ComputerName': 'TEST_ENDPOINT2', - 'EventCode': 529 + "ComputerName": "TEST_ENDPOINT2", + "EventCode": 529 }]) """ return ActiveSOCEventCollection(self) @@ -1608,10 +1909,14 @@ def cidr_mappings(self): client = mazerunner.connect(...) developers_segment = client.cidr_mapping.create( - cidr_block='192.168.5.0/24', + cidr_block="192.168.5.0/24", deployment_group=5, - comments='R&D', + comments="R&D", active=True) developers_segment.generate_endpoints() """ return CIDRMappingCollection(self) + + @property + def audit_log(self): + return AuditLogLineCollection(self) diff --git a/mazerunner/samples/assign_group_to_cidr.py b/mazerunner/samples/assign_group_to_cidr.py deleted file mode 100755 index 4310653..0000000 --- a/mazerunner/samples/assign_group_to_cidr.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python2 - -import argparse -from socket import gethostbyaddr -from netaddr import IPNetwork -import mazerunner - - -def get_hostname_for_ip(ip_address): - name = None - try: - name, alias, addresslist = gethostbyaddr(ip_address) - except Exception: - pass - return name - - -def find_deployment_group(client, group_name): - for group in client.deployment_groups: - if group.name == group_name: - return group - - raise ValueError('Deployment group "{}" could not be found'.format(group_name)) - - -def find_endpoint(endpoints, hostname): - hostname = hostname.lower() - for x in endpoints: - if (x.dns and x.dns.lower() == hostname) or \ - (x.hostname and x.hostname.lower() == hostname): - return x - return None - - -def assign_group_to_cidr(deployment_group, cidr, connection_params): - network = IPNetwork(cidr) - - client = mazerunner.connect(**connection_params) - group = find_deployment_group(client, deployment_group) - - for address in network.iter_hosts(): - name = get_hostname_for_ip(str(address)) - if not name: - print "Could not resolve hostname for {}".format(address) - continue - # Find the endpoint object - endpoints = client.endpoints.filter(name) - e = find_endpoint(endpoints, name) - if not e: - print "Could not find endpoint object for {}".format(name) - continue - try: - client.endpoints.reassign_to_group(group, [e]) - except mazerunner.exceptions.ValidationError: - # Workaround... - pass - - print "Endpoint at {} ({}) assigned to group {}".format(address, name, deployment_group) - - -def main(): - parser = argparse.ArgumentParser("Assign deployment group to all endpoints in a CIDR range") - parser.add_argument('mazerunner', help="IP address of MazeRunner management server") - parser.add_argument('api_key', help="API key") - parser.add_argument('api_secret', help="API secert key") - parser.add_argument('certificate', help="Path to MazeRunner's SSL certificate") - parser.add_argument('deployment_group', help="Name of the deployment group to assign to") - parser.add_argument('cidr', help='CIDR range to assign (e.g. 192.168.0.0/24)') - - args = parser.parse_args() - - connection_params = dict(ip_address=args.mazerunner, - api_key=args.api_key, - api_secret=args.api_secret, - certificate=args.certificate) - assign_group_to_cidr(args.deployment_group, args.cidr, connection_params) - - -if __name__ == '__main__': - main() diff --git a/mazerunner/samples/create_deployment_group_with_all_breadcrumb_types.py b/mazerunner/samples/create_deployment_group_with_all_breadcrumb_types.py index ab7359d..b97c3f6 100644 --- a/mazerunner/samples/create_deployment_group_with_all_breadcrumb_types.py +++ b/mazerunner/samples/create_deployment_group_with_all_breadcrumb_types.py @@ -1,7 +1,7 @@ -''' -This sample script creates a deployment group with each of the available breadcrumb types +""" +This sample script creates a deployment group with the requested breadcrumb type and downloads the deployment package -''' +""" import random import argparse import zipfile @@ -10,6 +10,9 @@ import time import mazerunner +from mazerunner.exceptions import ValidationError + +COMMON_SAMPLE_PASSWORD = ['password', '12345678', 'xyzvbnm,', 'qwertyui'] def _create_temp_file(): @@ -61,7 +64,7 @@ def _create_dummy_zip_file(): 'requires_username': False, 'requires_password': False, 'args': {'browser': 'chrome', 'subservice': 'phpmyadmin'}, - 'required_service':{'type': 'http', 'decoy': 'DMZ Server', 'args': { + 'required_service': {'type': 'http', 'decoy': 'DMZ Server', 'args': { 'web_apps': ['phpmyadmin'], 'zip_file_path': _create_dummy_zip_file() }} }, @@ -69,7 +72,7 @@ def _create_dummy_zip_file(): 'requires_username': True, 'requires_password': True, 'args': {}, - 'required_service':{'type': 'smb', 'decoy': 'Windows Server', 'args': { + 'required_service': {'type': 'smb', 'decoy': 'Windows Server', 'args': { 'share_name': 'transfer_files', 'zip_file_path': _create_dummy_zip_file() }} }, @@ -77,7 +80,7 @@ def _create_dummy_zip_file(): 'requires_username': True, 'requires_password': True, 'args': {}, - 'required_service':{'type': 'git', 'decoy': 'Development Server', 'args': { + 'required_service': {'type': 'git', 'decoy': 'Development Server', 'args': { 'repository_name': 'backend', 'zip_file_path': _create_dummy_zip_file() }} }, @@ -90,8 +93,8 @@ def _create_dummy_zip_file(): 'netshare': { 'requires_username': True, 'requires_password': True, - 'args': {}, - 'required_service':{'type': 'smb', 'decoy': 'Windows Server', 'args': { + 'args': {'persistence': 'non_persistent'}, + 'required_service': {'type': 'smb', 'decoy': 'Windows Server', 'args': { 'share_name': 'transfer_files', 'zip_file_path': _create_dummy_zip_file() }} }, @@ -99,7 +102,7 @@ def _create_dummy_zip_file(): 'requires_username': True, 'requires_password': True, 'args': {'network_name': 'office'}, - 'required_service':{'type': 'openvpn', 'decoy': 'DMZ Server', 'args': { + 'required_service': {'type': 'openvpn', 'decoy': 'DMZ Server', 'args': { 'cert_country': 'US', 'cert_state': 'CA', 'cert_city': 'San Francisco', @@ -111,19 +114,19 @@ def _create_dummy_zip_file(): 'requires_username': True, 'requires_password': True, 'args': {}, - 'required_service':{'type': 'rdp', 'decoy': 'Windows Server', 'args': {}} + 'required_service': {'type': 'rdp', 'decoy': 'Windows Server', 'args': {}} }, 'ssh': { 'requires_username': True, 'requires_password': True, 'args': {}, - 'required_service':{'type': 'ssh', 'decoy': 'DMZ Server', 'args': {}} + 'required_service': {'type': 'ssh', 'decoy': 'DMZ Server', 'args': {}} }, 'ssh_privatekey': { 'requires_username': True, 'requires_password': False, - 'args': {'deploy_for': 'last_login', 'installation_type': 'alias'}, - 'required_service':{'type': 'ssh', 'decoy': 'DMZ Server', 'args': {}} + 'args': {'deploy_for': 'root', 'installation_type': 'alias'}, + 'required_service': {'type': 'ssh', 'decoy': 'DMZ Server', 'args': {}} }, } @@ -143,12 +146,24 @@ def get_args(): type=str, help='The file path to the SSL certificate of the ' 'MazeRunner management server') - parser.add_argument('deployment_group_name', type=str, help='The name of the deployment group to be created') - parser.add_argument('username', type=str, help='The username that will be shared among the breadcrumbs') - parser.add_argument('os', type=str, choices=['Windows', 'Linux'], - help='The OS for which we want to get a deployment script pack') - parser.add_argument('-p', '--passwords', required=False, type=str, help='The file path to a list of passwords') - parser.add_argument('-f', '--format', required=False, type=str, choices=['ZIP', 'EXE', 'MSI'], default='ZIP', + parser.add_argument('deployment_group_name', + type=str, + help='The name of the deployment group to be created') + parser.add_argument('username', + type=str, + help='The username that will be shared among the breadcrumbs') + parser.add_argument('breadcrumb_type', type=str, choices=BREADCRUMB_DATA.keys()) + parser.add_argument('-p', + '--passwords', + required=False, + type=str, + help='The file path to a list of passwords') + parser.add_argument('-f', + '--format', + required=False, + type=str, + choices=['ZIP', 'EXE', 'MSI'], + default='ZIP', help='The file format for the deployment script pack') return parser.parse_args() @@ -162,7 +177,7 @@ def _get_password(passwords_file): data = f.read() passwords = [p.strip() for p in data.split('\n') if p.strip()] else: - passwords = ['password', '12345678', 'xyzvbnm,', 'qwertyui'] # default common passwords + passwords = COMMON_SAMPLE_PASSWORD return random.choice(passwords) @@ -216,7 +231,8 @@ def create_service_if_needed(client, service_data): args['name'] = service_name service = client.services.create(**args) - if 'zip_file_path' in service_data['args'] and os.path.exists(service_data['args']['zip_file_path']): + if 'zip_file_path' in service_data['args'] and \ + os.path.exists(service_data['args']['zip_file_path']): os.remove(service_data['args']['zip_file_path']) decoy = create_decoy_if_needed(client, service_data['decoy']) @@ -272,23 +288,22 @@ def main(): """ args = get_args() - client = mazerunner.connect(args.ip_address, args.api_key, args.api_secret, args.certificate, False) + client = mazerunner.connect(args.ip_address, args.api_key, args.api_secret, args.certificate) password = _get_password(args.passwords) deployment_group = client.deployment_groups.create(name=args.deployment_group_name) - for breadcrumb_type in BREADCRUMB_DATA: - breadcrumb = create_breadcrumb( - client, - breadcrumb_type, - BREADCRUMB_DATA[breadcrumb_type], - args.username, - password, - args.deployment_group_name) - breadcrumb.add_to_group(deployment_group.id) + breadcrumb = create_breadcrumb( + client, + args.breadcrumb_type, + BREADCRUMB_DATA[args.breadcrumb_type], + args.username, + password, + args.deployment_group_name) + breadcrumb.add_to_group(deployment_group.id) - print "Waiting for deployment group to be available - this takes a few minutes" + print "Waiting for deployment group to become available - this may take a few minutes" deployment_group.load() while not deployment_group.is_active: @@ -296,13 +311,20 @@ def main(): deployment_group.load() save_to = '%s' % args.deployment_group_name - deployment_group.deploy( - location_with_name=save_to, - os=args.os, - download_type="install", - download_format=args.format) - print "Deployment package saved to %s.%s" % (save_to, args.format.lower()) + for operating_system in ['Windows', 'Linux']: + try: + deployment_group.deploy( + location_with_name='%s_%s' % (save_to, operating_system), + os=operating_system, + download_type="install", + download_format=args.format) + except ValidationError: + print 'This breadcrumb is not supported on %s, skipping' % operating_system + + print "%s deployment package saved to %s.%s" % (operating_system, + save_to, + args.format.lower()) if __name__ == '__main__': main() diff --git a/mazerunner/samples/create_smb_breadcrumbs.py b/mazerunner/samples/create_smb_breadcrumbs.py index 833e069..51ac240 100644 --- a/mazerunner/samples/create_smb_breadcrumbs.py +++ b/mazerunner/samples/create_smb_breadcrumbs.py @@ -11,6 +11,7 @@ import mazerunner + def _create_temp_file(): """ this function creates an empty temporary file and returns a path to the file @@ -44,9 +45,12 @@ def get_args(): parser.add_argument('api_key', type=str, help="The API key") parser.add_argument('api_secret', type=str, help="The API secret") parser.add_argument('certificate', - type=str, help="The file path to the SSL certificate of the MazeRunner management server") + type=str, + help="The file path to the SSL certificate " + "of the MazeRunner management server") parser.add_argument('usernames_file', type=str, help="The file path to a list of usernames") - parser.add_argument('-p', '--passwords', required=False, type=str, help="The file path to a list of passwords") + parser.add_argument('-p', '--passwords', required=False, type=str, + help="The file path to a list of passwords") return parser.parse_args() @@ -70,14 +74,15 @@ def main(): # Connect to the MazeRunner API # The client object holds your session data and allows you to access the MazeRunner resources - client = mazerunner.connect(args.ip_address, args.api_key, args.api_secret, args.certificate, False) - + client = mazerunner.connect(args.ip_address, args.api_key, args.api_secret, args.certificate) + # Create a decoy # This will create a decoy virtual machine inside MazeRunner. # The decoy acts as a trap, # Any action done on the decoy is monitored and will generate an alert on MazeRunner print "Creating Decoy" - decoy = client.decoys.create(name="Backup Server Decoy", hostname="nas-backup-02", os="Ubuntu_1404", vm_type="KVM") + decoy = client.decoys.create(name="Backup Server Decoy", hostname="nas-backup-02", + os="Ubuntu_1404", vm_type="KVM") while decoy.machine_status != "not_seen": # make sure the decoy was created time.sleep(5) decoy.load() @@ -110,14 +115,17 @@ def main(): print "Creating Breadcrumbs" for idx, username in enumerate(usernames): # Create breadcrumb - # Breadcrumbs can be deployed on endpoints to trick an attacker to interact with a decoy and generate an alert + # Breadcrumbs can be deployed on endpoints to trick + # an attacker to interact with a decoy and generate an alert breadcrumb = client.breadcrumbs.create( name="SMB Breadcrumb %d" % idx, breadcrumb_type="netshare", username=username, - password=random.choice(passwords)) + password=random.choice(passwords), + persistence='persistent', + registry_entry_name='test registry entry') - # Connect the breadrumb to the service + # Connect the breadcrumb to the service # When a breadcrumb is connected to a service, the credential and other information # found in the breadcrumb will be usable with the service breadcrumb.connect_to_service(service.id) diff --git a/mazerunner/samples/delete_everything.py b/mazerunner/samples/delete_everything.py index 56397aa..4b614e1 100644 --- a/mazerunner/samples/delete_everything.py +++ b/mazerunner/samples/delete_everything.py @@ -21,7 +21,8 @@ def get_args(): parser.add_argument('api_key', type=str, help="The API key") parser.add_argument('api_secret', type=str, help="The API secret") parser.add_argument('certificate', - type=str, help="The file path to the SSL certificate of the MazeRunner management server") + type=str, help="The file path to the SSL certificate of the " + "MazeRunner management server") return parser.parse_args() @@ -33,33 +34,47 @@ def main(): * Create MazeRunner connection. * Get a collection of all breadcrumbs. * Delete all elements in the collection. - * Same for services. - * Same for decoys. + * Same for deployment groups, decoys, services, endpoints, cidr mappings, background tasks """ args = get_args() - client = mazerunner.connect(args.ip_address, args.api_key, args.api_secret, args.certificate, False) + client = mazerunner.connect(args.ip_address, args.api_key, args.api_secret, args.certificate) - # delete breadcrumbs: + # Delete breadcrumbs: breadcrumbs = client.breadcrumbs - print 'deleting %d breadcrumbs' % len(breadcrumbs) + print 'Deleting %d breadcrumbs' % len(breadcrumbs) _delete_items_in_collection(breadcrumbs) - # delete deployment groups: + # Delete deployment groups: deployment_groups = client.deployment_groups - print 'deleting %d deployment groups' % len(deployment_groups) + print 'Deleting %d deployment groups' % len(deployment_groups) _delete_items_in_collection(deployment_groups, exclude_persist=True) - # delete services: + # Delete services: services = client.services - print 'deleting %d services' % len(services) + print 'Deleting %d services' % len(services) _delete_items_in_collection(services) - # delete decoys: + # Delete decoys: decoys = client.decoys - print 'deleting %d decoys' % len(decoys) + print 'Deleting %d decoys' % len(decoys) _delete_items_in_collection(decoys) + # Delete CIDR mappings: + cidr_mappings = client.cidr_mappings + print 'Deleting %d cidr mappings' % len(cidr_mappings) + _delete_items_in_collection(cidr_mappings) + + # Delete endpoints: + endpoints = client.endpoints + print 'Deleting %d endpoints' % len(endpoints) + _delete_items_in_collection(endpoints) + + # Acknowledge all complete background tasks: + print 'Acknowledging all complete background tasks' + client.background_tasks.acknowledge_all_complete() + + if __name__ == '__main__': main() diff --git a/mazerunner/samples/deploy_to_linux.py b/mazerunner/samples/deploy_to_linux.py index baaf8be..0f6e397 100644 --- a/mazerunner/samples/deploy_to_linux.py +++ b/mazerunner/samples/deploy_to_linux.py @@ -1,6 +1,7 @@ """ -This sample script deploys (install/uninstall) a specific Deployment Group on linux endpoint[s] supplied by the user. -A unique endpoint can be provided from the command line, or use a csv file to deploy on multiple endpoints. +This sample script deploys (install/uninstall) a specific Deployment Group on linux endpoint[s] +supplied by the user. A unique endpoint can be provided from the command line, or use a csv file to +deploy on multiple endpoints. """ import tempfile import mazerunner @@ -26,13 +27,18 @@ def get_args(): parser.add_argument('ip_address', type=str, help="IP address of MazeRunner management server.") parser.add_argument('api_key', type=str, help="The API key.") parser.add_argument('api_secret', type=str, help="The API secret.") - parser.add_argument('deployment_type', type=str, help='Specify which deployment we want to do - install/uninstall.', + parser.add_argument('deployment_type', type=str, + help='Specify which deployment we want to do - install/uninstall.', choices=['install', 'uninstall']) - parser.add_argument('deployment_group', type=str, help='The name of the Deployment Group in MazeRunner.') - parser.add_argument('--certificate', - type=str, help="The file path to the SSL certificate of the MazeRunner management server.") + parser.add_argument('deployment_group', type=str, + help='The name of the Deployment Group in MazeRunner.') + parser.add_argument( + '--certificate', + type=str, + help="The file path to the SSL certificate of the MazeRunner management server.") parser.add_argument('--ip', type=str, help='ip of a linux endpoint to deploy.') - parser.add_argument('--port', type=int, help='ssh port of a linux endpoint - default to 22.', default='22') + parser.add_argument('--port', type=int, help='ssh port of a linux endpoint - default to 22.', + default='22') parser.add_argument('--user', type=str, help='username of the linux endpoint to deploy.') parser.add_argument('--passwd', type=str, help='password of the linux endpoint to deploy.') parser.add_argument('--csv', type=str, help='Name of a CSV file, each line of the CSV should ' @@ -58,6 +64,7 @@ def init_ssh_client(host, port, user, passwd): sftpclient = paramiko.SFTPClient.from_transport(transport) return sshclient, sftpclient + def run_cmd(ssh, cmd): """ Tun a command on an existing ssh connection. @@ -116,6 +123,10 @@ def deploy_zip_on_endpoints(zipfile, endpoints, deploy_type, deployment_group): ) run_cmd(ssh, run_setup_cmd) + # Delete the original zip file + rm_file_cmd = 'sudo rm -rf {} {}'.format(zipfile, zipfile[:-4]) + run_cmd(ssh, rm_file_cmd) + print("MazeRunner deployment group '{}' {}ed successfully on '{}'.".format( deployment_group, deploy_type, @@ -178,7 +189,7 @@ def main(): raise Exception("No endpoints found to work on.") # Init the MazeRunner client - client = mazerunner.connect(args.ip_address, args.api_key, args.api_secret, args.certificate, False) + client = mazerunner.connect(args.ip_address, args.api_key, args.api_secret, args.certificate) # Get a new tempfile name dep_file = tempfile.mkdtemp() @@ -195,7 +206,9 @@ def main(): deployment_group.deploy(dep_file, 'Linux', args.deployment_type, 'ZIP') # Deploy the file on all endpoints we have - deploy_zip_on_endpoints(dep_file_full, linux_endpoints_to_deploy, args.deployment_type, args.deployment_group) + deploy_zip_on_endpoints(dep_file_full, linux_endpoints_to_deploy, args.deployment_type, + args.deployment_group) + if __name__ == "__main__": main() diff --git a/mazerunner/samples/elasticsearch_responder_monitor.py b/mazerunner/samples/elasticsearch_responder_monitor.py index 3e62b66..3e7145a 100644 --- a/mazerunner/samples/elasticsearch_responder_monitor.py +++ b/mazerunner/samples/elasticsearch_responder_monitor.py @@ -1,38 +1,6 @@ """ -This script should be used together with MazeRunner's Responder feature. -The Responder feature generates "net use" requests in the network, which will -function as the breadcrumbs. -When an attacker runs the Responder.py attack tool, they will -"catch" that username, and when they make a failed attempt to authenticate using that username, -an event will be logged by the hacked endpoint. -From there, the logged event will be sent to Elasticsearch. - -This script will find the event in Elasticsearch and send it to MazeRunner. For other products, -like Splunk, MazeRunner already has a built-in integration. - -In order to use this script, please configure the following: - -1. Open the configuration screen in MazeRunner and go to the SOC tab. -Under "SOC Interfaces", click "Add", choose "SOC via MazeRunner API", and -give it a name (we named it "elasticsearch_responder" in the example below). - -2. Still on the SOC tab, turn on the Responder, and provide credentials for the Responder to -access the endpoints and run the "net use" commands. - -3. At the bottom of the SOC tab, in the 'Map Responder fields' section, click 'Add' and create a -mapping of the required fields to their names in your Elasticsearch system. - -4. On the Manage API keys screen, generate an API key-secret pair, and download -the certificate to your computer. - -5. On the Campaign screen, create a nested decoy, an SMB service, -and a "Responder - Pass the Hash" breadcrumb, and connect them all together. - -6. Create a deployment group and connect the Responder breadcrumb and several endpoints to that -deployment group. - -7. Create a configuration file and pass its name as the first parameter for this -script. Use the -h option for displaying a sample configuration JSON file. +This script will allow you to integrate the Responder feature in MazeRunner with ElasticSearch. +For usage information, run the script with no params. """ @@ -279,30 +247,30 @@ def get_example(): if __name__ == '__main__': parser = argparse.ArgumentParser( - usage='This script should be used together with the Responder feature of MazeRunner.\n' - 'The Responder generates "net use" requests in the network, which will\n' - 'function as the breadcrumbs, using which we want the attacker to try to\n' - 'authenticate. As soon as the attacker tries to login using this user,\n' - 'the event will be logged and sent to elastic search. This script will find\n' + usage='\n' + 'This script should be used together with the Responder feature of MazeRunner.\n' + 'The Responder breadcrumb and service detect Responder.py tool usage in your ' + 'network.\n' + 'When MazeRunner detects such tool it feeds it with the credentials you configured\n' + 'in the breadcrumb.\n' + 'As soon as the attacker tries to use these stolen credentials and log in\n' + 'to one of the monitored assets in your network,\n' + 'the event will be logged and sent to ElasticSearch. This script will find\n' 'the event in ElasticSearch and send it to MazeRunner. For other products,\n' 'like Splunk, MazeRunner already has a built-in integration.\n\n' 'In order to use this script, please configure the following:\n\n' - '1. Open the configuration screen in MazeRunner and go to the SOC tab.\n' - 'Under "SOC Interfaces" click "add", choose "SOC via MazeRunner API" and\n\n' - 'give it a name (we named it "elasticsearch_responder" in the example below).\n' - '2. Still on the SOC tab, turn on the Responder, and provide it credentials using ' - 'which it can access the endpoints and run the "net use" commands.\n\n' - '3. At the bottom of the SOC tab, in the "Map Responder fields" section, click ' - '"Add" and create a mapping of the required fields to their names in your ' - 'Elasticsearch system.\n\n' - '4. In the Manage API Keys screen, generate an api key-secret pair, and download \n' - 'the certificate to your computer.\n\n' - '5. In the campaign screen, create a nested decoy, an SMB service,\n' - 'and a "Responder - Pass the Hash" breadcrumb, and connect them all together.\n\n' - '6. Create a deployment group and connect to it the responder breadcrumb and ' - 'several endpoints.\n\n' - '7. Create a configuration file and pass its name as the first parameter for this\n' - 'script. Here is an example for the file:\n\n %s' % Config.get_example()) + '1. Open the ActiveSOC tab in MazeRunner.\n' + 'Under "SOC Interfaces" click "add", choose "SOC via MazeRunner API" and\n' + 'give it a name (we named it "elasticsearch_responder" in the example below).\n\n' + '2. In the Settings tab, go to the "API Keys" sub-tab. There, generate an api \n' + 'key-secret pair, and download the certificate to your computer.\n\n' + '3. Create a responder campaign. For more information about this step,\n' + 'see the "ActiveSoc and responder monitor activity feed" section in the user ' + 'manual.\n\n' + '4. Under the ActiveSOC tab, go to "Responder Monitor", and make sure ' + 'that "Responder monitor SOC integration" is activated.\n\n' + '5. Create a configuration file and pass the file name as the first parameter for \n' + 'this script. Here is an example for the file:\n\n %s' % Config.get_example()) parser.add_argument('config_file') args = parser.parse_args() diff --git a/mazerunner/samples/send_bin_files_to_cuckoo.py b/mazerunner/samples/send_bin_files_to_cuckoo.py new file mode 100644 index 0000000..0d6ed9f --- /dev/null +++ b/mazerunner/samples/send_bin_files_to_cuckoo.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python2 + +import argparse +import requests +import os +import tempfile +import shutil + +import mazerunner + + +def retrieve_files_from_mazerunner(client, dirpath): + + const_bin = ".bin" + + # Create a list to store our files in + alert_files = [] + + # Grab Code Execution Alerts + filter_params = dict( + filter_enabled = True, + only_alerts = False, + alert_types = ["code","unsigned_code"] + ) + alerts = client.alerts.filter(**filter_params) + + # Iterate through our alerts and download the bin files if one exists + + for alert in alerts: + + # Remove the extension as its added when we download the file from MazeRunner + dest_file_name = alert.image_file_name + + # Local filepath where we will store our executable + dest_file_path = os.path.join(dirpath, "%s_%s" % (alert.decoy["hostname"], dest_file_name)) + + try: + alert.download_image_file(dest_file_path[:-len(const_bin)]) + + # Store path for later use + alert_files.append(dest_file_path) + + except mazerunner.exceptions.ValidationError: + # No file to download + pass + + return alert_files + + +def send_files_to_cuckoo(cuckoo_api, verify_ssl, alert_files): + + # Iterate through our file list and send them to Cuckoo + for current_file in alert_files: + fname = os.path.basename(current_file) + cuckoo_url = 'https://%s/tasks/create/file' % cuckoo_api + + try: + with open(current_file, "rb") as sample: + files = {"file": (fname, sample)} + r = requests.post(cuckoo_url, files=files, verify=verify_ssl) + + # Ensure we successfully sent our file to Cuckoo + if r.status_code != 200: + print "Error uploading file:" + fname + continue + except IOError: + print current_file + " could not be opened for reading." + pass + + +def transfer_files_to_cuckoo(cuckoo_api, verify_ssl, connection_params): + + # Create a temporary directory to store our bin files in + dirpath = tempfile.mkdtemp() + + try: + # Instantiate our MazeRunner connection + client = mazerunner.connect(**connection_params) + + # Download files from MazeRunner, Upload files to Cuckoo + alert_files = retrieve_files_from_mazerunner(client, dirpath) + send_files_to_cuckoo(cuckoo_api, verify_ssl, alert_files) + + finally: + # Clean up after our work + shutil.rmtree(dirpath) + + print "Process complete" + + +def main(): + parser = argparse.ArgumentParser("Send code execution files from MazeRunner to a Cuckoo Instance") + parser.add_argument('mazerunner', help="IP address of MazeRunner management server") + parser.add_argument('api_key', help="API key") + parser.add_argument('api_secret', help="API secert key") + parser.add_argument('certificate', help="Path to MazeRunner's SSL certificate") + parser.add_argument('cuckoo_api', help="IP or FQDN of Cuckoo (192.168.1.10:4343 or cuckoo.yourdomain.com:4343") + parser.add_argument('--skip-verification', dest='verify_ssl', action='store_false', help='Skip SSL verification') + + args = parser.parse_args() + + connection_params = dict(ip_address=args.mazerunner, + api_key=args.api_key, + api_secret=args.api_secret, + certificate=args.certificate + ) + + transfer_files_to_cuckoo(args.cuckoo_api, args.verify_ssl, connection_params) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/mazerunner/samples/syslog_server_active_soc_reporter.py b/mazerunner/samples/syslog_server_active_soc_reporter.py index 2731234..0b7898b 100644 --- a/mazerunner/samples/syslog_server_active_soc_reporter.py +++ b/mazerunner/samples/syslog_server_active_soc_reporter.py @@ -130,8 +130,7 @@ def main(): client = mazerunner.connect(args.ip_address, args.api_key, args.api_secret, - args.certificate, - False) + args.certificate) try: server = SocketServer.UDPServer((HOST, PORT), get_syslog_handler(client)) diff --git a/mazerunner/samples/track_live_alerts.py b/mazerunner/samples/track_live_alerts.py index ba2e5cf..398988a 100644 --- a/mazerunner/samples/track_live_alerts.py +++ b/mazerunner/samples/track_live_alerts.py @@ -6,7 +6,7 @@ import time import mazerunner -WAIT_TIME = 3 # Time to wait in seconds +WAIT_TIME = 3 # Time to wait in seconds DISPLAY_FORMAT = '''Got a new alert! Decoy: {decoy_name} Alert Type: {alert_type}''' @@ -21,7 +21,9 @@ def get_args(): parser.add_argument('api_key', type=str, help="The API key") parser.add_argument('api_secret', type=str, help="The API secret") parser.add_argument('certificate', - type=str, help="The file path to the SSL certificate of the MazeRunner management server") + type=str, + help="The file path to the SSL certificate of the " + "MazeRunner management server") parser.add_argument('-m', '--show-muted', action='store_true', help="Show mute-level alerts as well as alert-level alerts") return parser.parse_args() @@ -41,24 +43,28 @@ def main(): """ args = get_args() - client = mazerunner.connect(args.ip_address, args.api_key, args.api_secret, args.certificate, False) + client = mazerunner.connect(args.ip_address, args.api_key, args.api_secret, args.certificate) # Get alerts - alerts = client.alerts - alert_types = alerts.params()['alert_type'] - filtered_alerts = alerts.filter(filter_enabled=True, only_alerts=not args.show_muted, alert_types=alert_types) - print "Showing all alerts live. Press Ctrl+C to exit." - last_length = len(list(filtered_alerts)) + alert_types = client.alerts.params()['alert_type'] + last_seen_id = 0 + + print("Showing all alerts live. Press Ctrl+C to exit.") + try: while True: time.sleep(WAIT_TIME) - current_length = len(list(filtered_alerts)) - if current_length > last_length: - alert_list = list(filtered_alerts) - for i in range(last_length, current_length): - print DISPLAY_FORMAT.format( - decoy_name=alert_list[i].decoy['name'], alert_type=alert_list[i].alert_type) - last_length = current_length + alert_filter = client.alerts.filter(filter_enabled=True, + only_alerts=not args.show_muted, + alert_types=alert_types, + id_greater_than=last_seen_id) + + for alert in alert_filter: + print(DISPLAY_FORMAT.format(decoy_name=alert.decoy['name'], + alert_type=alert.alert_type)) + + if alert.id > last_seen_id: + last_seen_id = alert.id except KeyboardInterrupt: sys.exit() diff --git a/setup.py b/setup.py index 168161d..29466d6 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,29 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + from setuptools import setup, find_packages +long_description = """ +This library implements a convenient client for MazeRunner™ API for Python. +Using this library, you will be able to easily configure and manipulate the key features +of MazeRunner, such as the creation of a deception campaign, turning decoys on or off, deployment on +remote endpoints, and inspecting alerts with their attached evidence. + +See the documentation at https://community.cymmetria.com/api + +Fork us at https://github.com/Cymmetria/mazerunner_sdk_python +""" + setup( name='mazerunner_sdk', packages=find_packages(), - version='1.1.3', + version='1.2.3', description='MazeRunner SDK', + long_description=long_description, author='Cymmetria', author_email='publicapi@cymmetria.com', url='https://github.com/Cymmetria/mazerunner_sdk_python', - download_url='https://github.com/Cymmetria/mazerunner_sdk_python/tarball/1.1.3', + download_url='https://github.com/Cymmetria/mazerunner_sdk_python/tarball/1.2.3', license='BSD 3-Clause', keywords=['cymmetria', 'mazerunner', 'sdk', 'api'], install_requires=["argparse==1.2.1", diff --git a/source/api_client.rst b/source/api_client.rst index d7b5670..35c5442 100644 --- a/source/api_client.rst +++ b/source/api_client.rst @@ -71,6 +71,24 @@ Alerts :inherited-members: :exclude-members: MODEL_CLASS +.. autoclass:: api_client.AlertProcess + :members: + :inherited-members: + +.. autoclass:: api_client.AlertProcessCollection + :members: + :inherited-members: + :exclude-members: MODEL_CLASS + +.. autoclass:: api_client.AlertProcessDLL + :members: + :inherited-members: + +.. autoclass:: api_client.AlertProcessDLLCollection + :members: + :inherited-members: + :exclude-members: MODEL_CLASS + Endpoints ========= @@ -130,3 +148,27 @@ ActiveSOC events :members: :inherited-members: :exclude-members: MODEL_CLASS + + +Audit log events +================ +.. autoclass:: api_client.AuditLogLineCollection + :members: + :inherited-members: + :exclude-members: MODEL_CLASS + + +Storage usage data +================== +.. autoclass:: api_client.StorageUsageData + :members: + :inherited-members: + :exclude-members: MODEL_CLASS + + +Forensic puller on demand +========================= +.. autoclass:: api_client.ForensicPullerOnDemand + :members: + :inherited-members: + :exclude-members: MODEL_CLASS diff --git a/source/conf.py b/source/conf.py index 7ad13e5..be91abe 100644 --- a/source/conf.py +++ b/source/conf.py @@ -60,7 +60,7 @@ # General information about the project. project = u'MazeRunner™ SDK' -copyright = u'2015-2017, Cymmetria' +copyright = u'2015-2018, Cymmetria' author = u'Cymmetria' # The version info for the project you're documenting, acts as replacement for @@ -68,9 +68,9 @@ # built documents. # # The short X.Y version. -version = u'1.1.3' +version = u'1.2.3' # The full version, including alpha/beta/rc tags. -release = u'1.1.3' +release = u'1.2.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -145,7 +145,7 @@ # The name for this set of Sphinx documents. # " v documentation" by default. # -# html_title = u'MazeRunner SDK v1.1.3' +# html_title = u'MazeRunner SDK v1.2.3' # A shorter title for the navigation bar. Default is the same as html_title. # diff --git a/source/index.rst b/source/index.rst index 396d51d..79ae2fd 100644 --- a/source/index.rst +++ b/source/index.rst @@ -17,4 +17,4 @@ Fork us at https://github.com/Cymmetria/mazerunner_sdk_python installation api_client samples - Download as PDF + Download as PDF diff --git a/Makefile b/sphinx_makefile similarity index 100% rename from Makefile rename to sphinx_makefile diff --git a/test/sample.docx b/test/sample.docx new file mode 100644 index 0000000..1352bc7 Binary files /dev/null and b/test/sample.docx differ diff --git a/test/test_sdk.py b/test/test_sdk.py index 9972a79..82a2e39 100644 --- a/test/test_sdk.py +++ b/test/test_sdk.py @@ -1,37 +1,48 @@ import StringIO import csv +import datetime import json import logging import shutil from stat import S_IRUSR import pytest -from contextlib2 import suppress from retrying import retry from subprocess import Popen import mazerunner import os -from mazerunner.api_client import DeploymentGroupCollection, \ - BreadcrumbCollection, ServiceCollection, DecoyCollection, Service, \ - AlertPolicy, CIDRMappingCollection, BackgroundTaskCollection, \ - EndpointCollection +from mazerunner.api_client import Service, AlertPolicy, Decoy, Breadcrumb, \ + DeploymentGroup, Endpoint, CIDRMapping, BackgroundTask, AuditLogLine, ISO_TIME_FORMAT from mazerunner.exceptions import ValidationError, ServerError, BadParamError, \ InvalidInstallMethodError from utils import TimeoutException, wait_until +CLEAR_SYSTEM_ERROR_MESSAGE = 'System must be clean before running this test. Use the '\ + '--initial_clean flag to do this automatically' ENDPOINT_IP_PARAM = 'endpoint_ip' ENDPOINT_USERNAME_PARAM = 'endpoint_username' ENDPOINT_PASSWORD_PARAM = 'endpoint_password' CODE_EXECUTION_ALERT_TYPE = 'code' +FORENSIC_DATA_ALERT_TYPE = 'forensic_puller' MAZERUNNER_IP_ADDRESS_PARAM = 'ip_address' API_ID_PARAM = 'id' API_SECRET_PARAM = 'secret' MAZERUNNER_CERTIFICATE_PATH_PARAM = 'mazerunner_certificate_path' -INITIAL_DEPLOYMENT_GROUPS = 1 + +ENTITIES_CONFIGURATION = { + Decoy: [], + Service: [], + Breadcrumb: [], + DeploymentGroup: [1], + Endpoint: [], + CIDRMapping: [], + BackgroundTask: [] +} + TEST_DEPLOYMENTS_FILE_PATH = os.path.join(os.path.dirname(__file__), 'test_deployments/dep.zip') TEST_DEPLOYMENTS_FOLDER_PATH = os.path.dirname(TEST_DEPLOYMENTS_FILE_PATH) @@ -60,10 +71,6 @@ class MachineStatus(object): # noinspection PyMethodMayBeStatic,PyAttributeOutsideInit class APITest(object): - DISPOSABLE_TYPES = [ - DecoyCollection, BreadcrumbCollection, DeploymentGroupCollection, - CIDRMappingCollection, BackgroundTaskCollection, EndpointCollection, ServiceCollection - ] runslow = pytest.mark.skipif(not pytest.config.getoption('--runslow'), reason='--runslow not activated') @@ -71,13 +78,13 @@ class APITest(object): reason='--lab_dependent not activated') def _assert_clean_system(self): - assert len(self.decoys) == 0 - assert len(self.services) == 0 - assert len(self.breadcrumbs) == 0 - assert len(self.deployment_groups) == INITIAL_DEPLOYMENT_GROUPS - assert len(self.cidr_mappings) == 0 - assert len(self.endpoints) == 0 - assert len(self.background_tasks) == 0 + for entity_collection in self.disposable_entities: + existing_ids = {entity.id for entity in entity_collection} + expected_ids = set(ENTITIES_CONFIGURATION[entity_collection.MODEL_CLASS]) + + assert existing_ids == expected_ids, CLEAR_SYSTEM_ERROR_MESSAGE + + assert len(self.background_tasks) == 0, CLEAR_SYSTEM_ERROR_MESSAGE def _configure_entities_groups(self): self.decoys = self.client.decoys @@ -89,6 +96,16 @@ def _configure_entities_groups(self): self.cidr_mappings = self.client.cidr_mappings self.endpoints = self.client.endpoints self.background_tasks = self.client.background_tasks + self.audit_log = self.client.audit_log + + self.disposable_entities = [ + self.decoys, + self.services, + self.breadcrumbs, + self.deployment_groups, + self.endpoints, + self.cidr_mappings + ] def setup_method(self, method): logger.debug("setup_method called") @@ -112,37 +129,27 @@ def setup_method(self, method): certificate=self.mazerunner_certificate_path) self._configure_entities_groups() - self._assert_clean_system() - self._register_existing_elements() + if pytest.config.option.initial_clean: + self._destroy_new_entities() + + self._assert_clean_system() self.file_paths_for_cleanup = [] _clear_deployment_path() - def _register_existing_elements(self): - - def _get_group_elements(group): - return [element.id - for element - in group(self.client)] - - self._existing_elements_by_type = { - group: _get_group_elements(group) - for group - in self.DISPOSABLE_TYPES - } - def _destroy_new_entities(self): + for entity_collection in self.disposable_entities: + for entity in list(entity_collection): + initial_ids = ENTITIES_CONFIGURATION[entity_collection.MODEL_CLASS] + if entity.id not in initial_ids: + wait_until(entity.delete, exc_list=[ServerError, ValidationError], + check_return_value=False) - def _get_items(group): - return list(group(self.client)) + self.background_tasks.acknowledge_all_complete() - for entity_group in self.DISPOSABLE_TYPES: - for entity in _get_items(entity_group): - if entity.id not in self._existing_elements_by_type[entity_group]: - with suppress(ServerError): - entity.delete() + wait_until(self._assert_clean_system, exc_list=[AssertionError], check_return_value=False) def teardown_method(self, method): logger.debug("teardown_method called") @@ -210,11 +217,15 @@ def assert_entity_name_not_in_collection(self, entity_name, collection): SSH_BREADCRUMB_NAME = "ssh_breadcrumb" SSH_SERVICE_NAME = "ssh_service" SSH_DECOY_NAME = "ssh_decoy" - SSH_GROUP_NAME_UPDATE = "ssh_deployment_group_update" SSH_BREADCRUMB_NAME_UPDATE = "ssh_breadcrumb_update" SSH_SERVICE_NAME_UPDATE = "ssh_service_update" SSH_DECOY_NAME_UPDATE = "ssh_decoy_update" +HONEYDOC_GROUP_NAME = "honeydoc_deployment_group" +HONEYDOC_BREADCRUMB_NAME = "honeydoc_breadcrumb" +HONEYDOC_SERVICE_NAME = "honeydoc_service" +HONEYDOC_SERVICE_SERVER_SUFFIX = "server_suffix" +HONEYDOC_DECOY_NAME = "honeydoc_decoy" OVA_DECOY = "ova_decoy" @@ -224,11 +235,13 @@ def test_api_setup_campaign(self): logger.debug("test_api_setup_campaign called") # Create deployment group: - assert len(self.deployment_groups) == INITIAL_DEPLOYMENT_GROUPS + assert {dg.id for dg in self.deployment_groups} == \ + set(ENTITIES_CONFIGURATION[DeploymentGroup]) deployment_group = self.deployment_groups.create(name=SSH_GROUP_NAME, description="test deployment group") self.assert_entity_name_in_collection(SSH_GROUP_NAME, self.deployment_groups) - assert len(self.deployment_groups) == INITIAL_DEPLOYMENT_GROUPS + 1 + assert {dg.id for dg in self.deployment_groups} == \ + set(ENTITIES_CONFIGURATION[DeploymentGroup] + [deployment_group.id]) # Create breadcrumb: assert len(self.breadcrumbs) == 0 @@ -242,7 +255,7 @@ def test_api_setup_campaign(self): # Create service: assert len(self.services) == 0 - service_ssh = self.services.create(name=SSH_SERVICE_NAME, service_type="ssh") + service_ssh = self.services.create(name=SSH_SERVICE_NAME, service_type="ssh", any_user="false") self.assert_entity_name_in_collection(SSH_SERVICE_NAME, self.services) assert len(self.services) == 1 @@ -308,7 +321,7 @@ def test_api_setup_campaign(self): self.assert_entity_name_in_collection(SSH_GROUP_NAME, self.deployment_groups) self.assert_entity_name_not_in_collection(SSH_GROUP_NAME_UPDATE, self.deployment_groups) - service_ssh.update(name=SSH_SERVICE_NAME_UPDATE) + service_ssh.update(name=SSH_SERVICE_NAME_UPDATE, any_user="false") breadcrumb_ssh.detach_from_service(service_ssh.id) self.assert_entity_name_not_in_collection(SSH_SERVICE_NAME, @@ -321,6 +334,43 @@ def test_api_setup_campaign(self): decoy_ssh.load() assert decoy_ssh.machine_status == MachineStatus.INACTIVE + invalid_service = "invalid_service" + with pytest.raises(ValidationError): + self.services.create(name=invalid_service, service_type=invalid_service) + self.assert_entity_name_not_in_collection(invalid_service, self.services) + + def test_honeydoc_breadcrumb(self): + logger.debug("test_honeydoc_breadcrumb called") + downloaded_docx_file_path = "test/downloaded.docx" + self.file_paths_for_cleanup.append(downloaded_docx_file_path) + deployment_group = self.deployment_groups.create(name=HONEYDOC_GROUP_NAME, + description="test deployment group") + breadcrumb_honeydoc = self.breadcrumbs.create(name=HONEYDOC_BREADCRUMB_NAME, + breadcrumb_type="honey_doc", + deployment_groups=[deployment_group.id], + monitor_from_external_host=False, + file_field_name="docx_file_content", + file_path="test/sample.docx") + service_honeydoc = self.services.create(name=HONEYDOC_SERVICE_NAME, + service_type="honey_doc", + server_suffix=HONEYDOC_SERVICE_SERVER_SUFFIX) + decoy_honeydoc = self.create_decoy(dict(name=HONEYDOC_DECOY_NAME, + hostname="decoyhoneydoc", + os="Ubuntu_1404", + vm_type="KVM")) + service_honeydoc.load() + breadcrumb_honeydoc.load() + self.assert_entity_name_in_collection(HONEYDOC_GROUP_NAME, breadcrumb_honeydoc.deployment_groups) + breadcrumb_honeydoc.connect_to_service(service_honeydoc.id) + service_honeydoc.connect_to_decoy(decoy_honeydoc.id) + service_honeydoc.load() + breadcrumb_honeydoc.load() + self.power_on_decoy(decoy_honeydoc) + decoy_honeydoc.load() + breadcrumb_honeydoc.download_breadcrumb_honeydoc(downloaded_docx_file_path) + assert os.path.exists(downloaded_docx_file_path) + assert os.path.getsize(downloaded_docx_file_path) > 0 + class TestDecoy(APITest): DECOY_STATUS_ACTIVE = 'active' @@ -341,7 +391,11 @@ def test_ova(self): # Download decoy: download_file_path = "mazerunner/ova_image" - ova_decoy.download(location_with_name=download_file_path) + + # Wait until the decoy becomes available and download the file + wait_until(ova_decoy.download, location_with_name=download_file_path, + check_return_value=False, exc_list=[ValidationError], total_timeout=60*10) + self.file_paths_for_cleanup.append("{}.ova".format(download_file_path)) def test_decoy_update(self): @@ -423,7 +477,7 @@ def test_check_conflicts(self): hostname="decoyssh", os="Ubuntu_1404", vm_type="KVM")) - service_ssh = self.services.create(name=SSH_SERVICE_NAME, service_type="ssh") + service_ssh = self.services.create(name=SSH_SERVICE_NAME, service_type="ssh", any_user="false") service_ssh.connect_to_decoy(decoy_ssh.id) dep_group = self.deployment_groups.create(name='test_check_conflicts') @@ -457,7 +511,7 @@ def test_check_conflicts(self): assert dep_group.check_conflicts('Windows') == [ { u'error': u"Conflict between breadcrumbs ssh1 and ssh2: " - u"Two SSH breadcrumb can't point to the same " + u"Two SSH breadcrumbs can't point to the same " u"user/decoy combination on the same endpoint" } ] @@ -468,7 +522,7 @@ def test_deployment(self): os="Ubuntu_1404", vm_type="KVM")) - service_ssh = self.services.create(name=SSH_SERVICE_NAME, service_type="ssh") + service_ssh = self.services.create(name=SSH_SERVICE_NAME, service_type="ssh", any_user="false") bc_ssh = self.breadcrumbs.create(name='ssh1', breadcrumb_type="ssh", @@ -483,6 +537,13 @@ def test_deployment(self): self.power_on_decoy(decoy_ssh) + def _has_complete_bg_tasks(): + return len([bg_task for bg_task in self.background_tasks.filter(running=False)]) > 0 + + def _wait_and_destroy_background_task(): + wait_until(_has_complete_bg_tasks, check_return_value=True) + self.background_tasks.acknowledge_all_complete() + def _test_manual_deployment(): dep_group.deploy(location_with_name=TEST_DEPLOYMENTS_FILE_PATH.replace('.zip', ''), os='Windows', @@ -502,14 +563,6 @@ def _test_manual_deployment(): os.remove(TEST_DEPLOYMENTS_FILE_PATH) def _test_auto_deployment(): - with pytest.raises(ValidationError): - dep_group.auto_deploy(username=None, - password=None, - install_method='PS_EXEC', - run_method='EXE_DEPLOY', - domain='', - deploy_on="all") - # Since this runs asynchronously and it has nothing to deploy on, we only want to see # that the request was accepted @@ -520,6 +573,8 @@ def _test_auto_deployment(): domain='', deploy_on="all") + _wait_and_destroy_background_task() + self.deployment_groups.auto_deploy_groups( username='some-user', password='some-pass', @@ -529,9 +584,24 @@ def _test_auto_deployment(): domain='', deploy_on="all") + _wait_and_destroy_background_task() + _test_manual_deployment() _test_auto_deployment() + def forensic_puller_alert_is_shown(self): + alerts = list(self.alerts.filter(filter_enabled=True, + only_alerts=False, + alert_types=[FORENSIC_DATA_ALERT_TYPE])) + return bool(alerts) + + @pytest.mark.skip("needs auto deploy setting credentials") + @APITest.lab_dependent + def test_forensic_puller_on_demand(self): + ## TODO: Add setting global deployment credentials here. + self.client.forensic_puller_on_demand.run_on_ip_list(ip_list=[self.lab_endpoint_ip]) + wait_until(self.forensic_puller_alert_is_shown) + @APITest.lab_dependent def test_deployment_credentials(self): assert self.client.deployment_groups.test_deployment_credentials( @@ -653,7 +723,7 @@ def _create_code_exec_alert(): hostname="decoyssh", os="Ubuntu_1404", vm_type="KVM")) - service_ssh = self.services.create(name=SSH_SERVICE_NAME, service_type="ssh") + service_ssh = self.services.create(name=SSH_SERVICE_NAME, service_type="ssh", any_user="false") service_ssh.connect_to_decoy(decoy_ssh.id) bc_ssh1 = self.breadcrumbs.create(name='ssh1', @@ -731,14 +801,17 @@ def test_params(self): class TestEntity(APITest): def test_repr(self): - service = self.services.create(name=SSH_SERVICE_NAME, service_type="ssh") - assert str(service) == "".format( - serv=self.mazerunner_ip_address, - service_id=service.id) + service = self.services.create(name=SSH_SERVICE_NAME, service_type="ssh", any_user="false") + + str_service = ""\ + .format(serv=self.mazerunner_ip_address, service_id=service.id, any_user=service.any_user) + assert str(service) == str_service def test_get_attribute(self): - service = self.services.create(name=SSH_SERVICE_NAME, service_type="ssh") + service = self.services.create(name=SSH_SERVICE_NAME, service_type="ssh", any_user="false") assert service.name == SSH_SERVICE_NAME with pytest.raises(AttributeError): @@ -786,7 +859,8 @@ def test_service_with_files(self): self.services.create(name=SSH_BREADCRUMB_NAME, service_type="http", zip_file_path=site_data_file, - web_apps=['phpmyadmin']) + web_apps=['phpmyadmin'], + https_active=False) assert len(self.client.services) == 1 @@ -922,7 +996,29 @@ def _test_clean(ep): def _test_reassignment(ep): dep_group = self.deployment_groups.create(name='ep1_test', description='test') + + # Assign via collection self.endpoints.reassign_to_group(dep_group, [ep]) + assert self.endpoints.get_item(ep.id).deployment_group.id == dep_group.id + + # Clear via collection + self.endpoints.clear_deployment_group([ep]) + assert self.endpoints.get_item(ep.id).deployment_group is None + + # Assign via entity + all_breadcrumbs_deployment_group = self.deployment_groups.get_item( + self.deployment_groups.ALL_BREADCRUMBS_DEPLOYMENT_GROUP_ID) + endpoint.reassign_to_group(all_breadcrumbs_deployment_group) + assert self.endpoints.get_item(ep.id).deployment_group.id == \ + all_breadcrumbs_deployment_group.id + + # Clear via entity + ep.clear_deployment_group() + assert self.endpoints.get_item(ep.id).deployment_group is None + + # Eventually leave the endpoint with the new deployment group assigned + ep.reassign_to_group(dep_group) + assert self.endpoints.get_item(ep.id).deployment_group.id == dep_group.id def _test_delete(): self.endpoints.filter('no.such.endpoints').delete_filtered() @@ -934,7 +1030,7 @@ def _test_delete(): endpoints = list(self.endpoints.filter(self.lab_endpoint_ip)) assert len(endpoints) > 0 - self.endpoints.delete_by_endpoints_ids([endpoints[0].id]) + self.endpoints.delete_by_endpoints_ids([curr_endpoint.id for curr_endpoint in endpoints]) assert len(self.endpoints.filter(self.lab_endpoint_ip)) == 0 def _test_data(): @@ -949,7 +1045,6 @@ def _test_data(): ]) assert isinstance(self.endpoints.filter_data(), dict) - assert isinstance(self.endpoints.status_dashboard(), list) def _test_stop_import(): _destroy_elements() @@ -979,3 +1074,192 @@ def _test_stop_import(): _test_delete() _test_data() _test_stop_import() + + def test_create_endpoint(self): + for params in [ + dict(ip_address='1.1.1.1'), + dict(dns='endpoint_address.endpoint.local'), + dict(hostname='hostname'), + dict(dns='endpoint_address.endpoint.local', ip_address='1.1.1.1'), + ]: + endpoint = self.endpoints.create(**params) + assert endpoint + for key, value in params.iteritems(): + assert getattr(endpoint, key) == value + assert len(self.endpoints) == 1 + endpoint.delete() + + def test_create_endpoint_with_deployment_group(self): + ip_address = "1.1.1.1" + endpoint = self.endpoints.create(ip_address=ip_address, deployment_group_id=1) + assert endpoint.ip_address == ip_address + assert endpoint.deployment_group.name == "All Breadcrumbs" + endpoint.delete() + + def test_create_invalid_endpoint(self): + for params, expected_error_message in [ + (dict(ip_address='1.1.1.1.1'), "Enter a valid IPv4 address."), + (dict(dns='A'*256), "Maximum field length is 255 characters"), + (dict(hostname='A'*16), "Maximum field length is 15 characters"), + (dict(), "You must provide either dns, hostname, or ip address"), + ]: + try: + self.endpoints.create(**params) + raise AssertionError, "Creation of the endpoint should raise an exception" + except ValidationError as e: + error = json.loads(e.message) + if params: + for key in params: + assert key in error + assert error[key] == [expected_error_message] + else: + assert error["non_field_errors"] == [expected_error_message] + + +class TestAuditLog(APITest): + + @staticmethod + def _format_time(date_obj): + return date_obj.strftime(ISO_TIME_FORMAT) + + def _test_time_based_queries(self): + today = datetime.datetime.now() + tomorrow = today + datetime.timedelta(days=1) + a_week_ago = today + datetime.timedelta(days=-7) + two_weeks_ago = today + datetime.timedelta(days=-14) + + # check that today has data - start date + assert len(self.audit_log.filter(start_date=self._format_time(today))) != 0, \ + "No data from today according to start date" + + # and that tomorrow doesn't - start date + assert len(self.audit_log.filter(start_date=self._format_time(tomorrow))) == 0, \ + "Data from tomorrow found!" + + # check that today has data - end date + assert len(self.audit_log.filter(end_date=self._format_time(today))) != 0, \ + "No data from today according to end date" + + # and that last week doesn't - end date + assert len(self.audit_log.filter(end_date=self._format_time(a_week_ago))) == 0, \ + "Data from a week ago found, even though we deleted everything!" + + # test time range + assert len(self.audit_log.filter(start_date=self._format_time(a_week_ago), + end_date=self._format_time(today))) != 0, \ + "No logs from the past week" + + assert len(self.audit_log.filter(start_date=self._format_time(two_weeks_ago), + end_date=self._format_time(a_week_ago))) == 0, \ + "Logs found from two weeks ago." + + def _test_object_ids_queries(self, log_count): + # get obj ids from server + object_ids = [log_line._param_dict.get("object_ids") for log_line in self.audit_log] + # and extract them + object_ids = list(set([object_id[0] if object_id else None for object_id in object_ids])) + + # and make sure you have enough obj ids + assert len(object_ids) >= 2, "No more than 1 object ID in the system" + + # test that you don't get all the alerts when filtering + usable_object_id = [object_id for object_id in object_ids if object_id][0] + + assert len(self.audit_log.filter(object_ids=usable_object_id)) != log_count, \ + "Object ID filter returned the same amount of logs as the full filter" + + def _test_username_queries(self, log_count): + user_id = self.client._auth.credentials['id'] + + # make sure that if the username is right you get data + assert len(self.audit_log.filter(username=[user_id])) != 0, "No logs found for the user" + + # Note: we don't have more than one user in the tests, therefore we don't + # have a test that filters one user's info + + # check that a bad username doesnt provide any data + bad_username = user_id * 2 + assert len(self.audit_log.filter(username=[bad_username])) == 0, \ + "Logs found for the (probably) nonexistent user {}".format(bad_username) + + # test users not list ERR + with pytest.raises(BadParamError): + self.audit_log.filter(username=user_id) + + def _test_category_queries(self, log_count): + # get categories from server + categories = list(set([log_line._param_dict.get("category") for log_line in self.audit_log])) + + # and make sure you have enough + assert len(categories) >= 2, "No more than 1 category in the system" + + # make sure the param is OK + assert len(self.audit_log.filter(category=[categories[0]])) != 0, \ + "No logs for previously existing filter value" + + # test that you don't get all the alerts when filtering + assert len(self.audit_log.filter(category=[categories[0]])) != log_count, \ + "Filtered list returned the same amount of logs as the full filter" + + # test categories not list ERR + with pytest.raises(BadParamError): + self.audit_log.filter(category=categories[0]) + + def _test_event_type_queries(self, log_count): + # get event_types from server + event_types = list(set([log_line._param_dict.get("event_type_label") for log_line in self.audit_log])) + + # and make sure you have enough + assert len(event_types) >= 2, "No more than 1 event type in the system." + + # make sure the param is OK + assert len(self.audit_log.filter(event_type=[event_types[0]])) != 0, \ + "No logs for previously existing filter value" + + # test that you don't get all the alerts when filtering + assert len(self.audit_log.filter(event_type=[event_types[0]])) != log_count, \ + "Filtered list returned the same amount of logs as the full filter" + + # test event type not list ERR + with pytest.raises(BadParamError): + self.audit_log.filter(event_type=event_types[0]) + + def test_audit_log_query(self): + + # test delete (at the start for a clean log) + self.audit_log.delete() + logger.info("Audit log cleared") + + # build all sorts of logs + decoy_ssh = self.create_decoy(dict(name=SSH_DECOY_NAME, + hostname="decoyssh", + os="Ubuntu_1404", + vm_type="KVM")) + + # test query + log_count = self.audit_log + assert len(log_count) != 0, "No logs found" + assert type(list(self.audit_log)[0]) == AuditLogLine, "Invalid output" + + self._test_time_based_queries() + self._test_object_ids_queries(log_count) + self._test_username_queries(log_count) + self._test_category_queries(log_count) + self._test_event_type_queries(log_count) + + # test filter=False with params + assert len(self.audit_log.filter(event_type=["Delete"], filter_enabled=False)) == \ + len(self.audit_log.filter(event_type=["Action"], filter_enabled=False)), \ + "filter_enabled = False should make other filters redundant" + + # test delete (again to make sure the log is actually cleaned) + self.audit_log.delete() + assert len(self.audit_log) != 0, "No delete log!" + assert len(self.audit_log) == 1, "Other logs found" + + # once this issue is fixed we need to use the provided data from the decoy creation to actually filter items + @pytest.mark.xfail(reason="Item filtering is broken on server side") + def test_audit_log_item_filter(self): + # look for an item that doesnt exist + INVALID_ITEM_FILTER = "qweasdzxc" + assert len(self.audit_log.filter(item=INVALID_ITEM_FILTER)) == 0 diff --git a/testing_requirements.txt b/testing_requirements.txt index c62a848..603e7e1 100644 --- a/testing_requirements.txt +++ b/testing_requirements.txt @@ -6,3 +6,4 @@ pytest-cov==2.4.0 retrying==1.3.3 Sphinx==1.4.5 sphinx-rtd-theme==0.1.9 +paramiko==2.2.1