Merge pull request #312 from guardicore/master

Back-merge Master into Develop

monkey/common/cloud/aws_instance.py

@@ -1,22 +1,48 @@
+import json
 import re
 import urllib2
+import logging
+
 
 __author__ = 'itay.mizeretz'
 
+AWS_INSTANCE_METADATA_LOCAL_IP_ADDRESS = "169.254.169.254"
+AWS_LATEST_METADATA_URI_PREFIX = 'http://{0}/latest/'.format(AWS_INSTANCE_METADATA_LOCAL_IP_ADDRESS)
+ACCOUNT_ID_KEY = "accountId"
+
+
+logger = logging.getLogger(__name__)
+
 
 class AwsInstance(object):
+    """
+    Class which gives useful information about the current instance you're on.
+    """
+
     def __init__(self):
+        self.instance_id = None
+        self.region = None
+        self.account_id = None
+
         try:
-            self.instance_id = urllib2.urlopen('http://169.254.169.254/latest/meta-data/instance-id', timeout=2).read()
+            self.instance_id = urllib2.urlopen(
+                AWS_LATEST_METADATA_URI_PREFIX + 'meta-data/instance-id', timeout=2).read()
             self.region = self._parse_region(
-                urllib2.urlopen('http://169.254.169.254/latest/meta-data/placement/availability-zone').read())
-        except urllib2.URLError:
-            self.instance_id = None
-            self.region = None
+                urllib2.urlopen(AWS_LATEST_METADATA_URI_PREFIX + 'meta-data/placement/availability-zone').read())
+        except urllib2.URLError as e:
+            logger.error("Failed init of AwsInstance while getting metadata: {}".format(e.message))
+
+        try:
+            self.account_id = self._extract_account_id(
+                urllib2.urlopen(
+                    AWS_LATEST_METADATA_URI_PREFIX + 'dynamic/instance-identity/document', timeout=2).read())
+        except urllib2.URLError as e:
+            logger.error("Failed init of AwsInstance while getting dynamic instance data: {}".format(e.message))
 
     @staticmethod
     def _parse_region(region_url_response):
-        # For a list of regions: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html
+        # For a list of regions, see:
+        # https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html
         # This regex will find any AWS region format string in the response.
         re_phrase = r'((?:us|eu|ap|ca|cn|sa)-[a-z]*-[0-9])'
         finding = re.findall(re_phrase, region_url_response, re.IGNORECASE)
@@ -33,3 +59,21 @@ def get_region(self):
 
     def is_aws_instance(self):
         return self.instance_id is not None
+
+    @staticmethod
+    def _extract_account_id(instance_identity_document_response):
+        """
+        Extracts the account id from the dynamic/instance-identity/document metadata path.
+        Based on https://forums.aws.amazon.com/message.jspa?messageID=409028 which has a few more solutions,
+        in case Amazon break this mechanism.
+        :param instance_identity_document_response: json returned via the web page ../dynamic/instance-identity/document
+        :return: The account id
+        """
+        return json.loads(instance_identity_document_response)[ACCOUNT_ID_KEY]
+
+    def get_account_id(self):
+        """
+        :return:    the AWS account ID which "owns" this instance.
+        See https://docs.aws.amazon.com/general/latest/gr/acct-identifiers.html
+        """
+        return self.account_id

monkey/common/cloud/aws_service.py

@@ -1,23 +1,44 @@
+import logging
+
 import boto3
+import botocore
 from botocore.exceptions import ClientError
 
-__author__ = 'itay.mizeretz'
+from common.cloud.aws_instance import AwsInstance
+
+__author__ = ['itay.mizeretz', 'shay.nehmad']
+
+INSTANCE_INFORMATION_LIST_KEY = 'InstanceInformationList'
+INSTANCE_ID_KEY = 'InstanceId'
+COMPUTER_NAME_KEY = 'ComputerName'
+PLATFORM_TYPE_KEY = 'PlatformType'
+IP_ADDRESS_KEY = 'IPAddress'
+
+
+logger = logging.getLogger(__name__)
+
+
+def filter_instance_data_from_aws_response(response):
+    return [{
+        'instance_id': x[INSTANCE_ID_KEY],
+        'name': x[COMPUTER_NAME_KEY],
+        'os': x[PLATFORM_TYPE_KEY].lower(),
+        'ip_address': x[IP_ADDRESS_KEY]
+    } for x in response[INSTANCE_INFORMATION_LIST_KEY]]
 
 
 class AwsService(object):
     """
-    Supplies various AWS services
+    A wrapper class around the boto3 client and session modules, which supplies various AWS services.
+
+    This class will assume:
+        1. That it's running on an EC2 instance
+        2. That the instance is associated with the correct IAM role. See
+        https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html#iam-role for details.
     """
 
-    access_key_id = None
-    secret_access_key = None
     region = None
 
-    @staticmethod
-    def set_auth_params(access_key_id, secret_access_key):
-        AwsService.access_key_id = access_key_id
-        AwsService.secret_access_key = secret_access_key
-
     @staticmethod
     def set_region(region):
         AwsService.region = region
@@ -26,15 +47,11 @@ def set_region(region):
     def get_client(client_type, region=None):
         return boto3.client(
             client_type,
-            aws_access_key_id=AwsService.access_key_id,
-            aws_secret_access_key=AwsService.secret_access_key,
             region_name=region if region is not None else AwsService.region)
 
     @staticmethod
     def get_session():
-        return boto3.session.Session(
-            aws_access_key_id=AwsService.access_key_id,
-            aws_secret_access_key=AwsService.secret_access_key)
+        return boto3.session.Session()
 
     @staticmethod
     def get_regions():
@@ -50,14 +67,22 @@ def test_client():
 
     @staticmethod
     def get_instances():
-        return \
-            [
-                {
-                    'instance_id': x['InstanceId'],
-                    'name': x['ComputerName'],
-                    'os': x['PlatformType'].lower(),
-                    'ip_address': x['IPAddress']
-                }
-                for x in AwsService.get_client('ssm').describe_instance_information()['InstanceInformationList']
-            ]
+        """
+        Get the information for all instances with the relevant roles.
+
+        This function will assume that it's running on an EC2 instance with the correct IAM role.
+        See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html#iam-role for details.
+
+        :raises: botocore.exceptions.ClientError if can't describe local instance information.
+        :return: All visible instances from this instance
+        """
+        current_instance = AwsInstance()
+        local_ssm_client = boto3.client("ssm", current_instance.get_region())
+        try:
+            response = local_ssm_client.describe_instance_information()
 
+            filtered_instances_data = filter_instance_data_from_aws_response(response)
+            return filtered_instances_data
+        except botocore.exceptions.ClientError as e:
+            logger.warning("AWS client error while trying to get instances: " + e.message)
+            raise e

monkey/common/cloud/test_filter_instance_data_from_aws_response.py

@@ -0,0 +1,59 @@
+from unittest import TestCase
+from aws_service import filter_instance_data_from_aws_response
+
+import json
+
+
+__author__ = 'shay.nehmad'
+
+
+class TestFilter_instance_data_from_aws_response(TestCase):
+    def test_filter_instance_data_from_aws_response(self):
+        json_response_full = """
+        {
+            "InstanceInformationList": [ 
+                { 
+                     "ActivationId": "string",
+                     "AgentVersion": "string",
+                     "AssociationOverview": { 
+                        "DetailedStatus": "string",
+                        "InstanceAssociationStatusAggregatedCount": { 
+                           "string" : 6 
+                        }
+                     },
+                     "AssociationStatus": "string",
+                     "ComputerName": "string",
+                     "IamRole": "string",
+                     "InstanceId": "string",
+                     "IPAddress": "string",
+                     "IsLatestVersion": "True",
+                     "LastAssociationExecutionDate": 6,
+                     "LastPingDateTime": 6,
+                     "LastSuccessfulAssociationExecutionDate": 6,
+                     "Name": "string",
+                     "PingStatus": "string",
+                     "PlatformName": "string",
+                     "PlatformType": "string",
+                     "PlatformVersion": "string",
+                     "RegistrationDate": 6,
+                     "ResourceType": "string"
+                }
+            ],
+           "NextToken": "string"
+        }
+        """
+
+        json_response_empty = """
+            {
+                "InstanceInformationList": [],
+                "NextToken": "string"
+            }
+            """
+
+        self.assertEqual(filter_instance_data_from_aws_response(json.loads(json_response_empty)), [])
+        self.assertEqual(
+            filter_instance_data_from_aws_response(json.loads(json_response_full)),
+            [{'instance_id': u'string',
+              'ip_address': u'string',
+              'name': u'string',
+              'os': u'string'}])

monkey/monkey_island/cc/app.py

@@ -30,6 +30,7 @@
 from monkey_island.cc.resources.pba_file_download import PBAFileDownload
 from monkey_island.cc.services.config import ConfigService
 from monkey_island.cc.consts import MONKEY_ISLAND_ABS_PATH
+from monkey_island.cc.services.remote_run_aws import RemoteRunAwsService
 from monkey_island.cc.resources.pba_file_upload import FileUpload
 from monkey_island.cc.resources.attack_telem import AttackTelem
 
@@ -101,6 +102,9 @@ def init_app(mongo_url):
         database.init()
         ConfigService.init_config()
 
+    # If running on AWS, this will initialize the instance data, which is used "later" in the execution of the island.
+    RemoteRunAwsService.init()
+
     app.add_url_rule('/', 'serve_home', serve_home)
     app.add_url_rule('/<path:static_path>', 'serve_static_file', serve_static_file)
 

monkey/monkey_island/cc/exporter_init.py

@@ -1,19 +1,19 @@
-from monkey_island.cc.environment.environment import load_env_from_file, AWS
+import logging
+
 from monkey_island.cc.report_exporter_manager import ReportExporterManager
 from monkey_island.cc.resources.aws_exporter import AWSExporter
+from monkey_island.cc.services.remote_run_aws import RemoteRunAwsService
 
-__author__ = 'maor.rayzin'
+logger = logging.getLogger(__name__)
 
 
 def populate_exporter_list():
-
     manager = ReportExporterManager()
-    if is_aws_exporter_required():
+    RemoteRunAwsService.init()
+    if RemoteRunAwsService.is_running_on_aws():
         manager.add_exporter_to_list(AWSExporter)
 
+    if len(manager.get_exporters_list()) != 0:
+        logger.debug(
+            "Populated exporters list with the following exporters: {0}".format(str(manager.get_exporters_list())))
 
-def is_aws_exporter_required():
-    if str(load_env_from_file()) == AWS:
-        return True
-    else:
-        return False

monkey/monkey_island/cc/report_exporter_manager.py

@@ -29,6 +29,7 @@ def add_exporter_to_list(self, exporter):
     def export(self, report):
         try:
             for exporter in self._exporters_set:
+                logger.debug("Trying to export using " + repr(exporter))
                 exporter().handle_report(report)
         except Exception as e:
-            logger.exception('Failed to export report')
+            logger.exception('Failed to export report, error: ' + e.message)