diff --git a/.gitignore b/.gitignore index 03e5e394a0..cc30a9f93c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ .*.swp .DS_Store /venv* +.venv/ .cache/ build/ .idea/ diff --git a/cartography/intel/aws/ec2/auto_scaling_groups.py b/cartography/intel/aws/ec2/auto_scaling_groups.py index 3e456be736..cbaa5cb472 100644 --- a/cartography/intel/aws/ec2/auto_scaling_groups.py +++ b/cartography/intel/aws/ec2/auto_scaling_groups.py @@ -1,24 +1,37 @@ import logging -from typing import Dict -from typing import List +from collections import namedtuple +from typing import Any import boto3 import neo4j from .util import get_botocore_config +from cartography.client.core.tx import load +from cartography.graph.job import GraphJob +from cartography.models.aws.ec2.auto_scaling_groups import AutoScalingGroupSchema +from cartography.models.aws.ec2.auto_scaling_groups import EC2InstanceAutoScalingGroupSchema +from cartography.models.aws.ec2.auto_scaling_groups import EC2SubnetAutoScalingGroupSchema +from cartography.models.aws.ec2.launch_configurations import LaunchConfigurationSchema from cartography.util import aws_handle_regions -from cartography.util import run_cleanup_job from cartography.util import timeit logger = logging.getLogger(__name__) +AsgData = namedtuple( + 'AsgData', [ + "group_list", + "instance_list", + "subnet_list", + ], +) + @timeit @aws_handle_regions -def get_ec2_auto_scaling_groups(boto3_session: boto3.session.Session, region: str) -> List[Dict]: +def get_ec2_auto_scaling_groups(boto3_session: boto3.session.Session, region: str) -> list[dict]: client = boto3_session.client('autoscaling', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_auto_scaling_groups') - asgs: List[Dict] = [] + asgs: list[dict] = [] for page in paginator.paginate(): asgs.extend(page['AutoScalingGroups']) return asgs @@ -26,218 +39,167 @@ def get_ec2_auto_scaling_groups(boto3_session: boto3.session.Session, region: st @timeit @aws_handle_regions -def get_launch_configurations(boto3_session: boto3.session.Session, region: str) -> List[Dict]: +def get_launch_configurations(boto3_session: boto3.session.Session, region: str) -> list[dict]: client = boto3_session.client('autoscaling', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_launch_configurations') - lcs: List[Dict] = [] + lcs: list[dict] = [] for page in paginator.paginate(): lcs.extend(page['LaunchConfigurations']) return lcs +def transform_launch_configurations(configurations: list[dict[str, Any]]) -> list[dict[str, Any]]: + transformed_configurations = [] + for config in configurations: + transformed_configurations.append({ + 'AssociatePublicIpAddress': config.get('AssociatePublicIpAddress'), + 'LaunchConfigurationARN': config.get('LaunchConfigurationARN'), + 'LaunchConfigurationName': config.get('LaunchConfigurationName'), + 'CreatedTime': config.get('CreatedTime'), + 'ImageId': config.get('ImageId'), + 'KeyName': config.get('KeyName'), + 'SecurityGroups': config.get('SecurityGroups'), + 'InstanceType': config.get('InstanceType'), + 'KernelId': config.get('KernelId'), + 'RamdiskId': config.get('RamdiskId'), + 'InstanceMonitoring': config.get('InstanceMonitoring', {}).get('Enabled'), + 'SpotPrice': config.get('SpotPrice'), + 'IamInstanceProfile': config.get('IamInstanceProfile'), + 'EbsOptimized': config.get('EbsOptimized'), + 'PlacementTenancy': config.get('PlacementTenancy'), + }) + return transformed_configurations + + +def transform_auto_scaling_groups(groups: list[dict[str, Any]]) -> AsgData: + transformed_groups = [] + related_vpcs = [] + related_instances = [] + for group in groups: + transformed_groups.append({ + 'AutoScalingGroupARN': group['AutoScalingGroupARN'], + 'CapacityRebalance': group.get('CapacityRebalance'), + 'CreatedTime': str(group.get('CreatedTime')), + 'DefaultCooldown': group.get('DefaultCooldown'), + 'DesiredCapacity': group.get('DesiredCapacity'), + 'HealthCheckGracePeriod': group.get('HealthCheckGracePeriod'), + 'HealthCheckType': group.get('HealthCheckType'), + 'LaunchConfigurationName': group.get('LaunchConfigurationName'), + 'LaunchTemplateName': group.get('LaunchTemplate', {}).get('LaunchTemplateName'), + 'LaunchTemplateId': group.get('LaunchTemplate', {}).get('LaunchTemplateId'), + 'LaunchTemplateVersion': group.get('LaunchTemplate', {}).get('Version'), + 'MaxInstanceLifetime': group.get('MaxInstanceLifetime'), + 'MaxSize': group.get('MaxSize'), + 'MinSize': group.get('MinSize'), + 'AutoScalingGroupName': group.get('AutoScalingGroupName'), + 'NewInstancesProtectedFromScaleIn': group.get('NewInstancesProtectedFromScaleIn'), + 'Status': group.get('Status'), + }) + + if group.get('VPCZoneIdentifier', None): + vpclist = group['VPCZoneIdentifier'] + subnet_ids = vpclist.split(',') if ',' in vpclist else [vpclist] + subnets = [] + for subnet_id in subnet_ids: + subnets.append({ + 'VPCZoneIdentifier': subnet_id, + 'AutoScalingGroupARN': group['AutoScalingGroupARN'], + }) + related_vpcs.extend(subnets) + + for instance_data in group.get('Instances', []): + related_instances.append({ + 'InstanceId': instance_data['InstanceId'], + 'AutoScalingGroupARN': group['AutoScalingGroupARN'], + }) + + return AsgData( + group_list=transformed_groups, + instance_list=related_instances, + subnet_list=related_vpcs, + ) + + @timeit def load_launch_configurations( - neo4j_session: neo4j.Session, data: List[Dict], region: str, current_aws_account_id: str, update_tag: int, + neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int, ) -> None: - ingest_lc = """ - UNWIND $launch_configurations as lc - MERGE (config:LaunchConfiguration{id: lc.LaunchConfigurationARN}) - ON CREATE SET config.firstseen = timestamp(), config.name = lc.LaunchConfigurationName, - config.arn = lc.LaunchConfigurationARN, - config.created_time = lc.CreatedTime - SET config.lastupdated = $update_tag, config.image_id = lc.ImageId, - config.key_name = lc.KeyName, - config.security_groups = lc.SecurityGroups, - config.instance_type = lc.InstanceType, - config.kernel_id = lc.KernelId, - config.ramdisk_id = lc.RamdiskId, - config.instance_monitoring_enabled = lc.InstanceMonitoring.Enabled, - config.spot_price = lc.SpotPrice, - config.iam_instance_profile = lc.IamInstanceProfile, - config.ebs_optimized = lc.EbsOptimized, - config.associate_public_ip_address = lc.AssociatePublicIpAddress, - config.placement_tenancy = lc.PlacementTenancy, - config.region=$Region - WITH config - MATCH (aa:AWSAccount{id: $AWS_ACCOUNT_ID}) - MERGE (aa)-[r:RESOURCE]->(config) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - """ - for lc in data: - lc['CreatedTime'] = str(int(lc['CreatedTime'].timestamp())) - - neo4j_session.run( - ingest_lc, - launch_configurations=data, - AWS_ACCOUNT_ID=current_aws_account_id, + load( + neo4j_session, + LaunchConfigurationSchema(), + data, Region=region, - update_tag=update_tag, + AWS_ID=current_aws_account_id, + lastupdated=update_tag, ) -@timeit -def load_ec2_auto_scaling_groups( - neo4j_session: neo4j.Session, data: List[Dict], region: str, current_aws_account_id: str, update_tag: int, +def load_groups( + neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int, ) -> None: - ingest_group = """ - UNWIND $autoscaling_groups_list as ag - MERGE (group:AutoScalingGroup{arn: ag.AutoScalingGroupARN}) - ON CREATE SET group.firstseen = timestamp(), - group.createdtime = ag.CreatedTime - SET group.launchconfigurationname = ag.LaunchConfigurationName, - group.launchtemplatename = ag.LaunchTemplate.LaunchTemplateName, - group.launchtemplateid = ag.LaunchTemplate.LaunchTemplateId, - group.launchtemplateversion = ag.LaunchTemplate.Version, - group.maxsize = ag.MaxSize, group.minsize = ag.MinSize, group.defaultcooldown = ag.DefaultCooldown, - group.desiredcapacity = ag.DesiredCapacity, group.healthchecktype = ag.HealthCheckType, - group.healthcheckgraceperiod = ag.HealthCheckGracePeriod, group.status = ag.Status, - group.newinstancesprotectedfromscalein = ag.NewInstancesProtectedFromScaleIn, - group.maxinstancelifetime = ag.MaxInstanceLifetime, group.capacityrebalance = ag.CapacityRebalance, - group.name = ag.AutoScalingGroupName, - group.lastupdated = $update_tag, - group.region=$Region - WITH group - MATCH (aa:AWSAccount{id: $AWS_ACCOUNT_ID}) - MERGE (aa)-[r:RESOURCE]->(group) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - """ - - ingest_vpc = """ - UNWIND $vpc_id_list as vpc_id - MERGE (subnet:EC2Subnet{subnetid: vpc_id}) - ON CREATE SET subnet.firstseen = timestamp() - SET subnet.lastupdated = $update_tag - WITH subnet - MATCH (group:AutoScalingGroup{arn: $GROUPARN}) - MERGE (subnet)<-[r:VPC_IDENTIFIER]-(group) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - """ - - ingest_instance = """ - UNWIND $instances_list as i - MERGE (instance:Instance:EC2Instance{id: i.InstanceId}) - ON CREATE SET instance.firstseen = timestamp() - SET instance.lastupdated = $update_tag, instance.region=$Region - WITH instance - MATCH (group:AutoScalingGroup{arn: $GROUPARN}) - MERGE (instance)-[r:MEMBER_AUTO_SCALE_GROUP]->(group) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - WITH instance - MATCH (aa:AWSAccount{id: $AWS_ACCOUNT_ID}) - MERGE (aa)-[r:RESOURCE]->(instance) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - """ - - ingest_lts = """ - UNWIND $autoscaling_groups_list as ag - MATCH (group:AutoScalingGroup{arn: ag.AutoScalingGroupARN}) - MATCH (template:LaunchTemplate{id: ag.LaunchTemplate.LaunchTemplateId}) - MERGE (group)-[r:HAS_LAUNCH_TEMPLATE]->(template) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - """ - - ingest_lcs = """ - UNWIND $autoscaling_groups_list as ag - MATCH (group:AutoScalingGroup{arn: ag.AutoScalingGroupARN}) - MATCH (config:LaunchConfiguration{name: ag.LaunchConfigurationName}) - MERGE (group)-[r:HAS_LAUNCH_CONFIG]->(config) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - """ - - launch_configs = [] - launch_templates = [] - for group in data: - if group.get('LaunchConfigurationName'): - launch_configs.append(group) - if group.get('LaunchTemplate'): - launch_templates.append(group) - - group['CreatedTime'] = str(group['CreatedTime']) - - neo4j_session.run( - ingest_group, - autoscaling_groups_list=data, - AWS_ACCOUNT_ID=current_aws_account_id, + load( + neo4j_session, + AutoScalingGroupSchema(), + data, Region=region, - update_tag=update_tag, + AWS_ID=current_aws_account_id, + lastupdated=update_tag, ) - neo4j_session.run( - ingest_lcs, - autoscaling_groups_list=launch_configs, - AWS_ACCOUNT_ID=current_aws_account_id, + + +def load_asg_subnets( + neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int, +) -> None: + load( + neo4j_session, + EC2SubnetAutoScalingGroupSchema(), + data, Region=region, - update_tag=update_tag, + AWS_ID=current_aws_account_id, + lastupdated=update_tag, ) - neo4j_session.run( - ingest_lts, - autoscaling_groups_list=launch_templates, - AWS_ACCOUNT_ID=current_aws_account_id, + + +def load_asg_instances( + neo4j_session: neo4j.Session, data: list[dict], region: str, current_aws_account_id: str, update_tag: int, +) -> None: + load( + neo4j_session, + EC2InstanceAutoScalingGroupSchema(), + data, Region=region, - update_tag=update_tag, + AWS_ID=current_aws_account_id, + lastupdated=update_tag, ) - for group in data: - group_arn = group["AutoScalingGroupARN"] - if group.get('VPCZoneIdentifier'): - vpclist = group["VPCZoneIdentifier"] - if ',' in vpclist: - data = vpclist.split(',') - else: - data = vpclist - neo4j_session.run( - ingest_vpc, - vpc_id_list=data, - GROUPARN=group_arn, - update_tag=update_tag, - ) - - if group.get("Instances"): - data = group["Instances"] - neo4j_session.run( - ingest_instance, - instances_list=data, - GROUPARN=group_arn, - AWS_ACCOUNT_ID=current_aws_account_id, - Region=region, - update_tag=update_tag, - ) - @timeit -def cleanup_ec2_auto_scaling_groups(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None: - run_cleanup_job( - 'aws_ingest_ec2_auto_scaling_groups_cleanup.json', - neo4j_session, - common_job_parameters, - ) +def load_auto_scaling_groups( + neo4j_session: neo4j.Session, data: AsgData, region: str, current_aws_account_id: str, update_tag: int, +) -> None: + load_groups(neo4j_session, data.group_list, region, current_aws_account_id, update_tag) + load_asg_instances(neo4j_session, data.instance_list, region, current_aws_account_id, update_tag) + load_asg_subnets(neo4j_session, data.subnet_list, region, current_aws_account_id, update_tag) @timeit -def cleanup_ec2_launch_configurations(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None: - run_cleanup_job( - 'aws_import_ec2_launch_configurations_cleanup.json', - neo4j_session, - common_job_parameters, - ) +def cleanup(neo4j_session: neo4j.Session, common_job_parameters: dict[str, Any]) -> None: + logger.debug("Running EC2 instance cleanup") + GraphJob.from_node_schema(AutoScalingGroupSchema(), common_job_parameters).run(neo4j_session) + GraphJob.from_node_schema(LaunchConfigurationSchema(), common_job_parameters).run(neo4j_session) @timeit def sync_ec2_auto_scaling_groups( - neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], - current_aws_account_id: str, update_tag: int, common_job_parameters: Dict, + neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: list[str], + current_aws_account_id: str, update_tag: int, common_job_parameters: dict, ) -> None: for region in regions: logger.debug("Syncing auto scaling groups for region '%s' in account '%s'.", region, current_aws_account_id) lc_data = get_launch_configurations(boto3_session, region) - load_launch_configurations(neo4j_session, lc_data, region, current_aws_account_id, update_tag) - data = get_ec2_auto_scaling_groups(boto3_session, region) - load_ec2_auto_scaling_groups(neo4j_session, data, region, current_aws_account_id, update_tag) - cleanup_ec2_auto_scaling_groups(neo4j_session, common_job_parameters) - cleanup_ec2_launch_configurations(neo4j_session, common_job_parameters) + asg_data = get_ec2_auto_scaling_groups(boto3_session, region) + lc_transformed = transform_launch_configurations(lc_data) + asg_transformed = transform_auto_scaling_groups(asg_data) + load_launch_configurations(neo4j_session, lc_transformed, region, current_aws_account_id, update_tag) + load_auto_scaling_groups(neo4j_session, asg_transformed, region, current_aws_account_id, update_tag) + cleanup(neo4j_session, common_job_parameters) diff --git a/cartography/intel/aws/ec2/instances.py b/cartography/intel/aws/ec2/instances.py index d288c27e63..1c6ae240c1 100644 --- a/cartography/intel/aws/ec2/instances.py +++ b/cartography/intel/aws/ec2/instances.py @@ -11,6 +11,7 @@ from cartography.client.core.tx import load from cartography.graph.job import GraphJob from cartography.intel.aws.ec2.util import get_botocore_config +from cartography.models.aws.ec2.auto_scaling_groups import EC2InstanceAutoScalingGroupSchema from cartography.models.aws.ec2.instances import EC2InstanceSchema from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceInstanceSchema @@ -308,6 +309,7 @@ def cleanup(neo4j_session: neo4j.Session, common_job_parameters: Dict[str, Any]) logger.debug("Running EC2 instance cleanup") GraphJob.from_node_schema(EC2ReservationSchema(), common_job_parameters).run(neo4j_session) GraphJob.from_node_schema(EC2InstanceSchema(), common_job_parameters).run(neo4j_session) + GraphJob.from_node_schema(EC2InstanceAutoScalingGroupSchema(), common_job_parameters).run(neo4j_session) @timeit diff --git a/cartography/intel/aws/ec2/subnets.py b/cartography/intel/aws/ec2/subnets.py index d306049835..a42aedbb4b 100644 --- a/cartography/intel/aws/ec2/subnets.py +++ b/cartography/intel/aws/ec2/subnets.py @@ -7,6 +7,7 @@ from .util import get_botocore_config from cartography.graph.job import GraphJob +from cartography.models.aws.ec2.auto_scaling_groups import EC2SubnetAutoScalingGroupSchema from cartography.models.aws.ec2.subnet_instance import EC2SubnetInstanceSchema from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job @@ -79,6 +80,7 @@ def load_subnets( def cleanup_subnets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None: run_cleanup_job('aws_ingest_subnets_cleanup.json', neo4j_session, common_job_parameters) GraphJob.from_node_schema(EC2SubnetInstanceSchema(), common_job_parameters).run(neo4j_session) + GraphJob.from_node_schema(EC2SubnetAutoScalingGroupSchema(), common_job_parameters).run(neo4j_session) @timeit diff --git a/cartography/models/aws/ec2/auto_scaling_groups.py b/cartography/models/aws/ec2/auto_scaling_groups.py new file mode 100644 index 0000000000..c86fc6a707 --- /dev/null +++ b/cartography/models/aws/ec2/auto_scaling_groups.py @@ -0,0 +1,204 @@ +from dataclasses import dataclass + +from cartography.models.core.common import PropertyRef +from cartography.models.core.nodes import CartographyNodeProperties +from cartography.models.core.nodes import CartographyNodeSchema +from cartography.models.core.relationships import CartographyRelProperties +from cartography.models.core.relationships import CartographyRelSchema +from cartography.models.core.relationships import LinkDirection +from cartography.models.core.relationships import make_target_node_matcher +from cartography.models.core.relationships import OtherRelationships +from cartography.models.core.relationships import TargetNodeMatcher + + +@dataclass(frozen=True) +class AutoScalingGroupNodeProperties(CartographyNodeProperties): + id: PropertyRef = PropertyRef('AutoScalingGroupARN') + arn: PropertyRef = PropertyRef('AutoScalingGroupARN') + capacityrebalance: PropertyRef = PropertyRef('CapacityRebalance') + createdtime: PropertyRef = PropertyRef('CreatedTime') + defaultcooldown: PropertyRef = PropertyRef('DefaultCooldown') + desiredcapacity: PropertyRef = PropertyRef('DesiredCapacity') + healthcheckgraceperiod: PropertyRef = PropertyRef('HealthCheckGracePeriod') + healthchecktype: PropertyRef = PropertyRef('HealthCheckType') + launchconfigurationname: PropertyRef = PropertyRef('LaunchConfigurationName') + launchtemplatename: PropertyRef = PropertyRef('LaunchTemplateName') + launchtemplateid: PropertyRef = PropertyRef('LaunchTemplateId') + launchtemplateversion: PropertyRef = PropertyRef('LaunchTemplateVersion') + maxinstancelifetime: PropertyRef = PropertyRef('MaxInstanceLifetime') + maxsize: PropertyRef = PropertyRef('MaxSize') + minsize: PropertyRef = PropertyRef('MinSize') + name: PropertyRef = PropertyRef('AutoScalingGroupName') + newinstancesprotectedfromscalein: PropertyRef = PropertyRef('NewInstancesProtectedFromScaleIn') + region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) + status: PropertyRef = PropertyRef('Status') + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +# EC2 to AutoScalingGroup +@dataclass(frozen=True) +class EC2InstanceToAwsAccountRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2InstanceToAWSAccount(CartographyRelSchema): + target_node_label: str = 'AWSAccount' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('AWS_ID', set_in_kwargs=True)}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "RESOURCE" + properties: EC2InstanceToAwsAccountRelProperties = EC2InstanceToAwsAccountRelProperties() + + +@dataclass(frozen=True) +class EC2InstanceToAutoScalingGroupRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2InstanceToAutoScalingGroup(CartographyRelSchema): + target_node_label: str = 'AutoScalingGroup' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('AutoScalingGroupARN')}, + ) + direction: LinkDirection = LinkDirection.OUTWARD + rel_label: str = "MEMBER_AUTO_SCALE_GROUP" + properties: EC2InstanceToAutoScalingGroupRelProperties = EC2InstanceToAutoScalingGroupRelProperties() + + +@dataclass(frozen=True) +class EC2InstanceAutoScalingGroupProperties(CartographyNodeProperties): + id: PropertyRef = PropertyRef('InstanceId') + instanceid: PropertyRef = PropertyRef('InstanceId', extra_index=True) + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2InstanceAutoScalingGroupSchema(CartographyNodeSchema): + label: str = 'EC2Instance' + properties: EC2InstanceAutoScalingGroupProperties = EC2InstanceAutoScalingGroupProperties() + sub_resource_relationship: EC2InstanceToAWSAccount = EC2InstanceToAWSAccount() + other_relationships: OtherRelationships = OtherRelationships( + [ + EC2InstanceToAutoScalingGroup(), + ], + ) + + +# EC2Subnet to AutoScalingGroup +@dataclass(frozen=True) +class EC2SubnetToAwsAccountRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2SubnetToAWSAccount(CartographyRelSchema): + target_node_label: str = 'AWSAccount' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('AWS_ID', set_in_kwargs=True)}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "RESOURCE" + properties: EC2SubnetToAwsAccountRelProperties = EC2SubnetToAwsAccountRelProperties() + + +@dataclass(frozen=True) +class EC2SubnetToAutoScalingGroupRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2SubnetToAutoScalingGroup(CartographyRelSchema): + target_node_label: str = 'AutoScalingGroup' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('AutoScalingGroupARN')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "VPC_IDENTIFIER" + properties: EC2SubnetToAutoScalingGroupRelProperties = EC2SubnetToAutoScalingGroupRelProperties() + + +@dataclass(frozen=True) +class EC2SubnetAutoScalingGroupNodeProperties(CartographyNodeProperties): + id: PropertyRef = PropertyRef('VPCZoneIdentifier') + subnetid: PropertyRef = PropertyRef('VPCZoneIdentifier', extra_index=True) + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2SubnetAutoScalingGroupSchema(CartographyNodeSchema): + label: str = 'EC2Subnet' + properties: EC2SubnetAutoScalingGroupNodeProperties = EC2SubnetAutoScalingGroupNodeProperties() + sub_resource_relationship: EC2SubnetToAWSAccount = EC2SubnetToAWSAccount() + other_relationships: OtherRelationships = OtherRelationships( + [ + EC2SubnetToAutoScalingGroup(), + ], + ) + + +# AutoScalingGroup +@dataclass(frozen=True) +class AutoScalingGroupToAwsAccountRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class AutoScalingGroupToAWSAccount(CartographyRelSchema): + target_node_label: str = 'AWSAccount' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('AWS_ID', set_in_kwargs=True)}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "RESOURCE" + properties: AutoScalingGroupToAwsAccountRelProperties = AutoScalingGroupToAwsAccountRelProperties() + + +@dataclass(frozen=True) +class AutoScalingGroupToLaunchTemplateRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class AutoScalingGroupToLaunchTemplate(CartographyRelSchema): + target_node_label: str = 'LaunchTemplate' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('LaunchTemplateId')}, + ) + direction: LinkDirection = LinkDirection.OUTWARD + rel_label: str = "HAS_LAUNCH_TEMPLATE" + properties: AutoScalingGroupToLaunchTemplateRelProperties = AutoScalingGroupToLaunchTemplateRelProperties() + + +@dataclass(frozen=True) +class AutoScalingGroupToLaunchConfigurationRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class AutoScalingGroupToLaunchConfiguration(CartographyRelSchema): + target_node_label: str = 'LaunchConfiguration' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'name': PropertyRef('LaunchConfigurationName')}, + ) + direction: LinkDirection = LinkDirection.OUTWARD + rel_label: str = "HAS_LAUNCH_CONFIG" + properties: AutoScalingGroupToLaunchConfigurationRelProperties = ( + AutoScalingGroupToLaunchConfigurationRelProperties() + ) + + +@dataclass(frozen=True) +class AutoScalingGroupSchema(CartographyNodeSchema): + label: str = 'AutoScalingGroup' + properties: AutoScalingGroupNodeProperties = AutoScalingGroupNodeProperties() + sub_resource_relationship: AutoScalingGroupToAWSAccount = AutoScalingGroupToAWSAccount() + other_relationships: OtherRelationships = OtherRelationships( + [ + AutoScalingGroupToLaunchTemplate(), + AutoScalingGroupToLaunchConfiguration(), + ], + ) diff --git a/cartography/models/aws/ec2/launch_configurations.py b/cartography/models/aws/ec2/launch_configurations.py new file mode 100644 index 0000000000..d80364e5a4 --- /dev/null +++ b/cartography/models/aws/ec2/launch_configurations.py @@ -0,0 +1,55 @@ +from dataclasses import dataclass + +from cartography.models.core.common import PropertyRef +from cartography.models.core.nodes import CartographyNodeProperties +from cartography.models.core.nodes import CartographyNodeSchema +from cartography.models.core.relationships import CartographyRelProperties +from cartography.models.core.relationships import CartographyRelSchema +from cartography.models.core.relationships import LinkDirection +from cartography.models.core.relationships import make_target_node_matcher +from cartography.models.core.relationships import TargetNodeMatcher + + +@dataclass(frozen=True) +class LaunchConfigurationNodeProperties(CartographyNodeProperties): + id: PropertyRef = PropertyRef('LaunchConfigurationARN') + arn: PropertyRef = PropertyRef('LaunchConfigurationARN') + created_time = PropertyRef('CreatedTime') + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + image_id: PropertyRef = PropertyRef('ImageId') + key_name: PropertyRef = PropertyRef('KeyName') + name: PropertyRef = PropertyRef('LaunchConfigurationName') + security_groups: PropertyRef = PropertyRef('SecurityGroups') + instance_type: PropertyRef = PropertyRef('InstanceType') + kernel_id: PropertyRef = PropertyRef('KernelId') + ramdisk_id: PropertyRef = PropertyRef('RamdiskId') + instance_monitoring_enabled: PropertyRef = PropertyRef('InstanceMonitoringEnabled') + spot_price: PropertyRef = PropertyRef('SpotPrice') + iam_instance_profile: PropertyRef = PropertyRef('IamInstanceProfile') + ebs_optimized: PropertyRef = PropertyRef('EbsOptimized') + associate_public_ip_address: PropertyRef = PropertyRef('AssociatePublicIpAddress') + placement_tenancy: PropertyRef = PropertyRef('PlacementTenancy') + region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) + + +@dataclass(frozen=True) +class LaunchConfigurationToAwsAccountRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class LaunchConfigurationToAwsAccount(CartographyRelSchema): + target_node_label: str = 'AWSAccount' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('AWS_ID', set_in_kwargs=True)}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "RESOURCE" + properties: LaunchConfigurationToAwsAccountRelProperties = LaunchConfigurationToAwsAccountRelProperties() + + +@dataclass(frozen=True) +class LaunchConfigurationSchema(CartographyNodeSchema): + label: str = 'LaunchConfiguration' + properties: LaunchConfigurationNodeProperties = LaunchConfigurationNodeProperties() + sub_resource_relationship: LaunchConfigurationToAwsAccount = LaunchConfigurationToAwsAccount() diff --git a/tests/data/aws/ec2/auto_scaling_groups.py b/tests/data/aws/ec2/auto_scaling_groups.py index 6012824083..079359ddb6 100644 --- a/tests/data/aws/ec2/auto_scaling_groups.py +++ b/tests/data/aws/ec2/auto_scaling_groups.py @@ -1,13 +1,74 @@ import datetime from datetime import timezone as tz - GET_LAUNCH_CONFIGURATIONS = [ { - 'LaunchConfigurationName': 'example', - 'LaunchConfigurationARN': 'arn:aws:autoscaling:us-east-1:000000000000:launchConfiguration:00000000-0000-0000-0000-000000000000:launchConfigurationName/example', # noqa:E501 + 'LaunchConfigurationName': 'example-lc-1', + 'LaunchConfigurationARN': 'arn:aws:autoscaling:us-east-1:000000000000:launchConfiguration:00000000-0000-0000-0000-000000000000:launchConfigurationName/example-lc-1', # noqa:E501 + 'ImageId': 'ami-00000000000000000', + 'KeyName': 'example-key', + 'SecurityGroups': [ + 'sg-00000000000000000', + ], + 'ClassicLinkVPCSecurityGroups': [], + 'UserData': '...', + 'InstanceType': 'r5.4xlarge', + 'KernelId': '', + 'RamdiskId': '', + 'BlockDeviceMappings': [ + { + 'DeviceName': '/dev/xvda', + 'Ebs': { + 'VolumeSize': 200, + 'VolumeType': 'gp2', + 'DeleteOnTermination': True, + }, + }, + ], + 'InstanceMonitoring': { + 'Enabled': False, + }, + 'IamInstanceProfile': 'example-lc-1', + 'CreatedTime': datetime.datetime(2021, 9, 21, 10, 55, 34, 222000, tzinfo=tz.utc), + 'EbsOptimized': True, + 'AssociatePublicIpAddress': True, + }, + { + 'LaunchConfigurationName': 'example-lc-2', + 'LaunchConfigurationARN': 'arn:aws:autoscaling:us-east-1:000000000000:launchConfiguration:00000000-0000-0000-0000-000000000000:launchConfigurationName/example-lc-2', # noqa:E501 + 'ImageId': 'ami-00000000000000000', + 'KeyName': 'example-key', + 'SecurityGroups': [ + 'sg-00000000000000000', + ], + 'ClassicLinkVPCSecurityGroups': [], + 'UserData': '...', + 'InstanceType': 'r5.4xlarge', + 'KernelId': '', + 'RamdiskId': '', + 'BlockDeviceMappings': [ + { + 'DeviceName': '/dev/xvda', + 'Ebs': { + 'VolumeSize': 200, + 'VolumeType': 'gp2', + 'DeleteOnTermination': True, + }, + }, + ], + 'InstanceMonitoring': { + 'Enabled': False, + }, + 'IamInstanceProfile': 'example-lc-2', + 'CreatedTime': datetime.datetime(2021, 9, 21, 10, 55, 34, 222000, tzinfo=tz.utc), + 'EbsOptimized': True, + 'AssociatePublicIpAddress': True, + }, + { + 'LaunchConfigurationName': 'example-lc-3', + 'LaunchConfigurationARN': 'arn:aws:autoscaling:us-east-1:000000000000:launchConfiguration:00000000-0000-0000-0000-000000000000:launchConfigurationName/example-lc-3', # noqa:E501 'ImageId': 'ami-00000000000000000', - 'KeyName': 'example-00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00', + 'KeyName': 'example-key', 'SecurityGroups': [ 'sg-00000000000000000', ], @@ -29,8 +90,75 @@ 'InstanceMonitoring': { 'Enabled': False, }, - 'IamInstanceProfile': 'example', + 'IamInstanceProfile': 'example-lc-3', 'CreatedTime': datetime.datetime(2021, 9, 21, 10, 55, 34, 222000, tzinfo=tz.utc), 'EbsOptimized': True, + 'AssociatePublicIpAddress': True, + }, +] + +GET_AUTO_SCALING_GROUPS = [ + { + 'AutoScalingGroupName': 'example-asg-1', + 'AutoScalingGroupARN': 'arn:aws:autoscaling:us-east-1:000000000000:autoScalingGroup:00000000-0000-0000-0000-000000000000:autoScalingGroupName/example-asg-1', # noqa:E501 + 'LaunchConfigurationName': 'example-lc-1', + 'MinSize': 1, + 'MaxSize': 1, + 'DesiredCapacity': 1, + 'DefaultCooldown': 300, + 'AvailabilityZones': [ + 'us-east-1a', + 'us-east-1b', + ], + 'LoadBalancerNames': [], + 'TargetGroupARNs': [], + 'HealthCheckType': 'EC2', + 'HealthCheckGracePeriod': 300, + 'Instances': [], + 'CreatedTime': datetime.datetime(2021, 9, 21, 10, 55, 34, 222000, tzinfo=tz.utc), + 'SuspendedProcesses': [], + 'VPCZoneIdentifier': 'subnet-00000000000000000,subnet-11111111111111111', + 'NewInstancesProtectedFromScaleIn': False, + 'ServiceLinkedRoleARN': 'arn:aws:iam::000000000000:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling', # noqa:E501 + 'MaxInstanceLifetime': 2592000, + 'CapacityRebalance': False, + }, + { + 'AutoScalingGroupName': 'example-asg-2', + 'AutoScalingGroupARN': 'arn:aws:autoscaling:us-east-1:000000000000:autoScalingGroup:00000000-0000-0000-0000-000000000000:autoScalingGroupName/example-asg-2', # noqa:E501 + 'MinSize': 1, + 'MaxSize': 1, + 'DesiredCapacity': 1, + 'DefaultCooldown': 300, + 'AvailabilityZones': [ + 'us-east-1a', + 'us-east-1b', + ], + 'LoadBalancerNames': [], + 'TargetGroupARNs': [], + 'HealthCheckType': 'EC2', + 'HealthCheckGracePeriod': 300, + # Should match instance IDs from cartography.tests.data.aws.ec2.instances.py + 'Instances': [ + { + "InstanceId": "i-01", + }, + { + "InstanceId": "i-02", + }, + { + "InstanceId": "i-03", + }, + { + "InstanceId": "i-04", + }, + ], + 'CreatedTime': datetime.datetime(2021, 9, 21, 10, 55, 34, 222000, tzinfo=tz.utc), + 'SuspendedProcesses': [], + 'VPCZoneIdentifier': 'subnet-00000000000000000,subnet-11111111111111111', + 'NewInstancesProtectedFromScaleIn': False, + 'ServiceLinkedRoleARN': 'arn:aws:iam::000000000000:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling', # noqa:E501 + 'MaxInstanceLifetime': 2592000, + 'CapacityRebalance': False, }, ] diff --git a/tests/integration/cartography/intel/aws/ec2/test_auto_scaling_groups.py b/tests/integration/cartography/intel/aws/ec2/test_auto_scaling_groups.py deleted file mode 100644 index 0b6245595d..0000000000 --- a/tests/integration/cartography/intel/aws/ec2/test_auto_scaling_groups.py +++ /dev/null @@ -1,40 +0,0 @@ -import cartography.intel.aws.ec2 -import tests.data.aws.ec2.auto_scaling_groups - - -TEST_ACCOUNT_ID = '000000000000' -TEST_REGION = 'us-east-1' -TEST_UPDATE_TAG = 123456789 - - -def test_load_launch_configurations(neo4j_session, *args): - data = tests.data.aws.ec2.auto_scaling_groups.GET_LAUNCH_CONFIGURATIONS - cartography.intel.aws.ec2.auto_scaling_groups.load_launch_configurations( - neo4j_session, - data, - TEST_REGION, - TEST_ACCOUNT_ID, - TEST_UPDATE_TAG, - ) - expected_nodes = { - ( - "example", - "arn:aws:autoscaling:us-east-1:000000000000:launchConfiguration:00000000-0000-0000-0000-000000000000:launchConfigurationName/example", # noqa:E501 - "1632221734", - ), - } - - nodes = neo4j_session.run( - """ - MATCH (n:LaunchConfiguration) return n.name, n.id, n.created_time - """, - ) - actual_nodes = { - ( - n['n.name'], - n['n.id'], - n['n.created_time'], - ) - for n in nodes - } - assert actual_nodes == expected_nodes diff --git a/tests/integration/cartography/intel/aws/ec2/test_ec2_auto_scaling_groups.py b/tests/integration/cartography/intel/aws/ec2/test_ec2_auto_scaling_groups.py new file mode 100644 index 0000000000..26543e8bbd --- /dev/null +++ b/tests/integration/cartography/intel/aws/ec2/test_ec2_auto_scaling_groups.py @@ -0,0 +1,112 @@ +from unittest.mock import MagicMock +from unittest.mock import patch + +from cartography.intel.aws.ec2 import auto_scaling_groups +from cartography.intel.aws.ec2 import instances +from cartography.intel.aws.ec2.auto_scaling_groups import sync_ec2_auto_scaling_groups +from tests.data.aws.ec2.auto_scaling_groups import GET_AUTO_SCALING_GROUPS +from tests.data.aws.ec2.auto_scaling_groups import GET_LAUNCH_CONFIGURATIONS +from tests.data.aws.ec2.instances import DESCRIBE_INSTANCES +from tests.integration.cartography.intel.aws.common import create_test_account +from tests.integration.util import check_nodes +from tests.integration.util import check_rels + + +TEST_ACCOUNT_ID = '000000000000' +TEST_REGION = 'us-east-1' +TEST_UPDATE_TAG = 123456789 + + +@patch.object(auto_scaling_groups, 'get_ec2_auto_scaling_groups', return_value=GET_AUTO_SCALING_GROUPS) +@patch.object(auto_scaling_groups, 'get_launch_configurations', return_value=GET_LAUNCH_CONFIGURATIONS) +@patch.object(instances, 'get_ec2_instances', return_value=DESCRIBE_INSTANCES['Reservations']) +def test_sync_ec2_auto_scaling_groups(mock_get_instances, mock_get_launch_configs, mock_get_asgs, neo4j_session): + """ + Ensure that auto scaling groups actually get loaded and have their key fields + """ + # Arrange + boto3_session = MagicMock() + create_test_account(neo4j_session, TEST_ACCOUNT_ID, TEST_UPDATE_TAG) + # Ensure there are instances in the graph + instances.sync_ec2_instances( + neo4j_session, + boto3_session, + [TEST_REGION], + TEST_ACCOUNT_ID, + TEST_UPDATE_TAG, + {'UPDATE_TAG': TEST_UPDATE_TAG, 'AWS_ID': TEST_ACCOUNT_ID}, + ) + + # Act + sync_ec2_auto_scaling_groups( + neo4j_session, + boto3_session, + [TEST_REGION], + TEST_ACCOUNT_ID, + TEST_UPDATE_TAG, + {'UPDATE_TAG': TEST_UPDATE_TAG, 'AWS_ID': TEST_ACCOUNT_ID}, + ) + + # Assert + assert check_nodes(neo4j_session, 'AutoScalingGroup', ['arn', 'name']) == { + (GET_AUTO_SCALING_GROUPS[0]['AutoScalingGroupARN'], GET_AUTO_SCALING_GROUPS[0]['AutoScalingGroupName']), + (GET_AUTO_SCALING_GROUPS[1]['AutoScalingGroupARN'], GET_AUTO_SCALING_GROUPS[1]['AutoScalingGroupName']), + } + + assert check_nodes(neo4j_session, 'LaunchConfiguration', ['id', 'arn', 'name']) == { + ( + GET_LAUNCH_CONFIGURATIONS[0]['LaunchConfigurationARN'], + GET_LAUNCH_CONFIGURATIONS[0]['LaunchConfigurationARN'], + GET_LAUNCH_CONFIGURATIONS[0]['LaunchConfigurationName'], + ), + ( + GET_LAUNCH_CONFIGURATIONS[1]['LaunchConfigurationARN'], + GET_LAUNCH_CONFIGURATIONS[1]['LaunchConfigurationARN'], + GET_LAUNCH_CONFIGURATIONS[1]['LaunchConfigurationName'], + ), + ( + GET_LAUNCH_CONFIGURATIONS[2]['LaunchConfigurationARN'], + GET_LAUNCH_CONFIGURATIONS[2]['LaunchConfigurationARN'], + GET_LAUNCH_CONFIGURATIONS[2]['LaunchConfigurationName'], + ), + } + + assert check_rels( + neo4j_session, + node_1_label='AutoScalingGroup', + node_1_attr='id', + node_2_label='AWSAccount', + node_2_attr='id', + rel_label='RESOURCE', + rel_direction_right=False, + ) == { + (GET_AUTO_SCALING_GROUPS[0]['AutoScalingGroupARN'], TEST_ACCOUNT_ID), + (GET_AUTO_SCALING_GROUPS[1]['AutoScalingGroupARN'], TEST_ACCOUNT_ID), + } + + assert check_rels( + neo4j_session, + node_1_label='AutoScalingGroup', + node_1_attr='id', + node_2_label='LaunchConfiguration', + node_2_attr='name', + rel_label='HAS_LAUNCH_CONFIG', + rel_direction_right=True, + ) == { + (GET_AUTO_SCALING_GROUPS[0]['AutoScalingGroupARN'], GET_LAUNCH_CONFIGURATIONS[0]['LaunchConfigurationName']), + } + + assert check_rels( + neo4j_session, + node_1_label='AutoScalingGroup', + node_1_attr='id', + node_2_label='EC2Instance', + node_2_attr='id', + rel_label='MEMBER_AUTO_SCALE_GROUP', + rel_direction_right=False, + ) == { + (GET_AUTO_SCALING_GROUPS[1]['AutoScalingGroupARN'], "i-01"), + (GET_AUTO_SCALING_GROUPS[1]['AutoScalingGroupARN'], "i-02"), + (GET_AUTO_SCALING_GROUPS[1]['AutoScalingGroupARN'], "i-03"), + (GET_AUTO_SCALING_GROUPS[1]['AutoScalingGroupARN'], "i-04"), + }