Clean up instance profiles in deprovision (#334)

* Clean up instance profiles in deprovision

* Fix bug in #239

* Fix pytype

* Update deprovisoin logic

* Cache pytype

* Fix pytype

* Disable pylint

.github/workflows/lint.yml

@@ -13,40 +13,32 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: "3.8"
-      - name: Cache pip dependencies
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ hashFiles('setup.py') }}-pytype
+          cache: 'pip'
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -e ".[gateway,solver]"
           pip install pytype
       - name: Run pytype
         run: pytype skylark --config .pytype.cfg -j auto
-  pylint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
-        with:
-          python-version: "3.8"
-      - name: Cache pip dependencies
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ hashFiles('setup.py') }}-pylint-pip
-      - name: Cache pip dependencies
-        uses: actions/cache@v2
-        with:
-          path: .pytype
-          key: ${{ hashFiles('setup.py') }}-pylint-build
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -e .
-          pip install pylint
-      - name: Run pylint
-        run: (find skylark -type f -name "*.py" | xargs pylint --rcfile=.pylintrc) || true
+  # pylint:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Set up Python 3.8
+  #       uses: actions/setup-python@v2
+  #       with:
+  #         python-version: "3.8"
+  #         cache: 'pip'
+  #     - name: Cache pip dependencies
+  #       uses: actions/cache@v2
+  #       with:
+  #         path: ~/.cache/pip
+  #         key: ${{ hashFiles('setup.py') }}-pylint-pip
+  #     - name: Install dependencies
+  #       run: |
+  #         python -m pip install --upgrade pip
+  #         pip install -e .
+  #         pip install pylint
+  #     - name: Run pylint
+  #       run: (find skylark -type f -name "*.py" | xargs pylint --rcfile=.pylintrc) || true

skylark/cli/cli_helper.py

@@ -400,14 +400,18 @@ def run():
         typer.secho(f"Deprovisioning {len(instances)} instances", fg="yellow", bold=True)
         do_parallel(lambda instance: instance.terminate_instance(), instances, desc="Deprovisioning", spinner=True, spinner_persist=True)
     else:
-        typer.secho("No instances to deprovision, exiting...", fg="yellow", bold=True)
+        typer.secho("No instances to deprovision", fg="yellow", bold=True)
 
-    # remove skylark vpc
     if AWSAuthentication().enabled():
         aws = AWSCloudProvider()
+        # remove skylark vpc
         vpcs = do_parallel(partial(aws.get_vpcs), aws.region_list(), desc="Querying VPCs", spinner=True)
         args = [(x[0], vpc.id) for x in vpcs for vpc in x[1]]
-        do_parallel(lambda args: aws.delete_vpc(*args), args, desc="Deleting VPCs", spinner=True, spinner_persist=True)
+        do_parallel(lambda args: aws.remove_sg_ips(*args), args, desc="Removing IPs from VPCs", spinner=True, spinner_persist=True)
+        # remove all instance profiles
+        profiles = aws.list_instance_profiles(prefix="skylark-aws")
+        if profiles:
+            do_parallel(aws.delete_instance_profile, profiles, desc="Deleting instance profiles", spinner=True, spinner_persist=True, n=4)
 
 
 def load_aws_config(config: SkylarkConfig) -> SkylarkConfig:

skylark/compute/aws/aws_cloud_provider.py

@@ -207,6 +207,34 @@ def delete_vpc(self, region: str, vpcid: str):
         # finally, delete the vpc
         ec2client.delete_vpc(VpcId=vpcid)
 
+    def remove_sg_ips(self, region: str, vpcid: str):
+        """Remove all IPs from a security group"""
+        ec2 = self.auth.get_boto3_resource("ec2", region)
+        vpc = ec2.Vpc(vpcid)
+        for sg in vpc.security_groups.all():
+            if sg.group_name == "default":
+                continue
+            # revoke all ingress rules except for ssh
+            sg.revoke_ingress(IpPermissions=[r for r in sg.ip_permissions if not (r.get("FromPort") == 22 and r.get("ToPort") == 22)])
+
+    def list_instance_profiles(self, prefix: Optional[str] = None):
+        """List instance profile names in a region"""
+        paginator = self.auth.get_boto3_client("iam").get_paginator("list_instance_profiles")
+        matched_names = []
+        for page in paginator.paginate():
+            for profile in page["InstanceProfiles"]:
+                if prefix is None or profile["InstanceProfileName"].startswith(prefix):
+                    matched_names.append(profile["InstanceProfileName"])
+        return matched_names
+
+    def delete_instance_profile(self, profile_name: str):
+        # remove all roles from the instance profile
+        profile = self.auth.get_boto3_resource("iam").InstanceProfile(profile_name)
+        for role in profile.roles:
+            profile.remove_role(RoleName=role.name)
+        # delete the instance profile
+        profile.delete()
+
     def create_iam(self, iam_name: str = "skylark_gateway", attach_policy_arn: Optional[str] = None):
         """Create IAM role if it doesn't exist and grant managed role if given."""
         iam = self.auth.get_boto3_client("iam")

skylark/compute/aws/aws_server.py

@@ -7,7 +7,6 @@
 from skylark.compute.aws.aws_auth import AWSAuthentication
 from skylark.compute.server import Server, ServerState
 from skylark.utils.cache import ignore_lru_cache
-from skylark.utils import logger
 
 
 class AWSServer(Server):
@@ -60,7 +59,22 @@ def __repr__(self):
         return f"AWSServer(region_tag={self.region_tag}, instance_id={self.instance_id})"
 
     def terminate_instance_impl(self):
-        self.auth.get_boto3_resource("ec2", self.aws_region).instances.filter(InstanceIds=[self.instance_id]).terminate()
+        iam = self.auth.get_boto3_resource("iam")
+
+        # get instance profile name that is associated with this instance
+        profile = self.get_boto3_instance_resource().iam_instance_profile
+        if profile:
+            profile = iam.InstanceProfile(profile["Arn"].split("/")[-1])
+
+            # remove all roles from instance profile
+            for role in profile.roles:
+                profile.remove_role(RoleName=role.name)
+
+            # delete instance profile
+            profile.delete()
+
+        # delete instance
+        self.get_boto3_instance_resource().terminate()
 
     def get_ssh_client_impl(self):
         client = paramiko.SSHClient()