Updates for overall cleanup

aws/reporting/cloudformation.py

@@ -10,10 +10,9 @@
 
 logger = logging.getLogger(__name__)
 
-def get_deleteable_cf_templates(client):
-    deleteable_stacks = []
-    response = client.describe_stacks()
-    for stack in response.get('Stacks', []):
+def default_filter(client, stacks):
+    filtered_stacks = []
+    for stack in stacks:
         stackName = stack.get("StackName", "")
         if stackName != "":
             is_eks_managed = False
@@ -23,23 +22,31 @@ def get_deleteable_cf_templates(client):
                     logger.info("{} Found EKS managed stack {}".format(client.meta.region_name, stackName))
             if not is_eks_managed:
                 if stack.get('StackStatus', '') in DELETEABLE_STATUS:
-                    deleteable_stacks.append(stack)
+                    filtered_stacks.append(stack)
                     logger.info("{} Found stack with deleteable status {}".format(client.meta.region_name, stackName))
                 elif not does_cf_template_have_ec2_instances(client, stackName):
-                    deleteable_stacks.append(stack)
+                    filtered_stacks.append(stack)
                     logger.info("{} Found stack without instances {}".format(client.meta.region_name, stackName))
-    return deleteable_stacks
+    return filtered_stacks
+
+def no_filter(client, stacks):
+    return stacks
+
+def get_deleteable_cf_templates(client, filter_func=default_filter):
+    deleteable_stacks = []
+    response = client.describe_stacks()
+    return filter_func(client, response.get('Stacks', []))
 
 def does_cf_template_have_ec2_instances(client, stack_name: str):
     for resource in client.describe_stack_resources(StackName=stack_name)['StackResources']:
         if resource.get('ResourceType', '') == AWS_RESOURCE_TYPE_EC2_INSTANCE:
             return True
     return False
 
-def delete_stacks(dry_run = False):
+def delete_stacks(dry_run = False, filter_func=default_filter):
     for region in get_all_regions():
         client = boto3.client('cloudformation', region_name=region)
-        stacks = get_deleteable_cf_templates(client)
+        stacks = get_deleteable_cf_templates(client, filter_func=filter_func)
         for stack in stacks:
             stackName = stack.get("StackName", "")
             if stackName != "":
@@ -50,3 +57,14 @@ def delete_stacks(dry_run = False):
                     logger.info("{} Deleted stack {}".format(region, stackName))
                 except:
                     logger.info("{} Failed deleting stack {}".format(region, stackName))
+
+if __name__ == "__main__":
+    import sys
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.DEBUG)
+    console_handler = logging.StreamHandler(sys.stderr)
+    console_handler.setLevel(logging.DEBUG)
+    formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")
+    console_handler.setFormatter(formatter)
+    logger.addHandler(console_handler)
+    delete_stacks(filter_func=no_filter)

aws/reporting/ec2.py

@@ -49,6 +49,17 @@ def reformat_instance_data(raw_instances):
             inst['Cost Per Day'] = "${}".format(calculate_bill_for_instance(instance_type, region, launch_time)[1])
         except:
             inst['Cost Per Day'] = "$0"
+    if not formatted_instances:
+        dummy_old_instance = {}
+        for key in EC2_KEYS:
+            split_keys = key.split('.')
+            if len(split_keys) == 1:
+                dummy_old_instance[key] = ''
+            else:
+                dummy_old_instance[split_keys[-1]] = ''
+        dummy_old_instance['TotalBill'] = ''
+        dummy_old_instance['Cost Per Day'] = ''
+        return [dummy_old_instance]
     return formatted_instances
 
 def get_all_eips():

aws/reporting/elbs.py

@@ -62,3 +62,16 @@ def delete_classic_elb(elb_name, region):
         logger.info("{} Error deleting elb {}".format(region, elb_name))
         logger.error(str(e))
     return response
+
+if __name__ == "__main__":
+    import sys
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.DEBUG)
+    console_handler = logging.StreamHandler(sys.stderr)
+    console_handler.setLevel(logging.DEBUG)
+    formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")
+    console_handler.setFormatter(formatter)
+    logger.addHandler(console_handler)
+    for elb in reformat_elbs_data(get_all_elbs()):
+        if elb.get("Region", "") != "" and elb.get("LoadBalancerName", "") != "":
+            response = delete_classic_elb(elb['LoadBalancerName'], elb['Region'])

aws/reporting/iam.py

@@ -2,7 +2,9 @@
 import re
 
 import boto3
+import logging
 
+logger = logging.getLogger(__name__)
 
 def get_all_users():
     users = []
@@ -27,7 +29,7 @@ def get_old_users(users, createdThreshold=60, lastUsedThreshold=120):
     old_users = []
     i = 1
     for user in users:
-        print("{} Analyzing user {}".format(i, user['UserName']))
+        logger.info("{} Analyzing user {}".format(i, user['UserName']))
         userResource = boto3.resource('iam').User(user['UserName'])
         allUsageDates = []
         if isinstance(userResource.password_last_used, datetime.date):
@@ -44,44 +46,62 @@ def get_old_users(users, createdThreshold=60, lastUsedThreshold=120):
             usedThresholdAgo = False
         if createdThresholdAgo and usedThresholdAgo and not re.match(r'[^@]+@[^@]+\.[^@]+', user['UserName']):
             old_users.append(user)
-            print("User {} is old".format(user["UserName"]))
+            logger.info("User {} is old".format(user["UserName"]))
         i += 1 
     return old_users
 
+def get_users_for_a_cluster(users):
+    filtered_users = []
+    for user in users:
+        print(user)
+        logger.info("Analyzing user {}".format(user['UserName']))
+        if user.get("UserName", "").startswith("cluster-"):
+            filtered_users.append(user)
+    return filtered_users
+
 def delete_user(user):
-    print("Attempting to delete user {}".format(user['UserName']))
+    logger.info("Attempting to delete user {}".format(user['UserName']))
     iamRes = boto3.resource('iam')
     userRes = iamRes.User(user['UserName'])
     try:
         login_profile = userRes.LoginProfile()
         login_profile.delete()
     except Exception as e:
-        print("Failed deleting login profile {}".format(str(e)))
+        logger.info("Failed deleting login profile {}".format(str(e)))
     for key in userRes.access_keys.all():
         try:
             key.delete()
         except:
-            print("Failed deleting key")
+            logger.info("Failed deleting key")
     for policy in userRes.policies.all():
         try:
             policy.delete()
         except:
-            print("Failed deleting policy")
+            logger.info("Failed deleting policy")
     for policy in userRes.attached_policies.all():
         try:
             policy.delete()
         except:
-            print("Failed deleting policy")
+            logger.info("Failed deleting policy")
     try:
         userRes.delete()
-        print("Deleted user")
+        logger.info("Deleted user")
     except:
-        print("Failed deleting user")
+        logger.info("Failed deleting user")
 
-users = get_all_users()
-old_users = get_old_users(users)
-for user in old_users:
-    try:
-        delete_user(user)
-    except:
-        print("Failed deleting user {}".format(user['UserName']))
+if __name__ == "__main__":
+    import sys
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.DEBUG)
+    console_handler = logging.StreamHandler(sys.stderr)
+    console_handler.setLevel(logging.DEBUG)
+    formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")
+    console_handler.setFormatter(formatter)
+    logger.addHandler(console_handler)
+    users = get_all_users()
+    users_to_delete = get_users_for_a_cluster(users)
+    for user in users_to_delete:
+        try:
+            delete_user(user)
+        except:
+            logger.info("Failed deleting user {}".format(user['UserName']))

aws/reporting/main.py

@@ -11,7 +11,7 @@
     delete_volume, delete_eip, terminate_instance, EC2_KEYS
 from elbs import get_all_elbs, reformat_elbs_data, delete_classic_elb
 from emailer import Emailer
-from s3 import get_all_buckets, reformat_buckets_data
+from s3 import get_all_buckets, reformat_buckets_data, delete_bucket
 from sheet import GoogleSheetEditor
 from vpc import get_all_vpcs, delete_orphan_vpcs
 
@@ -212,7 +212,8 @@ def start(argument):
         instances = reformat_instance_data(instances)
         instances_daily_bill = 0.0
         for instance in instances:
-            instances_daily_bill += float(re.sub(r'\$', '', instance['Cost Per Day']))
+            if instance['Cost Per Day']:
+                instances_daily_bill += float(re.sub(r'\$', '', instance['Cost Per Day']))
         summaryRow['EC2 Daily Cost'] = "${}".format(str(instances_daily_bill))
         print(allInstancesSheet.save_data_to_sheet(instances))
         # update old instance sheet
@@ -252,6 +253,14 @@ def start(argument):
         summaryRow['EC2 Cleanup'] = 'Deleted {} instances'.format(numberOfInstancesDeleted)
         delete_stacks()
 
+    elif argument == 'purge_s3':
+        buckets = oldS3Sheet.read_spreadsheet()
+        for bucket in buckets:
+            name = bucket.get('Name', '')
+            saved = bucket.get('Saved', '')
+            if name != '' and saved not in ["Saved", "Save", "save"]:
+                delete_bucket(bucket=bucket.get('Name'))
+
     elif argument == 'generate_ec2_deletion_summary':
         summaryEmail = get_old_instances_email_summary(oldInstancesSheet, allInstancesSheet, summarySheet)
         print("SummaryEmail", summaryEmail)

aws/reporting/route53.py

@@ -78,6 +78,11 @@ def delete_hosted_zones(dry_run = False):
 
 if __name__ == "__main__":
     import sys
+    logger = logging.getLogger(__name__)
     logger.setLevel(logging.DEBUG)
-    logger.addHandler(logging.StreamHandler(sys.stdout))
+    console_handler = logging.StreamHandler(sys.stderr)
+    console_handler.setLevel(logging.DEBUG)
+    formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")
+    console_handler.setFormatter(formatter)
+    logger.addHandler(console_handler)
     delete_hosted_zones()

aws/reporting/s3.py

@@ -1,6 +1,9 @@
 import boto3
+import logging
 from common import reformat_data
 
+logger = logging.getLogger(__name__)
+
 def get_all_buckets():
     client = boto3.client('s3')
     return client.list_buckets()['Buckets']
@@ -11,3 +14,17 @@ def reformat_buckets_data(buckets):
         'CreationDate',
     ]
     return reformat_data(buckets, keys)
+
+def delete_bucket(bucket):
+    try:
+        logger.info(f"Deleting bucket {bucket}")
+        s3 = boto3.resource('s3')
+        bucket = s3.Bucket(bucket)
+        logger.info(f"Deleting all objects from bucket")
+        bucket.objects.all().delete()
+        logger.info("Deleting object versions (if any)...")
+        bucket.object_versions.all().delete()
+        logger.info(f"Deleting bucket")
+        bucket.delete()
+    except:
+        logger.error(f"Failed to delete bucket {bucket}")