diff --git a/aws/reporting/cloudformation.py b/aws/reporting/cloudformation.py index 1fa614d..9a7e5a2 100644 --- a/aws/reporting/cloudformation.py +++ b/aws/reporting/cloudformation.py @@ -10,10 +10,9 @@ logger = logging.getLogger(__name__) -def get_deleteable_cf_templates(client): - deleteable_stacks = [] - response = client.describe_stacks() - for stack in response.get('Stacks', []): +def default_filter(client, stacks): + filtered_stacks = [] + for stack in stacks: stackName = stack.get("StackName", "") if stackName != "": is_eks_managed = False @@ -23,12 +22,20 @@ def get_deleteable_cf_templates(client): logger.info("{} Found EKS managed stack {}".format(client.meta.region_name, stackName)) if not is_eks_managed: if stack.get('StackStatus', '') in DELETEABLE_STATUS: - deleteable_stacks.append(stack) + filtered_stacks.append(stack) logger.info("{} Found stack with deleteable status {}".format(client.meta.region_name, stackName)) elif not does_cf_template_have_ec2_instances(client, stackName): - deleteable_stacks.append(stack) + filtered_stacks.append(stack) logger.info("{} Found stack without instances {}".format(client.meta.region_name, stackName)) - return deleteable_stacks + return filtered_stacks + +def no_filter(client, stacks): + return stacks + +def get_deleteable_cf_templates(client, filter_func=default_filter): + deleteable_stacks = [] + response = client.describe_stacks() + return filter_func(client, response.get('Stacks', [])) def does_cf_template_have_ec2_instances(client, stack_name: str): for resource in client.describe_stack_resources(StackName=stack_name)['StackResources']: @@ -36,10 +43,10 @@ def does_cf_template_have_ec2_instances(client, stack_name: str): return True return False -def delete_stacks(dry_run = False): +def delete_stacks(dry_run = False, filter_func=default_filter): for region in get_all_regions(): client = boto3.client('cloudformation', region_name=region) - stacks = get_deleteable_cf_templates(client) + stacks = get_deleteable_cf_templates(client, filter_func=filter_func) for stack in stacks: stackName = stack.get("StackName", "") if stackName != "": @@ -50,3 +57,14 @@ def delete_stacks(dry_run = False): logger.info("{} Deleted stack {}".format(region, stackName)) except: logger.info("{} Failed deleting stack {}".format(region, stackName)) + +if __name__ == "__main__": + import sys + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) + console_handler = logging.StreamHandler(sys.stderr) + console_handler.setLevel(logging.DEBUG) + formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s") + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) + delete_stacks(filter_func=no_filter) \ No newline at end of file diff --git a/aws/reporting/ec2.py b/aws/reporting/ec2.py index d525e87..749cb18 100644 --- a/aws/reporting/ec2.py +++ b/aws/reporting/ec2.py @@ -49,6 +49,17 @@ def reformat_instance_data(raw_instances): inst['Cost Per Day'] = "${}".format(calculate_bill_for_instance(instance_type, region, launch_time)[1]) except: inst['Cost Per Day'] = "$0" + if not formatted_instances: + dummy_old_instance = {} + for key in EC2_KEYS: + split_keys = key.split('.') + if len(split_keys) == 1: + dummy_old_instance[key] = '' + else: + dummy_old_instance[split_keys[-1]] = '' + dummy_old_instance['TotalBill'] = '' + dummy_old_instance['Cost Per Day'] = '' + return [dummy_old_instance] return formatted_instances def get_all_eips(): diff --git a/aws/reporting/elbs.py b/aws/reporting/elbs.py index 6182bc0..2c53574 100644 --- a/aws/reporting/elbs.py +++ b/aws/reporting/elbs.py @@ -62,3 +62,16 @@ def delete_classic_elb(elb_name, region): logger.info("{} Error deleting elb {}".format(region, elb_name)) logger.error(str(e)) return response + +if __name__ == "__main__": + import sys + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) + console_handler = logging.StreamHandler(sys.stderr) + console_handler.setLevel(logging.DEBUG) + formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s") + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) + for elb in reformat_elbs_data(get_all_elbs()): + if elb.get("Region", "") != "" and elb.get("LoadBalancerName", "") != "": + response = delete_classic_elb(elb['LoadBalancerName'], elb['Region']) diff --git a/aws/reporting/iam.py b/aws/reporting/iam.py index 6b4a79a..da0ab45 100644 --- a/aws/reporting/iam.py +++ b/aws/reporting/iam.py @@ -2,7 +2,9 @@ import re import boto3 +import logging +logger = logging.getLogger(__name__) def get_all_users(): users = [] @@ -27,7 +29,7 @@ def get_old_users(users, createdThreshold=60, lastUsedThreshold=120): old_users = [] i = 1 for user in users: - print("{} Analyzing user {}".format(i, user['UserName'])) + logger.info("{} Analyzing user {}".format(i, user['UserName'])) userResource = boto3.resource('iam').User(user['UserName']) allUsageDates = [] if isinstance(userResource.password_last_used, datetime.date): @@ -44,44 +46,62 @@ def get_old_users(users, createdThreshold=60, lastUsedThreshold=120): usedThresholdAgo = False if createdThresholdAgo and usedThresholdAgo and not re.match(r'[^@]+@[^@]+\.[^@]+', user['UserName']): old_users.append(user) - print("User {} is old".format(user["UserName"])) + logger.info("User {} is old".format(user["UserName"])) i += 1 return old_users +def get_users_for_a_cluster(users): + filtered_users = [] + for user in users: + print(user) + logger.info("Analyzing user {}".format(user['UserName'])) + if user.get("UserName", "").startswith("cluster-"): + filtered_users.append(user) + return filtered_users + def delete_user(user): - print("Attempting to delete user {}".format(user['UserName'])) + logger.info("Attempting to delete user {}".format(user['UserName'])) iamRes = boto3.resource('iam') userRes = iamRes.User(user['UserName']) try: login_profile = userRes.LoginProfile() login_profile.delete() except Exception as e: - print("Failed deleting login profile {}".format(str(e))) + logger.info("Failed deleting login profile {}".format(str(e))) for key in userRes.access_keys.all(): try: key.delete() except: - print("Failed deleting key") + logger.info("Failed deleting key") for policy in userRes.policies.all(): try: policy.delete() except: - print("Failed deleting policy") + logger.info("Failed deleting policy") for policy in userRes.attached_policies.all(): try: policy.delete() except: - print("Failed deleting policy") + logger.info("Failed deleting policy") try: userRes.delete() - print("Deleted user") + logger.info("Deleted user") except: - print("Failed deleting user") + logger.info("Failed deleting user") -users = get_all_users() -old_users = get_old_users(users) -for user in old_users: - try: - delete_user(user) - except: - print("Failed deleting user {}".format(user['UserName'])) \ No newline at end of file +if __name__ == "__main__": + import sys + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) + console_handler = logging.StreamHandler(sys.stderr) + console_handler.setLevel(logging.DEBUG) + formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s") + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) + users = get_all_users() + users_to_delete = get_users_for_a_cluster(users) + for user in users_to_delete: + try: + delete_user(user) + except: + logger.info("Failed deleting user {}".format(user['UserName'])) \ No newline at end of file diff --git a/aws/reporting/main.py b/aws/reporting/main.py index 737a65b..0ef1a53 100644 --- a/aws/reporting/main.py +++ b/aws/reporting/main.py @@ -11,7 +11,7 @@ delete_volume, delete_eip, terminate_instance, EC2_KEYS from elbs import get_all_elbs, reformat_elbs_data, delete_classic_elb from emailer import Emailer -from s3 import get_all_buckets, reformat_buckets_data +from s3 import get_all_buckets, reformat_buckets_data, delete_bucket from sheet import GoogleSheetEditor from vpc import get_all_vpcs, delete_orphan_vpcs @@ -212,7 +212,8 @@ def start(argument): instances = reformat_instance_data(instances) instances_daily_bill = 0.0 for instance in instances: - instances_daily_bill += float(re.sub(r'\$', '', instance['Cost Per Day'])) + if instance['Cost Per Day']: + instances_daily_bill += float(re.sub(r'\$', '', instance['Cost Per Day'])) summaryRow['EC2 Daily Cost'] = "${}".format(str(instances_daily_bill)) print(allInstancesSheet.save_data_to_sheet(instances)) # update old instance sheet @@ -252,6 +253,14 @@ def start(argument): summaryRow['EC2 Cleanup'] = 'Deleted {} instances'.format(numberOfInstancesDeleted) delete_stacks() + elif argument == 'purge_s3': + buckets = oldS3Sheet.read_spreadsheet() + for bucket in buckets: + name = bucket.get('Name', '') + saved = bucket.get('Saved', '') + if name != '' and saved not in ["Saved", "Save", "save"]: + delete_bucket(bucket=bucket.get('Name')) + elif argument == 'generate_ec2_deletion_summary': summaryEmail = get_old_instances_email_summary(oldInstancesSheet, allInstancesSheet, summarySheet) print("SummaryEmail", summaryEmail) diff --git a/aws/reporting/route53.py b/aws/reporting/route53.py index 11888a2..14ff5cc 100644 --- a/aws/reporting/route53.py +++ b/aws/reporting/route53.py @@ -78,6 +78,11 @@ def delete_hosted_zones(dry_run = False): if __name__ == "__main__": import sys + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) - logger.addHandler(logging.StreamHandler(sys.stdout)) + console_handler = logging.StreamHandler(sys.stderr) + console_handler.setLevel(logging.DEBUG) + formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s") + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) delete_hosted_zones() \ No newline at end of file diff --git a/aws/reporting/s3.py b/aws/reporting/s3.py index 28cfa4a..0051b3a 100644 --- a/aws/reporting/s3.py +++ b/aws/reporting/s3.py @@ -1,6 +1,9 @@ import boto3 +import logging from common import reformat_data +logger = logging.getLogger(__name__) + def get_all_buckets(): client = boto3.client('s3') return client.list_buckets()['Buckets'] @@ -11,3 +14,17 @@ def reformat_buckets_data(buckets): 'CreationDate', ] return reformat_data(buckets, keys) + +def delete_bucket(bucket): + try: + logger.info(f"Deleting bucket {bucket}") + s3 = boto3.resource('s3') + bucket = s3.Bucket(bucket) + logger.info(f"Deleting all objects from bucket") + bucket.objects.all().delete() + logger.info("Deleting object versions (if any)...") + bucket.object_versions.all().delete() + logger.info(f"Deleting bucket") + bucket.delete() + except: + logger.error(f"Failed to delete bucket {bucket}") \ No newline at end of file