Total refactor to add more type safety and testability

README.md

@@ -4,7 +4,7 @@ Tooling to deploy Safecast to AWS Elastic Beanstalk and work with AWS. Deploymen
 
 ## Installation
 
-It's best if this is run in its own virtualenv. It seems that `wheel` must be installed prior to other requirements.
+It's best if this is run in its own virtualenv; using [direnv](https://direnv.net/) is a common way to accomplish this. It seems that `wheel` must be installed prior to other requirements.
 
 ```
 pip install wheel
@@ -24,8 +24,18 @@ pip install --requirement requirements.txt
 
 Help on specific commands can be found by using `--help` with that command: `./deploy.py ssh --help`
 
+### Shell completion
+
+Shell completion is supported via [argcomplete](https://github.com/kislyuk/argcomplete). I have not been able to find a way to make global argcomplete support work within the direnv; instead, each time I have to run `eval "$(register-python-argcomplete deploy.py)"`.
+
 ## Known issues
 
 The scripts currently assume that a previous environment already exists, in all cases.
 
 When `new_env` is called, safecast_deploy creates a new environment from the existing application configuration templates stored in Elastic Beanstalk and named `dev`, `dev-wrk`, `prd`, `prd-wrk`, etc. The `new_env` command will set a new ARN for the environment; however, that new ARN is not saved back to the application template. This is not generally a problem, especially if we continue to use this tool for all new deployments. However, it does mean that the saved template does not accurately reflect what is being run any longer. We could create a task in the future to synchronize the saved templates to what is actually running.
+
+## Development
+
+Unit tests can be run with `python -m unittest`.
+
+Please run `pycodestyle` before committing.

deploy.py

@@ -1,15 +1,14 @@
 #!/usr/bin/env python3
 
-# Currently for deploying when you also want to create a new
-# environment, not for redeploying to an existing environment.
-
 import sys
-if sys.version_info.major < 3 or sys.version_info.minor < 7:
-    print("Error: This script requires at least Python 3.7.", file=sys.stderr)
+if sys.version_info.major < 3 or sys.version_info.minor < 8:
+    print("Error: This script requires at least Python 3.8.", file=sys.stderr)
     exit(1)
 
+import argcomplete
 import argparse
 import boto3
+import json
 import pprint
 import re
 import safecast_deploy
@@ -21,6 +20,10 @@
 import safecast_deploy.state
 import time
 
+from safecast_deploy.aws_state import AwsTierType, EnvType
+from safecast_deploy.extended_json_encoder import ExtendedJSONEncoder
+from safecast_deploy.result_logger import ResultLogger
+
 
 def parse_args():
     p = argparse.ArgumentParser()
@@ -30,9 +33,10 @@ def parse_args():
     list_arns_p.set_defaults(func=run_list_arns)
 
     apps = ['api', 'ingest', 'reporting']
-    environments = ['dev', 'prd']
+    environments = [type.value for type in list(EnvType)]
+    tiers = [type.value for type in list(AwsTierType)]
 
-    desc_metadata_p = ps.add_parser('desc_metadata', help="")
+    desc_metadata_p = ps.add_parser('desc_metadata', help="Describe the metadata available for this application.")
     desc_metadata_p.add_argument('app',
                                  choices=apps,
                                  help="The application to describe.",)
@@ -50,7 +54,7 @@ def parse_args():
                            choices=apps,
                            help="The target application to deploy to.",)
     new_env_p.add_argument('env',
-                           choices=['dev', 'prd'],
+                           choices=environments,
                            help="The target environment to deploy to.",)
     new_env_p.add_argument('version', help="The new version to deploy.")
     new_env_p.add_argument('arn', help="The ARN the new deployment should use.")
@@ -78,20 +82,20 @@ def parse_args():
     save_configs_p.add_argument('-e', '--env',
                                 choices=environments,
                                 help="Limit the overwrite to a specific environment.")
-    save_configs_p.add_argument('-r', '--role',
-                                choices=['web', 'wrk'],
-                                help="Limit the overwrite to a specific role.")
+    save_configs_p.add_argument('-t', '--tier',
+                                choices=tiers,
+                                help="Limit the overwrite to a specific tier.")
     save_configs_p.set_defaults(func=safecast_deploy.config_saver.run_cli)
 
     ssh_p = ps.add_parser('ssh', help='SSH to the selected environment.')
     ssh_p.add_argument('app',
                        choices=apps,
                        help="The target application.",)
     ssh_p.add_argument('env',
-                       choices=['dev', 'prd', ],
+                       choices=environments,
                        help="The target environment.",)
-    ssh_p.add_argument('role',
-                       choices=['web', 'wrk', ],
+    ssh_p.add_argument('tier',
+                       choices=tiers,
                        help="The type of server.",)
     ssh_p.add_argument('-s', '--select', action='store_true',
                        help="Choose a specific server. Otherwise, will connect to the first server found.",)
@@ -110,14 +114,14 @@ def parse_args():
                             help="The target application.",)
     versions_p.set_defaults(func=run_versions)
 
+    argcomplete.autocomplete(p)
     args = p.parse_args()
     if 'func' in args:
         args.func(args)
     else:
         p.error("too few arguments")
 
 
-
 def run_list_arns(args):
     c = boto3.client('elasticbeanstalk')
     platforms = c.list_platform_versions(
@@ -139,8 +143,9 @@ def run_list_arns(args):
 
 
 def run_desc_metadata(args):
-    state = safecast_deploy.state.State(args.app)
-    pprint.PrettyPrinter(stream=sys.stderr).pprint(state.env_metadata)
+    state = safecast_deploy.state.State(args.app, boto3.client('elasticbeanstalk'))
+    json.dump(state.old_aws_state.to_dict(), sys.stdout, sort_keys=True, indent=2)
+    print()
 
 
 def run_desc_template(args):
@@ -149,43 +154,50 @@ def run_desc_template(args):
         ApplicationName=args.app,
         TemplateName=args.template,
     )
-    pprint.PrettyPrinter(stream=sys.stderr).pprint(template)
+    json.dump(template, sys.stdout, sort_keys=True, indent=2, cls=ExtendedJSONEncoder)
+    print()
 
 
 def run_new_env(args):
-    state = safecast_deploy.state.State(
-        args.app,
-        args.env,
-        new_version=args.version,
-        new_arn=args.arn
-    )
-    safecast_deploy.new_env.NewEnv(state, not args.no_update_templates).run()
+    eb_client = boto3.client('elasticbeanstalk')
+    result_logger = ResultLogger()
+    state = safecast_deploy.state.State(args.app, eb_client)
+    config_saver = safecast_deploy.config_saver.ConfigSaver(eb_client, result_logger)
+    safecast_deploy.new_env.NewEnv(
+        EnvType(args.env),
+        state.old_aws_state,
+        state.new_aws_state(new_version=args.version),
+        boto3.client('elasticbeanstalk'),
+        result_logger,
+        config_saver,
+        (not args.no_update_templates),
+    ).run()
 
 
 def run_same_env(args):
-    state = safecast_deploy.state.State(
-        args.app,
-        args.env,
-        new_version=args.version,
-    )
-    safecast_deploy.same_env.SameEnv(state).run()
+    state = safecast_deploy.state.State(args.app, boto3.client('elasticbeanstalk'))
+    safecast_deploy.same_env.SameEnv(
+        EnvType(args.env),
+        state.old_aws_state,
+        state.new_aws_state(new_version=args.version),
+        boto3.client('elasticbeanstalk'),
+        ResultLogger(),
+    ).run()
 
 
 def run_ssh(args):
-    state = safecast_deploy.state.State(args.app, args.env)
-    safecast_deploy.ssh.Ssh(state, args).run()
+    aws_state = safecast_deploy.state.State(args.app, boto3.client('elasticbeanstalk')).old_aws_state
+    safecast_deploy.ssh.ssh(aws_state, EnvType(args.env), AwsTierType(args.tier), args.select)
 
 
 def run_versions(args):
-    state = safecast_deploy.state.State(args.app)
+    state = safecast_deploy.state.State(args.app, boto3.client('elasticbeanstalk'))
     print(*state.available_versions, sep='\n')
 
 
 def main():
     parse_args()
     # TODO method to switch to maintenance page
-    #
-    # TODO method to clean out old versions
 
 
 if __name__ == '__main__':

requirements.txt

@@ -1,3 +1,5 @@
+argcomplete>=1.12.1,<2.0
 boto3>=1.14.4,<2.0
+dataclasses-json>=0.5.2,<2.0
 GitPython>=3.1.3,<4.0
 pycodestyle

safecast_deploy/aws_state.py

@@ -0,0 +1,51 @@
+from dataclasses import dataclass
+from dataclasses_json import dataclass_json
+from enum import Enum, unique
+
+
+@unique
+class EnvType(str, Enum):
+    """Encodes Safecast's understanding of environment types.
+    """
+    DEV = 'dev'
+    PROD = 'prd'
+
+    def __repr__(self):
+        return '<%s.%s>' % (self.__class__.__name__, self.name)
+
+
+@unique
+class AwsTierType(str, Enum):
+    WEB = 'web'
+    WORKER = 'wrk'
+
+    def __repr__(self):
+        return '<%s.%s>' % (self.__class__.__name__, self.name)
+
+
+@dataclass_json
+@dataclass(frozen=True)
+class ParsedVersion:
+    app: str
+    circleci_build_num: int
+    clean_branch_name: str
+    git_commit: str
+    version_string: str
+
+
+@dataclass_json
+@dataclass(frozen=True)
+class AwsState:
+    aws_app_name: str
+    envs: dict  # dictionary mapping EnvTypes to a nested dictionary of AwsTierTypes to AwsTier objects
+
+
+@dataclass_json
+@dataclass(frozen=True)
+class AwsTier:
+    tier_type: AwsTierType
+    platform_arn: str
+    parsed_version: ParsedVersion
+    name: str
+    environment_id: str  # Note this is calculated by AWS and will not be available prior to new environment creation
+    num: int

safecast_deploy/config_saver.py

@@ -1,65 +1,66 @@
+import boto3
 import datetime
 import pprint
 import sys
 
-from safecast_deploy import git_logger, state, verbose_sleep
+from safecast_deploy import verbose_sleep
+from safecast_deploy.aws_state import AwsTierType, EnvType
+from safecast_deploy.result_logger import ResultLogger
+from safecast_deploy.state import State
 
 
 def run_cli(args):
-    ConfigSaver(app=args.app, env=args.env, role=args.role).run()
+    app = args.app
+    env = EnvType(args.env) if args.env else None
+    tier = AwsTierType(args.tier) if args.tier else None
+    ConfigSaver(boto3.client('elasticbeanstalk'), ResultLogger()).run(app=app, env=env, tier=tier)
 
 
 class ConfigSaver:
-    def __init__(self, app=None, env=None, role=None):
-        self.app = app
-        self.env = env
-        self.role = role
-        self.states = {}
+    # This class is not thread-safe.
+    def __init__(self, eb_client, result_logger):
+        self._c = eb_client
+        self._result_logger = result_logger
+        self._completed_list = []
+
+    def run(self, app=None, env=None, tier=None):
+        all_apps = ['api', 'ingest', 'recording']
+        states = {}
         if app is None:
-            self.states['api'] = state.State('api')
-            self.states['ingest'] = state.State('ingest')
-            self.states['reporting'] = state.State('reporting')
-            self._c = self.states['api'].eb_client
+            for default_app in all_apps:
+                states[default_app] = State(default_app, self._c).old_aws_state
         else:
-            self.states[app] = state.State(app)
-            self._c = self.states[app].eb_client
-        self.completed_list = []
+            states[app] = State(app, self._c).old_aws_state
 
-    def run(self):
-        for app in self.states:
-            env_metadata = self.states[app].env_metadata
-            if self.app is None:
-                self.process_app('api')
-                self.process_app('ingest')
-            else:
-                self.process_app(app)
-        git_logger.log_result(self.completed_list)
-        pprint.PrettyPrinter(stream=sys.stderr).pprint(self.completed_list)
+        for state in states:
+            self.process_state(states[state], env, tier)
+        self._result_logger.log_result(self._completed_list)
+        self._completed_list = []
 
-    def process_app(self, app):
-        if self.env is None:
-            self.process_env(app, 'dev')
-            self.process_env(app, 'prd')
+    def process_state(self, state, env, tier):
+        if env is None:
+            for available_env in state.envs:
+                self.process_env(state, available_env, tier)
         else:
-            self.process_env(app, self.env)
+            self.process_env(state, env, tier)
 
-    def process_env(self, app, env):
+    def process_env(self, state, env, tier):
         template_names = {
-            'web': env,
-            'wrk': '{}-wrk'.format(env),
+            AwsTierType.WEB: env.value,
+            AwsTierType.WORKER: f'{env.value}-wrk',
         }
-        if self.role is None:
-            self.process_role(app, env, 'web', template_names)
-            if self.states[app].has_worker:
-                self.process_role(app, env, 'wrk', template_names)
+        if tier is None:
+            for available_tier in state.envs[env]:
+                self.process_tier(state, env, available_tier, template_names)
         else:
-            self.process_role(app, env, self.role, template_names)
+            self.process_tier(state, env, tier, template_names)
 
-    def process_role(self, app, env, role, template_names):
+    def process_tier(self, state, env, tier, template_names):
         start_time = datetime.datetime.now(datetime.timezone.utc)
-        template_name = template_names[role]
-        env_id = self.states[app].env_metadata[template_name]['api_env']['EnvironmentId']
-        env_name = self.states[app].env_metadata[template_name]['name']
+        template_name = template_names[tier]
+        app = state.aws_app_name
+        env_id = state.envs[env][tier].environment_id
+        env_name = state.envs[env][tier].name
         print(f"Starting update of template {template_name} from {env_name}", file=sys.stderr)
         self._c.delete_configuration_template(
             ApplicationName=app,
@@ -73,14 +74,15 @@ def process_role(self, app, env, role, template_names):
         )
         print(f"Completed update of template {template_name} from {env_name}", file=sys.stderr)
         completed_time = datetime.datetime.now(datetime.timezone.utc)
-        self.completed_list.append({
+        self._completed_list.append({
             'app': app,
             'completed_at': completed_time,
             'elapsed_time': (completed_time - start_time).total_seconds(),
             'env': env,
             'event': 'save_configs',
-            'role': role,
+            'tier': tier,
+            'source_env_id': env_id,
             'source_env_name': env_name,
             'started_at': start_time,
-            'template_name': template_name
+            'template_name': template_name,
         })