Merge pull request #102 from CERTCC/development

Development merge for LocalStack support
CERTCC · Mar 28, 2023 · e6d1088 · e6d1088
2 parents a3c3043 + 46390d4
commit e6d1088
Show file tree

Hide file tree

Showing 25 changed files with 791 additions and 85 deletions.
diff --git a/.env b/.env
@@ -0,0 +1,15 @@
+POSTGRESQL_PASS=PASSWORD
+POSTGRES_HOST=bv-postgres
+LOCALSTACK_LOG_LEVEL=info
+LOCALSTACK_API_KEY=<localstackapikey>
+LOCALSTACK_SERVICES=sns,cognito-idp,lambda,s3,iam,cognito-identity
+HOSTNAME=localstack
+HOSTNAME_EXTERNAL=localstack
+BUCKET_NAME=testbucket
+REGION=us-east-1
+ADMIN_EMAIL=[email protected]
+ADMIN_USERNAME=admin
+ADMIN_PASSWORD=Admin123!
+DOMAIN=localstack
+CLIENT_NAME=testclient
+POOL_NAME=testpool
diff --git a/README.md b/README.md
@@ -74,6 +74,7 @@ reduce the risk of exposure.
 [<img src="https://github.com/CERTCC/VINCE/raw/main/Vince_Infrastructure.png" width="100%"></A>](https://github.com/CERTCC/VINCE/raw/main/Vince_Infrastructure.png)
 
 
+
 ### Local Install
 
 1. Clone the repo

diff --git a/bigvince/settings_.py b/bigvince/settings_.py
@@ -67,6 +67,8 @@
 VINCE_DEV_SYSTEM = os.environ.get('VINCE_DEV_SYSTEM', "")
 if VINCE_DEV_SYSTEM == '1':
     VINCE_DEV_SYSTEM = "title-dev"
+
+LOCALSTACK=os.environ.get('LOCALSTACK')
 
 GOOGLE_SITE_KEY = os.environ.get('GOOGLE_SITE_KEY')
 
@@ -156,13 +158,20 @@
     AWS_REGION = os.environ.get('AWS_REGION')
     AWS_DEPLOYED = True
     AWS_DEFAULT_ACL = None
-    LOGGER_HANDLER = 'watchtower'
+    if LOCALSTACK:
+        AWS_S3_ENDPOINT_URL = os.environ.get('AWS_S3_ENDPOINT_URL')
+        LOGGER_HANDLER = 'console'
+    else:
+        LOGGER_HANDLER = 'watchtower'
     AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME')
     AWS_S3_REGION_NAME = os.environ.get('AWS_REGION')
     # Tell django-storages the domain to use to refer to static files.
     AWS_S3_CUSTOM_DOMAIN = os.environ.get('AWS_S3_CUSTOM_DOMAIN')
     AWS_LOCATION = os.environ.get('AWS_LOCATION')
-    STATIC_URL = f'https://{AWS_S3_CUSTOM_DOMAIN}/{AWS_LOCATION}/'
+    if os.environ.get('BUCKET_URL'):
+        STATIC_URL = os.environ.get('BUCKET_URL') + '/'
+    else:
+        STATIC_URL = f'https://{AWS_S3_CUSTOM_DOMAIN}/{AWS_LOCATION}/'
     # Tell the staticfiles app to use S3Boto3 storage when writing the collected static files (when
     # you run `collectstatic`).
     STATICFILES_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
@@ -326,6 +335,9 @@
                 'django.contrib.auth.context_processors.auth',
                 'django.contrib.messages.context_processors.messages',
             ],
+            'libraries': {
+                'staticfiles': 'django.templatetags.static',
+            }
         },
     },
 ]
@@ -472,8 +484,13 @@ def get_secret(secret_arn):
 COGNITO_USER_POOL_ID = os.environ.get('AWS_COGNITO_USER_POOL_ID')
 COGNITO_APP_ID = os.environ.get('AWS_COGNITO_APP_ID')
 COGNITO_REGION = os.environ.get('AWS_COGNITO_REGION')
+BASE_URL=os.environ.get('BASE_URL')
 if COGNITO_REGION:
-    keys_url = 'https://cognito-idp.{}.amazonaws.com/{}/.well-known/jwks.json'.format(COGNITO_REGION, COGNITO_USER_POOL_ID)
+    if LOCALSTACK:
+        keys_url = f"http://cognito-idp.{COGNITO_REGION}.{BASE_URL}/{COGNITO_USER_POOL_ID}/.well-known/jwks.json"
+    else:
+        keys_url = 'https://cognito-idp.{}.amazonaws.com/{}/.well-known/jwks.json'.format(COGNITO_REGION, COGNITO_USER_POOL_ID)
+
     response = urllib.request.urlopen(keys_url)
     COGNITO_KEYS = json.loads(response.read())['keys']
 
@@ -626,7 +643,7 @@ def get_secret(secret_arn):
     },
 }
 
-if AWS_DEPLOYED:
+if AWS_DEPLOYED and not LOCALSTACK:
     LOG_GROUP_NAME = os.environ.get('VINCE_LOG_GROUP_NAME', 'VINCE')
     logging_dict['handlers']['watchtower'] = {
         'level': 'DEBUG',
@@ -638,7 +655,7 @@ def get_secret(secret_arn):
 
 IS_WORKER = os.environ.get('IS_ELASTICBEANSTALK_WORKER', False)
 IS_VINCEWORKER = os.environ.get('IS_VINCEWORKER', False)
-if IS_VINCEWORKER and AWS_DEPLOYED:
+if IS_VINCEWORKER and AWS_DEPLOYED and not LOCALSTACK:
     IS_VINCEWORKER = True
     INSTALLED_APPS.append('vinceworker')
     logging_dict['loggers']['vinceworker'] = {
@@ -648,7 +665,7 @@ def get_secret(secret_arn):
     logging_dict['handlers']['watchtower']['stream_name'] = 'vinceworker'
 
 IS_KBWORKER = os.environ.get('IS_KBWORKER', False)
-if IS_KBWORKER and AWS_DEPLOYED:
+if IS_KBWORKER and AWS_DEPLOYED and not LOCALSTACK:
     IS_KBWORKER = True
     INSTALLED_APPS.append('kbworker')
     logging_dict['loggers']['kbworker'] = {
@@ -659,7 +676,7 @@ def get_secret(secret_arn):
     # only need to define this for the worker
 
 IS_VCWORKER = os.environ.get('IS_VCWORKER', False)
-if IS_VCWORKER and AWS_DEPLOYED:
+if IS_VCWORKER and AWS_DEPLOYED and not LOCALSTACK:
     IS_VCWORKER = True
     INSTALLED_APPS.append('vincecommworker')
     logging_dict['loggers']['vincecommworker'] = {
@@ -724,7 +741,7 @@ def get_secret(secret_arn):
 CSRF_COOKIE_HTTPONLY=False
 
 # Set CSRF trusted origins to our generated list of trusted domains
-CSRF_TRUSTED_ORIGINS = trusted_domains
+CSRF_TRUSTED_ORIGINS = [f'http://{t_domain}' for t_domain in trusted_domains] + [f'https://{t_domain}' for t_domain in trusted_domains]
 
 CSRF_FAILURE_VIEW = "vinny.views.csrf_failure_view"
 

diff --git a/bigvince/storage_backends.py b/bigvince/storage_backends.py
@@ -34,55 +34,31 @@ class PrivateMediaStorage(S3Boto3Storage):
     location = settings.AWS_PRIVATE_MEDIA_LOCATION
 #    location = 'static'
     file_overwrite = False
-    default_acl = 'private'
     region_name=settings.AWS_REGION
     #region_name = "us-east-1"
     custom_domain=False
-    acl='private'
-    bucket_name = getattr(settings, 'PRIVATE_BUCKET_NAME')
-
-    def __init__(self, *args, **kwargs):
-        #kwargs['bucket'] = getattr(settings, 'PRIVATE_BUCKET_NAME')
-        #kwargs['default_acl']='private'
-        #kwargs['acl'] = 'private'
-        #if hasattr(settings, 'ATTACHMENT_URL'):
-        #    kwargs['custom_domain'] = settings.ATTACHMENT_URL
-        #else:
-        #    kwargs['custom_domain'] = False
-        super(PrivateMediaStorage, self).__init__(*args, **kwargs)
-
+    if not settings.LOCALSTACK:
+        bucket_name = getattr(settings, 'PRIVATE_BUCKET_NAME')
+        acl='private'
+        default_acl = 'private'
 
 class SharedMediaStorage(S3Boto3Storage):
     location = settings.AWS_PRIVATE_MEDIA_LOCATION
     file_overwrite = False
-    default_acl = 'private'
     region_name = settings.AWS_REGION
     #region_name = "us-east-1"
-    acl = 'private'
     custom_domain = False
-    bucket_name = getattr(settings, 'VINCE_SHARED_BUCKET')
+    if not settings.LOCALSTACK:
+        bucket_name = getattr(settings, 'VINCE_SHARED_BUCKET')
+        acl = 'private'
+        default_acl = 'private'
 
-    def __init__(self, *args, **kwargs):
-        #kwargs['bucket'] = getattr(settings, 'VINCE_SHARED_BUCKET')
-        #kwargs['default_acl']='private'
-        #kwargs['acl'] = 'private'
-        #if hasattr(settings, 'VC_ATTACHMENT_URL'):
-        #    kwargs['custom_domain'] = settings.VC_ATTACHMENT_URL
-        #else:
-        #    kwargs['custom_domain'] = False
-        super(SharedMediaStorage, self).__init__(*args, **kwargs)
-
 class VRFReportsStorage(S3Boto3Storage):
     location = settings.VRF_PRIVATE_MEDIA_LOCATION
     file_overwrite = False
-    default_acl = 'private'
     custom_domain=False
     region_name = settings.AWS_REGION
-    acl = 'private'
-    bucket_name = getattr(settings, 'S3_INCOMING_REPORTS')
-
-    def __init__(self, *args, **kwargs):
-        #kwargs['bucket'] = getattr(settings, 'S3_INCOMING_REPORTS')
-        #kwargs['default_acl']='private'
-        #kwargs['acl'] = 'private'
-        super(VRFReportsStorage, self).__init__(*args, **kwargs)
+    if not settings.LOCALSTACK:
+        bucket_name = getattr(settings, 'S3_INCOMING_REPORTS')
+        acl = 'private'
+        default_acl = 'private'
diff --git a/bigvince/utils.py b/bigvince/utils.py
@@ -0,0 +1,14 @@
+from django.conf import settings
+
+def get_cognito_pool_url():
+    return f"{get_cognito_url()}/{settings.COGNITO_USER_POOL_ID}"
+
+
+# adjusted as localstack is currently setting the hostname for cognito as just "localhost"
+def get_cognito_url():
+    if settings.LOCALSTACK:
+        base_url = settings.BASE_URL
+        return f"http://{base_url}"
+    else:
+        base_url = "amazonaws.com"
+        return f"https://cognito-idp.{settings.COGNITO_REGION}.{base_url}"
diff --git a/cogauth/backend.py b/cogauth/backend.py
@@ -50,6 +50,7 @@
 from rest_framework.authentication import BaseAuthentication, TokenAuthentication, get_authorization_header
 from django.utils.encoding import smart_text
 from django.utils.translation import ugettext as _
+from bigvince.utils import get_cognito_url, get_cognito_pool_url
 import traceback
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -178,7 +179,8 @@ def authenticate(self, request, username=None, password=None):
             # for tokens or we already have tokens - in which case we just need to get
             # the user and return
 
-            client= boto3.client('cognito-idp', region_name=settings.COGNITO_REGION) 
+            client= boto3.client('cognito-idp',
+             endpoint_url=get_cognito_url(), region_name=settings.COGNITO_REGION) 
             user = client.get_user(AccessToken=request.session['ACCESS_TOKEN'])
             # the username returned is the unique id, which doesn't help us since we use
             # emails for username - so get email and return CognitoUser
@@ -217,7 +219,8 @@ def authenticate(self, request, username=None, password=None):
 
                 u.check_token()
 
-                client= boto3.client('cognito-idp', region_name=settings.COGNITO_REGION)
+                client= boto3.client('cognito-idp',
+             endpoint_url=get_cognito_url(), region_name=settings.COGNITO_REGION)
                 user = client.get_user(AccessToken=access_token)
                 username = user['Username']
                 cognito_user = CognitoUser(

diff --git a/cogauth/utils.py b/cogauth/utils.py
@@ -46,6 +46,8 @@
 import time
 import logging
 import traceback
+import os
+from bigvince.utils import get_cognito_url, get_cognito_pool_url
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -58,12 +60,6 @@
 #response = urllib.request.urlopen(keys_url)
 #keys = json.loads(response.read())['keys']
 
-def get_cognito_pool_url(aws_region, aws_user_pool):
-
-    return (
-        "https://cognito-idp.{}.amazonaws.com/{}".
-        format(aws_region, aws_user_pool)
-    )
 
 
 def get_cognito(request):
@@ -109,7 +105,7 @@ def token_verify(token, expire_fail=False):
         logger.debug('Public key not found in jwks.json')
         return False
 
-    kargs = {"issuer": get_cognito_pool_url(settings.COGNITO_REGION, settings.COGNITO_USER_POOL_ID),
+    kargs = {"issuer": get_cognito_pool_url(),
              'audience': settings.COGNITO_APP_ID}
 
     try:
@@ -171,7 +167,8 @@ def get_group(token):
 
 def cognito_add_user_to_group(user, group):
     logger.debug("Sending request")
-    sns = boto3.client('sns', region_name=settings.AWS_REGION)
+    sns = boto3.client('sns',
+             endpoint_url=get_cognito_pool_url(), region_name=settings.AWS_REGION)
     response = sns.publish(
         TargetArn=settings.VINCE_ERROR_SNS_ARN,
         Subject='Please add user to cognito group',
@@ -399,14 +396,15 @@ def cognito_check_permissions(request):
 
 
 def cognito_verify_email(request):
-    client= boto3.client('cognito-idp', region_name=settings.COGNITO_REGION)
+    client= boto3.client('cognito-idp',
+             endpoint_url=get_cognito_url(),  region_name=settings.COGNITO_REGION)
     code = client.get_user_attribute_verification_code(AccessToken=request.session['ACCESS_TOKEN'],
                                                        AttributeName='email')
     print(code)
 
 
 def cognito_verify_sms(request):
-    client= boto3.client('cognito-idp', region_name=settings.COGNITO_REGION)
+    client= boto3.client('cognito-idp',  endpoint_url=get_cognito_url(), region_name=settings.COGNITO_REGION)
     code = client.get_user_attribute_verification_code(AccessToken=request.session['ACCESS_TOKEN'],
                                                        AttributeName='phone_number')
 
@@ -421,7 +419,7 @@ def cognito_to_dict(attr_list,mapping):
 
 
 def password_challenge_dance(username, password, new_password):
-    client = boto3.client('cognito-idp', region_name=settings.COGNITO_REGION)
+    client = boto3.client('cognito-idp',  endpoint_url=get_cognito_url(), region_name=settings.COGNITO_REGION)
     aws = AWSSRP(username=username,
                  password=password,
                  pool_id=settings.COGNITO_USER_POOL_ID,
@@ -440,7 +438,7 @@ def password_challenge_dance(username, password, new_password):
 
 def mfa_challenge(request, code):
 
-    client = boto3.client('cognito-idp', region_name=settings.COGNITO_REGION)
+    client = boto3.client('cognito-idp',  endpoint_url=get_cognito_url(), region_name=settings.COGNITO_REGION)
     aws = AWSSRP(username=request.session['username'], password="abc",
                  pool_id=settings.COGNITO_USER_POOL_ID,
                  client_id=settings.COGNITO_APP_ID, client=client)
@@ -461,7 +459,7 @@ def mfa_challenge(request, code):
 
 
 def rm_mfa(request):
-     client = boto3.client('cognito-idp', region_name=settings.COGNITO_REGION)
+     client = boto3.client('cognito-idp',  endpoint_url=get_cognito_url(), region_name=settings.COGNITO_REGION)
      if request.session['MFAREQUIRED'] == "SOFTWARE_TOKEN_MFA":
          client.set_user_mfa_preference(
              SoftwareTokenMfaSettings={
@@ -494,7 +492,7 @@ def send_courtesy_email(template_name, user):
     )
 
 def get_user_details(username):
-    client = boto3.client('cognito-idp', region_name=settings.COGNITO_REGION)
+    client = boto3.client('cognito-idp',  endpoint_url=get_cognito_url(), region_name=settings.COGNITO_REGION)
 
     res = client.admin_get_user(
         Username=username,
@@ -521,7 +519,7 @@ def create_service_account(request):
     return True, None
 
 def create_new_user(request, old_user=None):
-    client = boto3.client('cognito-idp', region_name=settings.COGNITO_REGION)
+    client = boto3.client('cognito-idp',  endpoint_url=get_cognito_url(), region_name=settings.COGNITO_REGION)
     messageaction = "SUPPRESS"
     if request.POST.get('send_email'):
         #user checked the box to send an email
@@ -588,7 +586,7 @@ def create_new_user(request, old_user=None):
 
 
 def admin_change_user_details(request, old_email):
-    client = boto3.client('cognito-idp', region_name=settings.COGNITO_REGION)
+    client = boto3.client('cognito-idp',  endpoint_url=get_cognito_url(), region_name=settings.COGNITO_REGION)
     try:
         res = client.admin_update_user_attributes(
             UserPoolId=settings.COGNITO_USER_POOL_ID,
@@ -643,7 +641,7 @@ def admin_change_user_details(request, old_email):
 
 
 def disable_sms_mfa(username):
-    client = boto3.client('cognito-idp', region_name=settings.COGNITO_REGION)
+    client = boto3.client('cognito-idp',  endpoint_url=get_cognito_url(), region_name=settings.COGNITO_REGION)
     res = client.admin_set_user_mfa_preference(
         SMSMfaSettings={
             'Enabled': False,
@@ -657,7 +655,7 @@ def disable_sms_mfa(username):
 
 def disable_totp_mfa(username):
 
-    client = boto3.client('cognito-idp', region_name=settings.COGNITO_REGION)
+    client = boto3.client('cognito-idp',  endpoint_url=get_cognito_url(), region_name=settings.COGNITO_REGION)
 
     res = client.admin_set_user_mfa_preference(
         SoftwareTokenMfaSettings={
Original file line number	Diff line number	Diff line change
Expand Up		@@ -74,6 +74,7 @@ reduce the risk of exposure.
		[<img src="https://github.com/CERTCC/VINCE/raw/main/Vince_Infrastructure.png" width="100%"></A>](https://github.com/CERTCC/VINCE/raw/main/Vince_Infrastructure.png)



		### Local Install

		1. Clone the repo
Expand Down