S3

.dockerignore

@@ -3,3 +3,4 @@ container_data
 .cache
 node_modules
 OpenOversight/app/static/dist/*
+/OpenOversight/tests/resources/.minio.sys/

.gitignore

@@ -94,4 +94,5 @@ vagrant/puppet/.tmp
 node_modules/
 OpenOverSight/app/static/dist/
 
-
+/OpenOversight/tests/resources/.minio.sys/
+OpenOversight/tests/resources/openoversight-test/

CONTRIB.md

@@ -40,17 +40,23 @@ Once you're done, `make stop` and `make clean` to stop and remove the containers
 ## Testing S3 Functionality
 
 We use an S3 bucket for image uploads. If you are working on functionality involving image uploads,
-then you should follow the "S3 Image Hosting" section in [DEPLOY.md](/DEPLOY.md) to make a test S3 bucket
-on Amazon Web Services.
+you can utilize a local S3 compatible storage layer named Minio.
+Test resources are located in `OpenOversight/tests/resources/openoversight-test`, note that `openoversight-test`
+in this instance is the name of our local S3 "bucket".
 
-Once you have done this, you can put your AWS credentials in the following environmental variables:
+For testing in a staging / production environment, then you should follow the "S3 Image Hosting" section
+in [DEPLOY.md](/DEPLOY.md) to make a test S3 bucket on Amazon Web Services.
+
+Once you have done this, you can update the `.env` file in the root of this repository, e.g.
+*IMPORTANT NOTE: DO NOT COMMIT ANY CHANGES TO THE `.env` FILE, INCLUDING ANY AWS CREDENTIALS OR PASSWORDS.*
 
 ```sh
-$ export S3_BUCKET_NAME=openoversight-test
-$ export AWS_ACCESS_KEY_ID=testtest
-$ export AWS_SECRET_ACCESS_KEY=testtest
-$ export AWS_DEFAULT_REGION=us-east-1
+S3_BUCKET_NAME=openoversight-test
+AWS_ACCESS_KEY_ID=minio123
+AWS_SECRET_ACCESS_KEY=minio123
+AWS_DEFAULT_REGION=us-east-1
 ```
+*IMPORTANT NOTE: DO NOT COMMIT ANY CHANGES TO THE `.env` FILE, INCLUDING ANY AWS CREDENTIALS OR PASSWORDS.*
 
 Now when you run `make dev` as usual in the same session, you will be able to submit images to
 your test bucket.

Makefile

@@ -18,11 +18,11 @@ create_db: start
 	done
 	@echo "Postgres is up"
 	## Creating database
-	docker-compose run --rm web python ../create_db.py
+	docker-compose exec web python ../create_db.py
 
 .PHONY: assets
 assets:
-	docker-compose run --rm web yarn build
+	docker-compose exec web yarn build
 
 .PHONY: dev
 dev: build start create_db populate
@@ -35,15 +35,19 @@ populate: create_db  ## Build and run containers
 	done
 	@echo "Postgres is up"
 	## Populate database with test data
-	docker-compose run --rm web python ../test_data.py -p
+	docker-compose exec web python ../test_data.py -p
 
 .PHONY: test
 test: start  ## Run tests
 	if [ -z "$(name)" ]; \
-	    then FLASK_ENV=testing docker-compose run --rm web pytest -n 4 --dist=loadfile -v tests/; \
-	    else FLASK_ENV=testing docker-compose run --rm web pytest -n 4 --dist=loadfile -v tests/ -k $(name); \
+	    then FLASK_ENV=testing docker-compose exec web pytest --doctest-modules -n 4 --dist=loadfile -v tests/ app; \
+	    else FLASK_ENV=testing docker-compose exec web pytest --doctest-modules -n 4 --dist=loadfile -v tests app -k $(name); \
 	fi
 
+.PHONY: lint
+lint: start
+	docker-compose exec web flake8
+
 .PHONY: cleanassets
 cleanassets:
 	rm -rf ./OpenOversight/app/static/dist/
@@ -71,3 +75,6 @@ help: ## Print this message and exit
 	@awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \
 		| sort \
 		| column -s ':' -t
+
+attach:
+	docker-compose exec postgres psql -h localhost -U openoversight openoversight-dev

OpenOversight/app/config.py

@@ -53,6 +53,7 @@ class DevelopmentConfig(BaseConfig):
     SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI')
     NUM_OFFICERS = 15000
     SITEMAP_URL_SCHEME = 'http'
+    S3_ENDPOINT = os.getenv('S3_ENDPOINT')
 
 
 class TestingConfig(BaseConfig):
@@ -62,6 +63,7 @@ class TestingConfig(BaseConfig):
     NUM_OFFICERS = 120
     APPROVE_REGISTRATIONS = False
     SITEMAP_URL_SCHEME = 'http'
+    S3_ENDPOINT = os.getenv('S3_ENDPOINT')
 
 
 class ProductionConfig(BaseConfig):

OpenOversight/app/utils.py

@@ -4,6 +4,7 @@
 from io import BytesIO
 
 import boto3
+from botocore.client import Config
 from botocore.exceptions import ClientError
 import botocore
 import datetime
@@ -29,6 +30,13 @@
 SAVED_UMASK = os.umask(0o077)
 
 
+def get_s3_connection(config=None):
+    if current_app.config['S3_ENDPOINT']:
+        return boto3.client('s3', config=config, endpoint_url=current_app.config['S3_ENDPOINT'])
+    else:
+        return boto3.client('s3', config=config)
+
+
 def set_dynamic_default(form_field, value):
     # First we ensure no value is set already
     if not form_field.data:
@@ -226,8 +234,22 @@ def compute_hash(data_to_hash):
     return hashlib.sha256(data_to_hash).hexdigest()
 
 
+def get_presigned_image_url(s3_path: str):
+    unsigned = Config(signature_version=botocore.UNSIGNED)
+    s3_client_unsigned = get_s3_connection(unsigned)
+
+    url = s3_client_unsigned.generate_presigned_url(
+        'get_object',
+        Params={'Bucket': current_app.config['S3_BUCKET_NAME'], 'Key': s3_path}
+    )
+    return url
+
+
 def upload_obj_to_s3(file_obj, dest_filename):
-    s3_client = boto3.client('s3')
+    if current_app.config['S3_ENDPOINT']:
+        s3_client = get_s3_connection(Config(signature_version='s3v4'))
+    else:
+        s3_client = get_s3_connection()
 
     # Folder to store files in on S3 is first two chars of dest_filename
     s3_folder = dest_filename[0:2]
@@ -240,16 +262,7 @@ def upload_obj_to_s3(file_obj, dest_filename):
                              current_app.config['S3_BUCKET_NAME'],
                              s3_path,
                              ExtraArgs={'ContentType': s3_content_type, 'ACL': 'public-read'})
-
-    config = s3_client._client_config
-    config.signature_version = botocore.UNSIGNED
-    url = boto3.resource(
-        's3', config=config).meta.client.generate_presigned_url(
-        'get_object',
-        Params={'Bucket': current_app.config['S3_BUCKET_NAME'],
-                'Key': s3_path})
-
-    return url
+    return get_presigned_image_url(s3_path)
 
 
 def filter_by_form(form, officer_query, department_id=None):

OpenOversight/tests/conftest.py

@@ -5,7 +5,6 @@
 import random
 from selenium import webdriver
 import time
-import threading
 from xvfbwrapper import Xvfb
 from faker import Faker
 import csv
@@ -14,7 +13,7 @@
 import os
 from PIL import Image as Pimage
 
-from OpenOversight.app import create_app, models
+from OpenOversight.app import create_app, models, utils
 from OpenOversight.app.utils import merge_dicts
 from OpenOversight.app.models import db as _db
 
@@ -131,10 +130,11 @@ def build_salary(officer):
 
 def assign_faces(officer, images):
     if random.uniform(0, 1) >= 0.5:
-        for num in range(1, len(images)):
-            return models.Face(officer_id=officer.id,
-                               img_id=num,
-                               original_image_id=random.choice(images).id)
+        # for num in range(1, len(images)):  # <-- bug here, we always return on the first one, the for-loop is unneeded
+        img_id = random.choice(images).id
+        return models.Face(officer_id=officer.id,
+                           img_id=img_id,
+                           original_image_id=img_id)
     else:
         return False
 
@@ -251,10 +251,15 @@ def add_mockdata(session):
 
     unit1 = models.Unit(descrip="test")
 
-    test_images = [models.Image(filepath='/static/images/test_cop{}.png'.format(x + 1), department_id=1) for x in range(5)] + \
-        [models.Image(filepath='/static/images/test_cop{}.png'.format(x + 1), department_id=2) for x in range(5)]
+    test_images = [
+        models.Image(
+            filepath=utils.get_presigned_image_url('/images/test_cop{}.png'.format(x + 1)),
+            department_id=1) for x in range(5)] + \
+        [models.Image(
+            filepath=utils.get_presigned_image_url('/images/test_cop{}.png'.format(x + 1)),
+            department_id=2) for x in range(5)]
 
-    officers = [generate_officer() for o in range(NUM_OFFICERS)]
+    officers = [generate_officer() for _ in range(NUM_OFFICERS)]
     session.add_all(officers)
     session.add_all(test_images)
 
@@ -549,12 +554,7 @@ def teardown():
 
 
 @pytest.fixture
-def browser(app, request):
-    # start server
-    threading.Thread(target=app.run).start()
-    # give the server a few seconds to ensure it is up
-    time.sleep(10)
-
+def browser():
     # start headless webdriver
     vdisplay = Xvfb()
     vdisplay.start()
@@ -563,10 +563,6 @@ def browser(app, request):
     time.sleep(3)
     yield driver
 
-    # shutdown server
-    driver.get("http://localhost:5000/shutdown")
-    time.sleep(3)
-
     # shutdown headless webdriver
     driver.quit()
     vdisplay.stop()