From 596380da68ae9fc25e0b86aae8ffdad7343459cf Mon Sep 17 00:00:00 2001 From: F D Date: Mon, 27 Jul 2020 20:43:05 -0400 Subject: [PATCH 01/13] add .gitignore and .dockerignore for minio, move static assets to minio --- .dockerignore | 1 + .gitignore | 3 ++- .../openoversight-test}/images/test_cop1.png | Bin .../openoversight-test}/images/test_cop2.png | Bin .../openoversight-test}/images/test_cop3.png | Bin .../openoversight-test}/images/test_cop4.png | Bin .../openoversight-test}/images/test_cop5.png | Bin 7 files changed, 3 insertions(+), 1 deletion(-) rename OpenOversight/{app/static => tests/resources/openoversight-test}/images/test_cop1.png (100%) rename OpenOversight/{app/static => tests/resources/openoversight-test}/images/test_cop2.png (100%) rename OpenOversight/{app/static => tests/resources/openoversight-test}/images/test_cop3.png (100%) rename OpenOversight/{app/static => tests/resources/openoversight-test}/images/test_cop4.png (100%) rename OpenOversight/{app/static => tests/resources/openoversight-test}/images/test_cop5.png (100%) diff --git a/.dockerignore b/.dockerignore index 63ba87096..26a1f1b6a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,3 +3,4 @@ container_data .cache node_modules OpenOversight/app/static/dist/* +/OpenOversight/tests/resources/.minio.sys/ diff --git a/.gitignore b/.gitignore index da5707c66..d4e722143 100644 --- a/.gitignore +++ b/.gitignore @@ -96,4 +96,5 @@ vagrant/puppet/.tmp node_modules/ OpenOverSight/app/static/dist/ - +!/OpenOversight/tests/resources/.minio.sys/ +OpenOversight/tests/resources/openoversight-test/ diff --git a/OpenOversight/app/static/images/test_cop1.png b/OpenOversight/tests/resources/openoversight-test/images/test_cop1.png similarity index 100% rename from OpenOversight/app/static/images/test_cop1.png rename to OpenOversight/tests/resources/openoversight-test/images/test_cop1.png diff --git a/OpenOversight/app/static/images/test_cop2.png b/OpenOversight/tests/resources/openoversight-test/images/test_cop2.png similarity index 100% rename from OpenOversight/app/static/images/test_cop2.png rename to OpenOversight/tests/resources/openoversight-test/images/test_cop2.png diff --git a/OpenOversight/app/static/images/test_cop3.png b/OpenOversight/tests/resources/openoversight-test/images/test_cop3.png similarity index 100% rename from OpenOversight/app/static/images/test_cop3.png rename to OpenOversight/tests/resources/openoversight-test/images/test_cop3.png diff --git a/OpenOversight/app/static/images/test_cop4.png b/OpenOversight/tests/resources/openoversight-test/images/test_cop4.png similarity index 100% rename from OpenOversight/app/static/images/test_cop4.png rename to OpenOversight/tests/resources/openoversight-test/images/test_cop4.png diff --git a/OpenOversight/app/static/images/test_cop5.png b/OpenOversight/tests/resources/openoversight-test/images/test_cop5.png similarity index 100% rename from OpenOversight/app/static/images/test_cop5.png rename to OpenOversight/tests/resources/openoversight-test/images/test_cop5.png From a6912f954f1567d0221dd4345fed7ad1886e941a Mon Sep 17 00:00:00 2001 From: F D Date: Mon, 27 Jul 2020 20:43:28 -0400 Subject: [PATCH 02/13] update docs for minio --- CONTRIB.md | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/CONTRIB.md b/CONTRIB.md index b8eeba74a..d7ea1f28c 100644 --- a/CONTRIB.md +++ b/CONTRIB.md @@ -44,17 +44,23 @@ Once you're done, `make stop` and `make clean` to stop and remove the containers ## Testing S3 Functionality We use an S3 bucket for image uploads. If you are working on functionality involving image uploads, -then you should follow the "S3 Image Hosting" section in [DEPLOY.md](/DEPLOY.md) to make a test S3 bucket -on Amazon Web Services. +you can utilize a local S3 compatible storage layer named Minio. +Test resources are located in `OpenOversight/tests/resources/openoversight-test`, note that `openoversight-test` +in this instance is the name of our local S3 "bucket". -Once you have done this, you can put your AWS credentials in the following environmental variables: +For testing in a staging / production environment, then you should follow the "S3 Image Hosting" section +in [DEPLOY.md](/DEPLOY.md) to make a test S3 bucket on Amazon Web Services. + +Once you have done this, you can update the `.env` file in the root of this repository, e.g. +*IMPORTANT NOTE: DO NOT COMMIT ANY CHANGES TO THE `.env` FILE, INCLUDING ANY AWS CREDENTIALS OR PASSWORDS.* ```sh -$ export S3_BUCKET_NAME=openoversight-test -$ export AWS_ACCESS_KEY_ID=testtest -$ export AWS_SECRET_ACCESS_KEY=testtest -$ export AWS_DEFAULT_REGION=us-east-1 +S3_BUCKET_NAME=openoversight-test +AWS_ACCESS_KEY_ID=minio123 +AWS_SECRET_ACCESS_KEY=minio123 +AWS_DEFAULT_REGION=us-east-1 ``` +*IMPORTANT NOTE: DO NOT COMMIT ANY CHANGES TO THE `.env` FILE, INCLUDING ANY AWS CREDENTIALS OR PASSWORDS.* Now when you run `make dev` as usual in the same session, you will be able to submit images to your test bucket. From f1adde506784b51769e7fb2bce777e35a38546a8 Mon Sep 17 00:00:00 2001 From: F D Date: Mon, 27 Jul 2020 20:47:16 -0400 Subject: [PATCH 03/13] add minio service, update app to use --- OpenOversight/app/utils.py | 35 +++++++++----- docker-compose.yml | 93 ++++++++++++++++++++++++-------------- 2 files changed, 83 insertions(+), 45 deletions(-) diff --git a/OpenOversight/app/utils.py b/OpenOversight/app/utils.py index 1ef660408..ed44138cd 100644 --- a/OpenOversight/app/utils.py +++ b/OpenOversight/app/utils.py @@ -6,6 +6,7 @@ from io import BytesIO import boto3 +from botocore.client import Config from botocore.exceptions import ClientError import botocore import datetime @@ -32,6 +33,13 @@ SAVED_UMASK = os.umask(0o077) +def get_s3_connection(config=None): + if os.getenv("S3_ENDPOINT"): + return boto3.client('s3', config=config, endpoint_url=os.getenv('S3_ENDPOINT')) + else: + return boto3.client('s3', config=config) + + def set_dynamic_default(form_field, value): # First we ensure no value is set already if not form_field.data: @@ -225,8 +233,22 @@ def compute_hash(data_to_hash): return hashlib.sha256(data_to_hash).hexdigest() +def get_presigned_image_url(s3_path: str): + unsigned = Config(signature_version=botocore.UNSIGNED) + s3_client_unsigned = get_s3_connection(unsigned) + + url = s3_client_unsigned.generate_presigned_url( + 'get_object', + Params={'Bucket': current_app.config['S3_BUCKET_NAME'], 'Key': s3_path} + ) + return url + + def upload_obj_to_s3(file_obj, dest_filename): - s3_client = boto3.client('s3') + if os.getenv('S3_CONNECTION'): + s3_client = get_s3_connection(Config(signature_version='s3v4')) + else: + s3_client = get_s3_connection() # Folder to store files in on S3 is first two chars of dest_filename s3_folder = dest_filename[0:2] @@ -239,16 +261,7 @@ def upload_obj_to_s3(file_obj, dest_filename): current_app.config['S3_BUCKET_NAME'], s3_path, ExtraArgs={'ContentType': s3_content_type, 'ACL': 'public-read'}) - - config = s3_client._client_config - config.signature_version = botocore.UNSIGNED - url = boto3.resource( - 's3', config=config).meta.client.generate_presigned_url( - 'get_object', - Params={'Bucket': current_app.config['S3_BUCKET_NAME'], - 'Key': s3_path}) - - return url + return get_presigned_image_url(s3_path) def filter_by_form(form, officer_query, department_id=None): diff --git a/docker-compose.yml b/docker-compose.yml index d99292fed..24960d86d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,41 +5,66 @@ volumes: driver: local services: - postgres: - restart: always - image: postgres:latest - environment: - POSTGRES_USER: openoversight - POSTGRES_PASSWORD: terriblepassword - POSTGRES_DB: openoversight-dev - volumes: - - postgres:/var/lib/postgresql/data - ports: - - "5432:5432" + postgres: + restart: always + image: postgres:latest + environment: + POSTGRES_USER: openoversight + POSTGRES_PASSWORD: terriblepassword + POSTGRES_DB: openoversight-dev + volumes: + - postgres:/var/lib/postgresql/data + ports: + - "5432:5432" - web: - restart: always - build: + web: + restart: always + build: context: . args: - - DOCKER_BUILD_ENV - - TRAVIS_PYTHON_VERSION + - DOCKER_BUILD_ENV + - TRAVIS_PYTHON_VERSION dockerfile: ./dockerfiles/web/Dockerfile - environment: - AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID}" - AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}" - AWS_DEFAULT_REGION: "${AWS_DEFAULT_REGION}" - S3_BUCKET_NAME: "${S3_BUCKET_NAME}" - APPROVE_REGISTRATIONS: "${APPROVE_REGISTRATIONS}" - FLASK_APP: app - FLASK_ENV: "${FLASK_ENV:-development}" - volumes: - - ./OpenOversight/:/usr/src/app/OpenOversight/:z - user: "${UID:?Docker-compose needs UID set to the current user id number. Try 'export UID' and run docker-compose again}" - links: - - postgres:postgres - expose: - - "3000" - command: scripts/entrypoint.sh - ports: - - "3000:3000" + environment: + AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID}" + AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}" + S3_ENDPOINT: "http://localhost:9000" + AWS_DEFAULT_REGION: "${AWS_DEFAULT_REGION}" + S3_BUCKET_NAME: "${S3_BUCKET_NAME}" + APPROVE_REGISTRATIONS: "${APPROVE_REGISTRATIONS}" + FLASK_APP: app + FLASK_ENV: "${FLASK_ENV:-development}" + volumes: + - ./OpenOversight/:/usr/src/app/OpenOversight/:z + user: "${UID:?Docker-compose needs UID set to the current user id number. Try 'export UID' and run docker-compose again}" + links: + - postgres:postgres + expose: + - "3000" + command: scripts/entrypoint.sh + ports: + - "3000:3000" + + minio: + image: minio/minio + environment: + MINIO_ACCESS_KEY: "${AWS_ACCESS_KEY_ID}" + MINIO_SECRET_KEY: "${AWS_SECRET_ACCESS_KEY}" + command: server /data + volumes: + - ./OpenOversight/tests/resources/:/data:rw + ports: + - "9000:9000" + network_mode: "host" + + mc: + image: minio/mc + depends_on: + - minio + entrypoint: > + /bin/sh -c " + sleep 10; + /usr/bin/mc config host add minio http://localhost:9000 ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY}; + /usr/bin/mc policy set public minio/${S3_BUCKET_NAME}; + " + network_mode: "host" From d9d7527a2b117f69522639e3213f4e47684eb4b7 Mon Sep 17 00:00:00 2001 From: F D Date: Mon, 27 Jul 2020 20:53:14 -0400 Subject: [PATCH 04/13] update test data images to use minio --- OpenOversight/tests/conftest.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/OpenOversight/tests/conftest.py b/OpenOversight/tests/conftest.py index 25f54cc70..8266d9a0d 100644 --- a/OpenOversight/tests/conftest.py +++ b/OpenOversight/tests/conftest.py @@ -16,7 +16,7 @@ import os from PIL import Image as Pimage -from OpenOversight.app import create_app, models +from OpenOversight.app import create_app, models, utils from OpenOversight.app.utils import merge_dicts from OpenOversight.app.models import db as _db, Unit, Job, Officer @@ -286,8 +286,13 @@ def add_mockdata(session): session.add_all(test_units) session.commit() - test_images = [models.Image(filepath='/static/images/test_cop{}.png'.format(x + 1), department_id=1) for x in range(5)] + \ - [models.Image(filepath='/static/images/test_cop{}.png'.format(x + 1), department_id=2) for x in range(5)] + test_images = [ + models.Image( + filepath=utils.get_presigned_image_url('/images/test_cop{}.png'.format(x + 1)), + department_id=1) for x in range(5)] + \ + [models.Image( + filepath=utils.get_presigned_image_url('/images/test_cop{}.png'.format(x + 1)), + department_id=2) for x in range(5)] test_officer_links = [ models.Link( @@ -302,7 +307,7 @@ def add_mockdata(session): author='the internet'), ] - officers = [generate_officer() for o in range(NUM_OFFICERS)] + officers = [generate_officer() for _ in range(NUM_OFFICERS)] officers[0].links = test_officer_links session.add_all(officers) session.add_all(test_images) From 1f190fd405f1511c05b937c3f2dcf4dc5d87983e Mon Sep 17 00:00:00 2001 From: F D Date: Mon, 27 Jul 2020 23:42:46 -0400 Subject: [PATCH 05/13] add host network to docker, so UI can access minio --- docker-compose.yml | 6 ++---- dockerfiles/web/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 24960d86d..b1ddf159b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,7 @@ services: - postgres:/var/lib/postgresql/data ports: - "5432:5432" + network_mode: "host" web: restart: always @@ -37,13 +38,10 @@ services: volumes: - ./OpenOversight/:/usr/src/app/OpenOversight/:z user: "${UID:?Docker-compose needs UID set to the current user id number. Try 'export UID' and run docker-compose again}" - links: - - postgres:postgres - expose: - - "3000" command: scripts/entrypoint.sh ports: - "3000:3000" + network_mode: "host" minio: image: minio/minio diff --git a/dockerfiles/web/Dockerfile b/dockerfiles/web/Dockerfile index 20eee30db..dfea8e3a1 100644 --- a/dockerfiles/web/Dockerfile +++ b/dockerfiles/web/Dockerfile @@ -38,7 +38,7 @@ EXPOSE 3000 ENV PATH="/usr/src/app/geckodriver:${PATH}" ENV SECRET_KEY 4Q6ZaQQdiqtmvZaxP1If -ENV SQLALCHEMY_DATABASE_URI postgresql://openoversight:terriblepassword@postgres/openoversight-dev +ENV SQLALCHEMY_DATABASE_URI postgresql://openoversight:terriblepassword@localhost:5432/openoversight-dev WORKDIR /usr/src/app/OpenOversight From 584d8ede274daf0e887fc23c41ac28ac889a5fb2 Mon Sep 17 00:00:00 2001 From: F D Date: Mon, 27 Jul 2020 23:44:00 -0400 Subject: [PATCH 06/13] fix gitignore for minio --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d4e722143..779558f10 100644 --- a/.gitignore +++ b/.gitignore @@ -96,5 +96,5 @@ vagrant/puppet/.tmp node_modules/ OpenOverSight/app/static/dist/ -!/OpenOversight/tests/resources/.minio.sys/ +/OpenOversight/tests/resources/.minio.sys/ OpenOversight/tests/resources/openoversight-test/ From 4d931b0b45b729fb21bdc38dd6d7fbc8f5d18fdb Mon Sep 17 00:00:00 2001 From: F D Date: Fri, 31 Jul 2020 16:23:04 -0400 Subject: [PATCH 07/13] steal dockerfile from officer_unit_view for linting --- dockerfiles/web/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/web/Dockerfile b/dockerfiles/web/Dockerfile index dfea8e3a1..20eee30db 100644 --- a/dockerfiles/web/Dockerfile +++ b/dockerfiles/web/Dockerfile @@ -38,7 +38,7 @@ EXPOSE 3000 ENV PATH="/usr/src/app/geckodriver:${PATH}" ENV SECRET_KEY 4Q6ZaQQdiqtmvZaxP1If -ENV SQLALCHEMY_DATABASE_URI postgresql://openoversight:terriblepassword@localhost:5432/openoversight-dev +ENV SQLALCHEMY_DATABASE_URI postgresql://openoversight:terriblepassword@postgres/openoversight-dev WORKDIR /usr/src/app/OpenOversight From 7d2ef544d38dd0be7074dd4014307822ca11753c Mon Sep 17 00:00:00 2001 From: F D Date: Mon, 3 Aug 2020 16:50:28 -0400 Subject: [PATCH 08/13] fix postgres in dockerfile --- OpenOversight/tests/conftest.py | 1 - dockerfiles/web/Dockerfile | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/OpenOversight/tests/conftest.py b/OpenOversight/tests/conftest.py index 8266d9a0d..411ba1381 100644 --- a/OpenOversight/tests/conftest.py +++ b/OpenOversight/tests/conftest.py @@ -7,7 +7,6 @@ import random from selenium import webdriver import time -import threading from xvfbwrapper import Xvfb from faker import Faker import csv diff --git a/dockerfiles/web/Dockerfile b/dockerfiles/web/Dockerfile index 20eee30db..dfea8e3a1 100644 --- a/dockerfiles/web/Dockerfile +++ b/dockerfiles/web/Dockerfile @@ -38,7 +38,7 @@ EXPOSE 3000 ENV PATH="/usr/src/app/geckodriver:${PATH}" ENV SECRET_KEY 4Q6ZaQQdiqtmvZaxP1If -ENV SQLALCHEMY_DATABASE_URI postgresql://openoversight:terriblepassword@postgres/openoversight-dev +ENV SQLALCHEMY_DATABASE_URI postgresql://openoversight:terriblepassword@localhost:5432/openoversight-dev WORKDIR /usr/src/app/OpenOversight From aa16d7f62dd363cc5734af067557d384313b3aba Mon Sep 17 00:00:00 2001 From: F D Date: Sun, 9 Aug 2020 21:32:24 -0400 Subject: [PATCH 09/13] move os.getenv to config.py --- OpenOversight/app/config.py | 2 ++ OpenOversight/app/utils.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/OpenOversight/app/config.py b/OpenOversight/app/config.py index 1e2077b20..92e3e0cdd 100644 --- a/OpenOversight/app/config.py +++ b/OpenOversight/app/config.py @@ -53,6 +53,7 @@ class DevelopmentConfig(BaseConfig): SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI') NUM_OFFICERS = 15000 SITEMAP_URL_SCHEME = 'http' + S3_ENDPOINT = os.getenv('S3_ENDPOINT') class TestingConfig(BaseConfig): @@ -62,6 +63,7 @@ class TestingConfig(BaseConfig): NUM_OFFICERS = 120 APPROVE_REGISTRATIONS = False SITEMAP_URL_SCHEME = 'http' + S3_ENDPOINT = os.getenv('S3_ENDPOINT') class ProductionConfig(BaseConfig): diff --git a/OpenOversight/app/utils.py b/OpenOversight/app/utils.py index ed44138cd..7c3b2c5dd 100644 --- a/OpenOversight/app/utils.py +++ b/OpenOversight/app/utils.py @@ -34,8 +34,8 @@ def get_s3_connection(config=None): - if os.getenv("S3_ENDPOINT"): - return boto3.client('s3', config=config, endpoint_url=os.getenv('S3_ENDPOINT')) + if current_app.config['S3_ENDPOINT']: + return boto3.client('s3', config=config, endpoint_url=current_app.config['S3_ENDPOINT']) else: return boto3.client('s3', config=config) @@ -245,7 +245,7 @@ def get_presigned_image_url(s3_path: str): def upload_obj_to_s3(file_obj, dest_filename): - if os.getenv('S3_CONNECTION'): + if current_app.config['S3_ENDPOINT']: s3_client = get_s3_connection(Config(signature_version='s3v4')) else: s3_client = get_s3_connection() From 698105a0adfbda0ca039f46085e069a4546a5dc3 Mon Sep 17 00:00:00 2001 From: F D Date: Sun, 9 Aug 2020 22:11:01 -0400 Subject: [PATCH 10/13] move env vars to docker-compose as defaults --- docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b1ddf159b..6ec72854a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,11 +27,11 @@ services: - TRAVIS_PYTHON_VERSION dockerfile: ./dockerfiles/web/Dockerfile environment: - AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID}" - AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY}" - S3_ENDPOINT: "http://localhost:9000" - AWS_DEFAULT_REGION: "${AWS_DEFAULT_REGION}" - S3_BUCKET_NAME: "${S3_BUCKET_NAME}" + AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID:-minio123}" + AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY:-minio123}" + S3_ENDPOINT: "${S3_ENDPOINT:-http://localhost:9000}" + AWS_DEFAULT_REGION: "${AWS_DEFAULT_REGION:-us-east-1}" + S3_BUCKET_NAME: "${S3_BUCKET_NAME:-openoversight-test}" APPROVE_REGISTRATIONS: "${APPROVE_REGISTRATIONS}" FLASK_APP: app FLASK_ENV: "${FLASK_ENV:-development}" From 72f74273f220c86a088c8302aeb234f796386b0e Mon Sep 17 00:00:00 2001 From: F D Date: Thu, 13 Aug 2020 16:09:08 -0400 Subject: [PATCH 11/13] quote defaults in docker-compose --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6ec72854a..0750e7ee2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,9 +29,9 @@ services: environment: AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID:-minio123}" AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY:-minio123}" - S3_ENDPOINT: "${S3_ENDPOINT:-http://localhost:9000}" - AWS_DEFAULT_REGION: "${AWS_DEFAULT_REGION:-us-east-1}" - S3_BUCKET_NAME: "${S3_BUCKET_NAME:-openoversight-test}" + S3_ENDPOINT: "${S3_ENDPOINT:-'http://localhost:9000'}" + AWS_DEFAULT_REGION: "${AWS_DEFAULT_REGION:-'us-east-1'}" + S3_BUCKET_NAME: "${S3_BUCKET_NAME:-'openoversight-test'}" APPROVE_REGISTRATIONS: "${APPROVE_REGISTRATIONS}" FLASK_APP: app FLASK_ENV: "${FLASK_ENV:-development}" From c99e8fff9741ac069a4d83f97bcdc3a02936c691 Mon Sep 17 00:00:00 2001 From: abandoned-prototype <41744410+abandoned-prototype@users.noreply.github.com> Date: Thu, 29 Oct 2020 01:14:25 -0500 Subject: [PATCH 12/13] minio connections --- .dockerignore | 3 +++ Makefile | 6 +++--- OpenOversight/scripts/entrypoint.sh | 2 ++ OpenOversight/scripts/minio_forward.sh | 1 + OpenOversight/tests/conftest.py | 1 + docker-compose.yml | 18 +++++++----------- dockerfiles/web/Dockerfile | 4 ++-- 7 files changed, 19 insertions(+), 16 deletions(-) create mode 100755 OpenOversight/scripts/minio_forward.sh diff --git a/.dockerignore b/.dockerignore index 26a1f1b6a..5c496c43b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,9 @@ backup/** +*.pyc container_data .cache node_modules OpenOversight/app/static/dist/* /OpenOversight/tests/resources/.minio.sys/ +__pycache__ + diff --git a/Makefile b/Makefile index 29c195f58..b5b01b824 100644 --- a/Makefile +++ b/Makefile @@ -41,12 +41,12 @@ populate: create_db ## Build and run containers test: start ## Run tests if [ -z "$(name)" ]; then \ if [ "$$(uname)" == "Darwin" ]; then \ - FLASK_ENV=testing docker-compose run --rm web pytest --doctest-modules -n $$(sysctl -n hw.logicalcpu) --dist=loadfile -v tests/ app; \ + FLASK_ENV=testing docker-compose run --rm web /bin/bash -c 'scripts/minio_forward.sh & pytest --doctest-modules -n "$$(sysctl -n hw.logicalcpu)" --dist=loadfile -v tests/ app'; \ else \ - FLASK_ENV=testing docker-compose run --rm web pytest --doctest-modules -n $$(nproc --all) --dist=loadfile -v tests/ app; \ + FLASK_ENV=testing docker-compose run --rm web /bin/bash -c 'scripts/minio_forward.sh & pytest --doctest-modules -n "$$(nproc --all)" --dist=loadfile -v tests/ app'; \ fi; \ else \ - FLASK_ENV=testing docker-compose run --rm web pytest --doctest-modules -v tests/ app -k $(name); \ + FLASK_ENV=testing docker-compose run --rm web /bin/bash -c 'scripts/minio_forward.sh & pytest --doctest-modules -v tests/ app -k $(name)'; \ fi .PHONY: lint diff --git a/OpenOversight/scripts/entrypoint.sh b/OpenOversight/scripts/entrypoint.sh index 677931b40..7bf938374 100755 --- a/OpenOversight/scripts/entrypoint.sh +++ b/OpenOversight/scripts/entrypoint.sh @@ -2,6 +2,8 @@ if [ "${DOCKER_BUILD_ENV:-}" == "production" ]; then flask run --host=0.0.0.0 --port=3000 else + # forwarding port 9000 to minio for local S3 bucket + scripts/minio_forward.sh & yarn build yarn watch & flask run --host=0.0.0.0 --port=3000 diff --git a/OpenOversight/scripts/minio_forward.sh b/OpenOversight/scripts/minio_forward.sh new file mode 100755 index 000000000..67fd8e5e1 --- /dev/null +++ b/OpenOversight/scripts/minio_forward.sh @@ -0,0 +1 @@ +ncat -k -l localhost 9000 --sh-exec "ncat minio 9000" diff --git a/OpenOversight/tests/conftest.py b/OpenOversight/tests/conftest.py index 411ba1381..8266d9a0d 100644 --- a/OpenOversight/tests/conftest.py +++ b/OpenOversight/tests/conftest.py @@ -7,6 +7,7 @@ import random from selenium import webdriver import time +import threading from xvfbwrapper import Xvfb from faker import Faker import csv diff --git a/docker-compose.yml b/docker-compose.yml index 0750e7ee2..071aae473 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,6 @@ services: - postgres:/var/lib/postgresql/data ports: - "5432:5432" - network_mode: "host" web: restart: always @@ -29,9 +28,9 @@ services: environment: AWS_ACCESS_KEY_ID: "${AWS_ACCESS_KEY_ID:-minio123}" AWS_SECRET_ACCESS_KEY: "${AWS_SECRET_ACCESS_KEY:-minio123}" - S3_ENDPOINT: "${S3_ENDPOINT:-'http://localhost:9000'}" - AWS_DEFAULT_REGION: "${AWS_DEFAULT_REGION:-'us-east-1'}" - S3_BUCKET_NAME: "${S3_BUCKET_NAME:-'openoversight-test'}" + S3_ENDPOINT: "${S3_ENDPOINT:-http://localhost:9000}" + AWS_DEFAULT_REGION: "${AWS_DEFAULT_REGION:-us-east-1}" + S3_BUCKET_NAME: "${S3_BUCKET_NAME:-openoversight-test}" APPROVE_REGISTRATIONS: "${APPROVE_REGISTRATIONS}" FLASK_APP: app FLASK_ENV: "${FLASK_ENV:-development}" @@ -41,19 +40,17 @@ services: command: scripts/entrypoint.sh ports: - "3000:3000" - network_mode: "host" minio: image: minio/minio environment: - MINIO_ACCESS_KEY: "${AWS_ACCESS_KEY_ID}" - MINIO_SECRET_KEY: "${AWS_SECRET_ACCESS_KEY}" + MINIO_ACCESS_KEY: "${AWS_ACCESS_KEY_ID:-minio123}" + MINIO_SECRET_KEY: "${AWS_SECRET_ACCESS_KEY:-minio123}" command: server /data volumes: - ./OpenOversight/tests/resources/:/data:rw ports: - "9000:9000" - network_mode: "host" mc: image: minio/mc @@ -62,7 +59,6 @@ services: entrypoint: > /bin/sh -c " sleep 10; - /usr/bin/mc config host add minio http://localhost:9000 ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY}; - /usr/bin/mc policy set public minio/${S3_BUCKET_NAME}; + /usr/bin/mc config host add minio http://minio:9000 ${AWS_ACCESS_KEY_ID:-minio123} ${AWS_SECRET_ACCESS_KEY:-minio123}; + /usr/bin/mc policy set public minio/${S3_BUCKET_NAME:-openoversight-test}; " - network_mode: "host" diff --git a/dockerfiles/web/Dockerfile b/dockerfiles/web/Dockerfile index dfea8e3a1..0b02582f9 100644 --- a/dockerfiles/web/Dockerfile +++ b/dockerfiles/web/Dockerfile @@ -9,7 +9,7 @@ ENV DISPLAY=:1 ENV GECKODRIVER_VERSION="v0.26.0" RUN echo "deb http://deb.debian.org/debian stretch-backports main" > /etc/apt/sources.list.d/backports.list RUN wget -O - https://deb.nodesource.com/setup_12.x | bash - -RUN apt-get update && apt-get install -y xvfb firefox-esr libpq-dev python3-dev nodejs && \ +RUN apt-get update && apt-get install -y xvfb firefox-esr libpq-dev python3-dev nodejs ncat && \ apt-get install -y -t stretch-backports libsqlite3-0 && apt-get clean RUN wget https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz @@ -38,7 +38,7 @@ EXPOSE 3000 ENV PATH="/usr/src/app/geckodriver:${PATH}" ENV SECRET_KEY 4Q6ZaQQdiqtmvZaxP1If -ENV SQLALCHEMY_DATABASE_URI postgresql://openoversight:terriblepassword@localhost:5432/openoversight-dev +ENV SQLALCHEMY_DATABASE_URI postgresql://openoversight:terriblepassword@postgres/openoversight-dev WORKDIR /usr/src/app/OpenOversight From b3f2dab1e0f0ff0bfb1c10854ecebd13dcf496d2 Mon Sep 17 00:00:00 2001 From: abandoned-prototype <41744410+abandoned-prototype@users.noreply.github.com> Date: Thu, 6 Jan 2022 01:07:02 -0600 Subject: [PATCH 13/13] pin wtforms version to avoid errors --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 0db5d2719..7cadf5279 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ Flask==1.0.2 python-dotenv==0.7.1 boto3==1.5.10 werkzeug==1.0.1 +wtforms==2.3.3 Flask-Bootstrap==3.3.7.1 Flask-Limiter==1.0.1 Flask-Login==0.4.1