Merge branch 'develop' into main

.circleci/config.yml

@@ -0,0 +1,120 @@
+version: 2.1
+
+steps:
+  - &setup_dependencies
+    run:
+      name: Install dependencies (required for fabric)
+      command: |
+        pyenv global 3.5.2
+        sudo apt-get update
+        sudo apt-get upgrade
+        sudo apt-get install libpq-dev
+        virtualenv --python=python3 .venv
+        source .venv/bin/activate
+        pip3 install -r requirements.txt
+        pip3 install -r dev-requirements.txt
+
+  - &add_ssh_keys
+    add_ssh_keys:
+      fingerprints:
+        - "44:81:8d:36:86:55:fa:82:eb:97:65:f4:d7:9a:0b:fa"
+
+  - &encrypt_and_save_backup
+    run:
+      name: Encrypt backup to shared GPG key, save to S3
+      command: |
+        export BACKUP_FILENAME=$(ls ~/project/backup/backup-*.tar.gz)
+
+        # Not using keyservers because they are hot garbage, pubkey is in env var
+        echo -e $GPG_KEY > OPENOVERSIGHT_GPG_PUB_KEY.asc
+        gpg --import OPENOVERSIGHT_GPG_PUB_KEY.asc
+        gpg --output backup.tar.gz.gpg --encrypt --trust-model always --batch --no-tty --recipient 0x3C4C259402A0E3B2 $BACKUP_FILENAME
+
+        # Upload this encrypted backup to S3
+        source .venv/bin/activate
+        pip3 install awscli
+        export TIMESTAMP=$(date +%s)
+        cp backup.tar.gz.gpg backup-$TIMESTAMP.tar.gz.gpg
+        aws s3 cp backup-$TIMESTAMP.tar.gz.gpg s3://openoversight-backups/
+
+  - &backup_production
+    run:
+      name: Backup production server
+      command: |
+        source .venv/bin/activate
+        fab production backup
+
+jobs:
+  staging_backup_and_deploy:
+    machine:
+      enabled: true
+      image: ubuntu-1604:201903-01
+    steps:
+      - checkout
+      - *setup_dependencies
+      - *add_ssh_keys
+
+      - run:
+          name: Backup staging server
+          command: |
+            source .venv/bin/activate
+            fab staging backup
+
+      - *encrypt_and_save_backup
+
+      - run:
+          name: Run migrations and deploy
+          command: |
+            source .venv/bin/activate
+            fab staging migrate
+
+  production_backup_and_deploy:
+    machine:
+      enabled: true
+      image: ubuntu-1604:201903-01
+    steps:
+      - checkout
+      - *setup_dependencies
+      - *add_ssh_keys
+      - *backup_production
+      - *encrypt_and_save_backup
+
+      - run:
+          name: Run migrations and deploy
+          command: |
+            source .venv/bin/activate
+            fab production migrate
+
+  production_backup_only:
+    docker:
+      - image: circleci/python:3.6
+    steps:
+      - checkout
+      - *setup_dependencies
+      - *add_ssh_keys
+      - *backup_production
+      - *encrypt_and_save_backup
+
+workflows:
+  staging_cd:
+    jobs:
+      - staging_backup_and_deploy:
+          filters:
+            branches:
+              only: develop
+  production_cd:
+    jobs:
+      - production_backup_and_deploy:
+          filters:
+            branches:
+              only: main
+  weekly:
+    triggers:
+      - schedule:
+          cron: "0 0 * * 0"
+          filters:
+            branches:
+              only:
+                - main
+    jobs:
+      - production_backup_only

.dockerignore

@@ -1,2 +1,5 @@
 backup/**
-container_data
+container_data
+.cache
+node_modules
+OpenOversight/app/static/dist/*

.gitignore

@@ -25,6 +25,7 @@ __pycache__/
 # Distribution / packaging
 .Python
 env/
+ENV_DIR/
 build/
 develop-eggs/
 dist/
@@ -86,3 +87,10 @@ vagrant/puppet/.tmp
 
 # Editor files
 .idea/
+.vscode/
+.vscode/**
+
+node_modules/
+OpenOverSight/app/static/dist/
+
+

.travis.yml

@@ -1,23 +1,26 @@
 language: python
-dist: trusty
-sudo: false
-addons:
-  firefox: "45.4.0esr"
+dist: xenial
+sudo: required
 python:
-  - "2.7"
+  - "3.5"
   - "3.6"
+  - "3.7"
+services:
+  - docker
+env:
+  - DOCKER_COMPOSE_VERSION=1.24.0
 before_install:
-  - wget https://github.com/mozilla/geckodriver/releases/download/v0.11.1/geckodriver-v0.11.1-linux64.tar.gz
-  - mkdir geckodriver
-  - tar -xzf geckodriver-v0.11.1-linux64.tar.gz -C geckodriver
-  - export PATH=$PATH:$PWD/geckodriver
+  - sudo rm /usr/local/bin/docker-compose
+  - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+  - chmod +x docker-compose
+  - sudo mv docker-compose /usr/local/bin
 install:
+  - true
 before_script:
-  - pip install -r dev-requirements.txt
-  - pip install coveralls
-  - cd OpenOversight
+  - pip install coveralls flake8==3.5.0
+  - sudo service postgresql stop
 script:
-  - py.test -v -n 4 --dist=loadfile --cov=app
-  - cd .. && flake8 --ignore=E501,E722
+  - sudo make test
+  - flake8 --ignore=E501,E722
 after_success:
   - coveralls

CONTRIB.md

@@ -97,6 +97,44 @@ $ flask db upgrade
 
 You can also downgrade the database using `flask db downgrade`.
 
+## Using a Virtual Environment
+One way to avoid hitting version incompatibility errors when running `flask` commands is to use a virtualenv.  See [Python Packaging user guidelines](https://packaging.python.org/guides/installing-using-pip-and-virtualenv/) for instructions on installing virtualenv.  After installing virtualenv, you can create a virtual environment by navigating to the OpenOversight directory and running the below
+
+```bash
+python3 -m virtualenv env
+```
+
+Confirm you're in the virtualenv by running 
+
+```bash
+which python  
+```
+
+The response should point to your `env` directory.  
+If you want to exit the virtualenv, run 
+
+```bash
+deactivate
+```
+
+To reactivate the virtualenv, run
+
+```bash
+source env/bin/activate
+```
+
+While in the virtualenv, you can install project dependencies by running 
+
+```bash
+pip install -r requirements.txt
+```
+
+and
+
+```bash
+pip install -r dev-requirements.txt
+```
+
 ## OpenOversight Management Interface
 
 In addition to generating database migrations, the Flask CLI can be used to run additional commands:
@@ -121,6 +159,7 @@ Options:
   --help     Show this message and exit.
 
 Commands:
+  bulk-add-officers            Bulk adds officers.
   db                           Perform database migrations.
   link-images-to-department    Link existing images to first department
   link-officers-to-department  Links officers and units to first department
@@ -140,3 +179,25 @@ Password:
 Type your password again:
 Administrator redshiftzero successfully added
 ```
+
+## Debugging OpenOversight - Use pdb with the app itself
+In `docker-compose.yml`, below the line specifying the port number, add the following lines to the `web` service:
+```yml
+   stdin_open: true
+   tty: true    
+```
+Also in `docker-compose.yml`, below the line specifying the `FLASK_ENV`, add the following to the `environment` portion of the `web` service:
+```yml
+  FLASK_DEBUG: 0
+```
+The above line disables the werkzeug reloader, which can otherwise cause a bug when you place a breakpoint in code that loads at import time, such as classes.  The werkzeug reloader will start one pdb process at import time and one when you navigate to the class.  This makes it impossible to interact with the pdb prompt, but we can fix it by disabling the reloader.
+
+To set a breakpoint in OpenOversight, first import the pdb module by adding `import pdb` to the file you want to debug.  Call `pdb.set_trace()` on its own line wherever you want to break for debugging.
+Next, in your terminal run `docker ps` to find the container id of the `openoversight_web` image, then run `docker attach ${container_id}` to connect to the debugger in your terminal.  You can now use pdb prompts to step through the app.
+
+## Debugging OpenOversight - Use pdb with a test
+If you want to run an individual test in debug mode, use the below command.
+```yml
+`docker-compose run --rm web pytest --pdb -v tests/ -k <test_name_here>`
+```
+Again, add `import pdb` to the file you want to debug, then write `pdb.set_trace()` wherever you want to drop a breakpoint.  Once the test is up and running in your terminal, you can debug it using pdb prompts.

DEPLOY.md

@@ -6,7 +6,7 @@ When this application is deployed you will need to do some setup. These instruct
 
 We distribute a `requirements.txt` file listing the things the application depends upon. `pip install -r requirements.txt` will install prerequisites.
 
-You may also need the `libpq-dev` package if psycopg2 fails to install.
+You will also need the `libpq-dev` and `python3-dev` packages (required to build `psycopg2`).
 
 # S3 Image Hosting
 
@@ -82,7 +82,7 @@ server {
                 proxy_pass http://127.0.0.1:4000;
                 proxy_redirect     off;
                 proxy_set_header Host $host;
-                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;                
+                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                 proxy_connect_timeout 300s;
                 proxy_read_timeout 300s;
                 client_max_body_size 20M;

Makefile

@@ -1,3 +1,5 @@
+export UID=$(shell id -u)
+
 default: build start create_db populate test stop clean
 
 .PHONY: build
@@ -16,7 +18,11 @@ create_db: start
 	done
 	@echo "Postgres is up"
 	## Creating database
-	docker-compose run --rm web /usr/local/bin/python ../create_db.py
+	docker-compose run --rm web python ../create_db.py
+
+.PHONY: assets
+assets:
+	docker-compose run --rm web yarn build
 
 .PHONY: dev
 dev: build start create_db populate
@@ -29,26 +35,30 @@ populate: create_db  ## Build and run containers
 	done
 	@echo "Postgres is up"
 	## Populate database with test data
-	docker-compose run --rm web /usr/local/bin/python ../test_data.py -p
+	docker-compose run --rm web python ../test_data.py -p
 
 .PHONY: test
 test: start  ## Run tests
 	if [ -z "$(name)" ]; \
-	    then FLASK_ENV=testing docker-compose run --rm web /usr/local/bin/pytest -n 4 --dist=loadfile -v tests/; \
-	    else FLASK_ENV=testing docker-compose run --rm web /usr/local/bin/pytest -n 4 --dist=loadfile -v tests/ -k $(name); \
+	    then FLASK_ENV=testing docker-compose run --rm web pytest -n 4 --dist=loadfile -v tests/; \
+	    else FLASK_ENV=testing docker-compose run --rm web pytest -n 4 --dist=loadfile -v tests/ -k $(name); \
 	fi
 
+.PHONY: cleanassets
+cleanassets:
+	rm -rf ./OpenOversight/app/static/dist/
+
 .PHONY: stop
 stop:  ## Stop containers
 	docker-compose stop
 
 .PHONY: clean
-clean: stop  ## Remove containers
+clean: cleanassets stop  ## Remove containers
 	docker-compose rm -f
 
 .PHONY: clean_all
 clean_all: clean stop ## Wipe database
-	rm -rf container_data
+	docker-compose down -v
 
 .PHONY: docs
 docs: ## Build project documentation in live reload for editing