From 923271df6424aae782f86a8f04a37f08db4fb38b Mon Sep 17 00:00:00 2001
From: Mason D <masond@gmail.com>
Date: Fri, 13 Nov 2020 13:18:37 +1100
Subject: [PATCH 01/15] Update Pillow (#850)

A couple low-priority CVEs have been found, and also we're running a
pretty old release.
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 0522e8674..09c5dd4c7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,7 @@ Flask-Migrate==2.1.1
 Flask-SQLAlchemy==2.4.4
 Flask-WTF==0.14.3
 Flask-Sitemap==0.3.0
-Pillow==5.2.0
+Pillow==7.2.0
 psycopg2==2.8.5 --no-binary psycopg2
 python-dateutil==2.8.1
 SQLAlchemy==1.3.0

From ccb8cf6fe7e93d498f4c6c322f0c3dbe6817dac5 Mon Sep 17 00:00:00 2001
From: StringCheez <73414616+StringCheez@users.noreply.github.com>
Date: Thu, 12 Nov 2020 21:44:56 -0500
Subject: [PATCH 02/15] normalizing dates, setting default assignment (#849)

* normalizing dates, setting default assignment

* comment fix, cleanup
---
 OpenOversight/app/commands.py | 29 ++++++++++++++++++++++++-----
 1 file changed, 24 insertions(+), 5 deletions(-)

diff --git a/OpenOversight/app/commands.py b/OpenOversight/app/commands.py
index 2ca6b6989..6d991fc24 100644
--- a/OpenOversight/app/commands.py
+++ b/OpenOversight/app/commands.py
@@ -3,7 +3,8 @@
 import csv
 import sys
 from builtins import input
-from datetime import datetime
+from datetime import datetime, date
+from dateutil.parser import parse
 from getpass import getpass
 from typing import Dict, List
 
@@ -190,7 +191,24 @@ def update_officer_field(fieldname):
             if row[fieldname] == '':
                 row[fieldname] = None
             old_value = getattr(officer, fieldname)
-            new_value = row[fieldname]
+            # If we're expecting a date type, attempt to parse row[fieldname] as a datetime
+            # This also normalizes all date formats, ensuring the following comparison works properly
+            if isinstance(old_value, (date, datetime)):
+                try:
+                    new_value = parse(row[fieldname])
+                    if isinstance(old_value, date):
+                        new_value = new_value.date()
+                except Exception as e:
+                    msg = 'Field {} is a date-type, but "{}" was specified for Officer {} {} and cannot be parsed as a date-type.\nError message from dateutil: {}'.format(
+                        fieldname,
+                        row[fieldname],
+                        officer.first_name,
+                        officer.last_name,
+                        e
+                    )
+                    raise Exception(msg)
+            else:
+                new_value = row[fieldname]
             if old_value is None:
                 update_officer_field(fieldname)
             elif str(old_value) != str(new_value):
@@ -264,8 +282,9 @@ def try_else_false(comparable):
 
 def process_assignment(row, officer, compare=False):
     assignment_fields = {
-        'required': ['job_title'],
+        'required': [],
         'optional': [
+            'job_title',
             'star_no',
             'unit_id',
             'star_date',
@@ -292,13 +311,13 @@ def process_assignment(row, officer, compare=False):
                         i += 1
                 if i == len(assignment_fieldnames):
                     job_title = job.job_title
-                    if (job_title and 'job_title' in row and row['job_title'] == job_title) or \
+                    if (job_title and row.get('job_title', 'Not Sure') == job_title) or \
                             (not job_title and ('job_title' not in row or not row['job_title'])):
                         # Found match, so don't add new assignment
                         add_assignment = False
         if add_assignment:
             job = Job.query\
-                     .filter_by(job_title=row['job_title'],
+                     .filter_by(job_title=row.get('job_title', 'Not Sure'),
                                 department_id=officer.department_id)\
                      .one_or_none()
             if not job:

From 5f7a464e5567cf4d76a70a33ab0bb0d1e72db0b3 Mon Sep 17 00:00:00 2001
From: abandoned-prototype
 <41744410+abandoned-prototype@users.noreply.github.com>
Date: Thu, 12 Nov 2020 22:40:42 -0600
Subject: [PATCH 03/15] fix link from image sorting to image tagging

---
 OpenOversight/app/templates/sort.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/OpenOversight/app/templates/sort.html b/OpenOversight/app/templates/sort.html
index 63b2fcccb..3a1ce8a7c 100644
--- a/OpenOversight/app/templates/sort.html
+++ b/OpenOversight/app/templates/sort.html
@@ -71,7 +71,7 @@ <h3>Your account has been disabled due to too many incorrect classifications/tag
       {% else %}
       <h3>All images have been classfied!</h3>
       <p><a href="{{ url_for('main.submit_data')}}" class="btn btn-lg btn-primary" role="button">Submit officer pictures to us</a></p>
-      <p><a href="{{ url_for('main.label_data')}}" class="btn btn-lg btn-primary" role="button">identify officers in the classified images</a></p>
+      <p><a href="{{ url_for('main.label_data', department_id=department_id)}}" class="btn btn-lg btn-primary" role="button">identify officers in the classified images</a></p>
       {% endif %}
     {% endif %}
 

From f372c707bc1ed41f3a11fff8f6097f8e8fbf3ef2 Mon Sep 17 00:00:00 2001
From: abandoned-prototype
 <41744410+abandoned-prototype@users.noreply.github.com>
Date: Thu, 19 Nov 2020 01:05:54 -0600
Subject: [PATCH 04/15] update CONTRIB.md and PR template (#853)

---
 CONTRIB.md               | 16 ++++++++++++++--
 PULL_REQUEST_TEMPLATE.md | 11 +++++++++++
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/CONTRIB.md b/CONTRIB.md
index b8eeba74a..be58ad20d 100644
--- a/CONTRIB.md
+++ b/CONTRIB.md
@@ -2,12 +2,24 @@
 
 First, thanks for being interested in helping us out! If you find an issue you're interested in, feel free to make a comment about how you're thinking of approaching implementing it in the issue and we can give you feedback.  Please also read our [code of conduct](/CODE_OF_CONDUCT.md) before getting started.
 
-## Submitting a PR
+## Submitting a Pull Request (PR)
 
-When you come to implement your new feature, you should branch off `develop` and add commits to implement your feature. If your git history is not so clean, please do rewrite before you submit your PR - if you're not sure if you need to do this, go ahead and submit and we can let you know when you submit.
+When you come to implement your new feature, clone the repository and then create a branch off `develop` locally and add commits to implement your feature.
+
+If your git history is not so clean, please do rewrite before you submit your PR - if you're not sure if you need to do this, go ahead and submit and we can let you know when you submit.
+
+To submit your changes for review you have to fork the repository, push your new branch there and then create a Pull Request with `OpenOversight:develop` as the target.
 
 Use [PULL_REQUEST_TEMPLATE.md](/PULL_REQUEST_TEMPLATE.md) to create the description for your PR! (The template should populate automatically when you go to open the pull request.)
 
+### Recommended privacy settings
+Whenever you make a commit with `git` the name and email saved locally is stored with that commit and will become part of the public history of the project. This can be an unwanted, for example when using a work computer. We recommond changing the email-settings in the github account at https://github.com/settings/emails and selecting "Keep my email addresses private" as well as "Block command line pushes that expose my email". Also find your github-email address of the form `<id>+<username>@users.noreply.github.com` in that section. Then you can change the email and username stored with your commits by running the following commands
+```
+git config user.email "<your-github-email>"
+git config user.name "<your-github-username>"
+```
+This will make sure that all commits you make locally are associated with your github account and do not contain any additional identifying information. More detailed information on this topic can be found [here](https://docs.github.com/en/free-pro-team@latest/github/setting-up-and-managing-your-github-user-account/setting-your-commit-email-address).
+
 ### Linting / Style Checks
 
  `flake8` is a tool for automated linting and style checks. Be sure to run `flake8` and fix any errors before submitting a PR.
diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md
index ec4d6d3eb..aa65bf4cc 100644
--- a/PULL_REQUEST_TEMPLATE.md
+++ b/PULL_REQUEST_TEMPLATE.md
@@ -1,3 +1,14 @@
+<!-- New Contributor? Welcome!
+
+We recommend you check your privacy settings, so the name and email associated with
+the commit are what you want them to be. See the contribution guide at
+https://github.com/lucyparsons/OpenOversight/blob/develop/CONTRIB.md#recommended-privacy-settings for more infos.
+
+Also make sure you have read and abide by the code of conduct:
+https://github.com/lucyparsons/OpenOversight/blob/develop/CODE_OF_CONDUCT.md 
+
+-->
+
 ## Status
 
 Ready for review / in progress

From d9ddf2fd1890a3048ea2f52ca6c7f73294cf5e73 Mon Sep 17 00:00:00 2001
From: Emily Cain <11471197+EMCain@users.noreply.github.com>
Date: Wed, 2 Dec 2020 23:19:58 -0800
Subject: [PATCH 05/15] #711: normalize gender in DB and forms, improve filter
 by gender behavior (#833)

* add check constraint to officer model

* normalize existing gender data in migration

* Update gender options for test data

* Update forms.py to save None when gender unsure

* Normalize gender on input

* normalize gender when bulk adding officers

* do not add a filter if Not Sure is specified

* add test for including all genders

* Alter filtering forms to account for null genders

* fix failing tests

* allow_blank is unused now

* Add test_create_officer_from_row_adds_new_officer_and_normalizes_gender

* reorder GENDER_OPTIONS, fix save/load for EditOfficerForm

* Replace coerce_gender with lambda

* Update AddOfficerForm to use same gender field as EditOfficerForm

* add None representation to models.officer.gender_label

* Fix tests and add more detailed notes about testing

* Clean up linter issues

Co-authored-by: Julia Holgado <jholgado@newrelic.com>
---
 CONTRIB.md                                    | 16 +++-
 OpenOversight/app/commands.py                 | 23 +++++-
 OpenOversight/app/main/choices.py             |  3 +-
 OpenOversight/app/main/forms.py               | 24 +++++-
 OpenOversight/app/models.py                   | 10 ++-
 OpenOversight/app/templates/list_officer.html |  4 +-
 OpenOversight/app/utils.py                    | 24 +++++-
 .../versions/86eb228e4bc0_refactor_links.py   |  2 +-
 ...3b5360_constrain_officer_gender_options.py | 76 +++++++++++++++++++
 OpenOversight/tests/conftest.py               |  2 +-
 .../routes/test_officer_and_department.py     | 12 ++-
 OpenOversight/tests/test_commands.py          | 30 +++++++-
 OpenOversight/tests/test_utils.py             | 22 +++++-
 13 files changed, 221 insertions(+), 27 deletions(-)
 create mode 100644 OpenOversight/migrations/versions/cd39b33b5360_constrain_officer_gender_options.py

diff --git a/CONTRIB.md b/CONTRIB.md
index be58ad20d..14f92168c 100644
--- a/CONTRIB.md
+++ b/CONTRIB.md
@@ -213,7 +213,19 @@ Next, in your terminal run `docker ps` to find the container id of the `openover
 
 ## Debugging OpenOversight - Use pdb with a test
 If you want to run an individual test in debug mode, use the below command.
-```yml
-`docker-compose run --rm web pytest --pdb -v tests/ -k <test_name_here>`
+```bash
+docker-compose run --rm web pytest --pdb -v tests/ -k <test_name_here>
 ```
+
+where `<test_name_here>` is the name of a single test function, such as `test_ac_cannot_add_new_officer_not_in_their_dept`
+
+Similarly, you can run all the tests in a file by specifying the file path:
+
+```bash
+docker-compose run --rm web pytest --pdb -v path/to/test/file 
+```
+
+where `path/to/test/file` is the relative file path, minus the initial `OpenOversight`, such as 
+`tests/routes/test_officer_and_department.py`.
+
 Again, add `import pdb` to the file you want to debug, then write `pdb.set_trace()` wherever you want to drop a breakpoint.  Once the test is up and running in your terminal, you can debug it using pdb prompts.
\ No newline at end of file
diff --git a/OpenOversight/app/commands.py b/OpenOversight/app/commands.py
index 6d991fc24..0983ff113 100644
--- a/OpenOversight/app/commands.py
+++ b/OpenOversight/app/commands.py
@@ -12,9 +12,10 @@
 from flask import current_app
 from flask.cli import with_appcontext
 
+from .models import db, Assignment, Department, Officer, User, Salary, Job
+from .utils import get_officer, str_is_true, normalize_gender, prompt_yes_no
+
 from .csv_imports import import_csv_files
-from .models import Assignment, Department, Job, Officer, Salary, User, db
-from .utils import get_officer, prompt_yes_no, str_is_true
 
 
 @click.command()
@@ -156,6 +157,8 @@ def set_field_from_row(row, obj, attribute, allow_blank=True, fieldname=None):
             val = datetime.strptime(row[fieldname], '%Y-%m-%d').date()
         except ValueError:
             val = row[fieldname]
+            if attribute == 'gender':
+                val = normalize_gender(val)
         setattr(obj, attribute, val)
 
 
@@ -163,9 +166,12 @@ def update_officer_from_row(row, officer, update_static_fields=False):
     def update_officer_field(fieldname):
         if fieldname not in row:
             return
-        if row[fieldname] == '':
-            row[fieldname] = None
+
+        if fieldname == 'gender':
+            row[fieldname] = normalize_gender(row[fieldname])
+
         if row[fieldname] and getattr(officer, fieldname) != row[fieldname]:
+
             ImportLog.log_change(
                 officer,
                 'Updated {}: {} --> {}'.format(
@@ -176,6 +182,7 @@ def update_officer_field(fieldname):
     update_officer_field('last_name')
     update_officer_field('first_name')
     update_officer_field('middle_initial')
+
     update_officer_field('suffix')
     update_officer_field('gender')
 
@@ -403,7 +410,15 @@ def process_salary(row, officer, compare=False):
 @with_appcontext
 def bulk_add_officers(filename, no_create, update_by_name, update_static_fields):
     """Add or update officers from a CSV file."""
+
+    encoding = 'utf-8'
+
+    # handles unicode errors that can occur when the file was made in Excel
     with open(filename, 'r') as f:
+        if u'\ufeff' in f.readline():
+            encoding = 'utf-8-sig'
+
+    with open(filename, 'r', encoding=encoding) as f:
         ImportLog.clear_logs()
         csvfile = csv.DictReader(f)
         departments = {}
diff --git a/OpenOversight/app/main/choices.py b/OpenOversight/app/main/choices.py
index 2d3cbd7a3..2f159eb00 100644
--- a/OpenOversight/app/main/choices.py
+++ b/OpenOversight/app/main/choices.py
@@ -9,8 +9,7 @@
                 ('PACIFIC ISLANDER', 'Pacific Islander'),
                 ('Other', 'Other'), ('Not Sure', 'Not Sure')]
 
-GENDER_CHOICES = [('M', 'Male'), ('F', 'Female'), ('Other', 'Other'),
-                  ('Not Sure', 'Not Sure')]
+GENDER_CHOICES = [('Not Sure', 'Not Sure'), ('M', 'Male'), ('F', 'Female'), ('Other', 'Other')]
 
 STATE_CHOICES = [('', '')] + [(state.abbr, state.name) for state in states.STATES]
 LINK_CHOICES = [('', ''), ('link', 'Link'), ('video', 'YouTube Video'), ('other_video', 'Other Video')]
diff --git a/OpenOversight/app/main/forms.py b/OpenOversight/app/main/forms.py
index fc40c130f..1e739237c 100644
--- a/OpenOversight/app/main/forms.py
+++ b/OpenOversight/app/main/forms.py
@@ -17,6 +17,14 @@
 import datetime
 import re
 
+# Normalizes the "not sure" option to what it needs to be when writing to the database.
+# Note this should only be used for forms which save a record to the DB--not those that
+# are used to look up existing records.
+db_genders = list(GENDER_CHOICES)
+for index, choice in enumerate(db_genders):
+    if choice == ('Not Sure', 'Not Sure'):
+        db_genders[index] = (None, 'Not Sure')  # type: ignore
+
 
 def allowed_values(choices, empty_allowed=True):
     return [x[0] for x in choices if empty_allowed or x[0]]
@@ -205,8 +213,12 @@ class AddOfficerForm(Form):
                          validators=[AnyOf(allowed_values(SUFFIX_CHOICES))])
     race = SelectField('Race', default='WHITE', choices=RACE_CHOICES,
                        validators=[AnyOf(allowed_values(RACE_CHOICES))])
-    gender = SelectField('Gender', default='M', choices=GENDER_CHOICES,
-                         validators=[AnyOf(allowed_values(GENDER_CHOICES))])
+    gender = SelectField(
+        'Gender',
+        choices=GENDER_CHOICES,
+        coerce=lambda x: None if x == 'Not Sure' else x,
+        validators=[AnyOf(allowed_values(db_genders))]
+    )
     star_no = StringField('Badge Number', default='', validators=[
         Regexp(r'\w*'), Length(max=50)])
     unique_internal_identifier = StringField('Unique Internal Identifier', default='', validators=[Regexp(r'\w*'), Length(max=50)])
@@ -258,8 +270,12 @@ class EditOfficerForm(Form):
                          validators=[AnyOf(allowed_values(SUFFIX_CHOICES))])
     race = SelectField('Race', choices=RACE_CHOICES, coerce=lambda x: x or None,
                        validators=[AnyOf(allowed_values(RACE_CHOICES))])
-    gender = SelectField('Gender', choices=GENDER_CHOICES, coerce=lambda x: x or None,
-                         validators=[AnyOf(allowed_values(GENDER_CHOICES))])
+    gender = SelectField(
+        'Gender',
+        choices=GENDER_CHOICES,
+        coerce=lambda x: None if x == 'Not Sure' else x,
+        validators=[AnyOf(allowed_values(db_genders))]
+    )
     employment_date = DateField('Employment Date', validators=[Optional()])
     birth_year = IntegerField('Birth Year', validators=[Optional()])
     unique_internal_identifier = StringField('Unique Internal Identifier',
diff --git a/OpenOversight/app/models.py b/OpenOversight/app/models.py
index f03605988..67ca37c18 100644
--- a/OpenOversight/app/models.py
+++ b/OpenOversight/app/models.py
@@ -3,7 +3,7 @@
 from flask_sqlalchemy import SQLAlchemy
 from flask_sqlalchemy.model import DefaultMeta
 from sqlalchemy.orm import validates
-from sqlalchemy import UniqueConstraint
+from sqlalchemy import UniqueConstraint, CheckConstraint
 from werkzeug.security import generate_password_hash, check_password_hash
 from itsdangerous import TimedJSONWebSignatureSerializer as Serializer
 from itsdangerous import BadSignature, BadData
@@ -98,7 +98,7 @@ class Officer(BaseModel):
     middle_initial = db.Column(db.String(120), unique=False, nullable=True)
     suffix = db.Column(db.String(120), index=True, unique=False)
     race = db.Column(db.String(120), index=True, unique=False)
-    gender = db.Column(db.String(120), index=True, unique=False)
+    gender = db.Column(db.String(5), index=True, unique=False, nullable=True)
     employment_date = db.Column(db.Date, index=True, unique=False, nullable=True)
     birth_year = db.Column(db.Integer, index=True, unique=False, nullable=True)
     assignments = db.relationship('Assignment', backref='officer', lazy='dynamic')
@@ -117,6 +117,10 @@ class Officer(BaseModel):
     descriptions = db.relationship('Description', back_populates='officer', order_by='Description.date_created')
     salaries = db.relationship('Salary', back_populates='officer', order_by='Salary.year.desc()')
 
+    __table_args__ = (
+        CheckConstraint("gender in ('M', 'F', 'Other')", name='gender_options'),
+    )
+
     def full_name(self):
         if self.middle_initial:
             middle_initial = self.middle_initial + '.' if len(self.middle_initial) == 1 else self.middle_initial
@@ -135,6 +139,8 @@ def race_label(self):
                 return label
 
     def gender_label(self):
+        if self.gender is None:
+            return 'Data Missing'
         from .main.choices import GENDER_CHOICES
         for gender, label in GENDER_CHOICES:
             if self.gender == gender:
diff --git a/OpenOversight/app/templates/list_officer.html b/OpenOversight/app/templates/list_officer.html
index f362e62aa..cbcbf039b 100644
--- a/OpenOversight/app/templates/list_officer.html
+++ b/OpenOversight/app/templates/list_officer.html
@@ -67,10 +67,10 @@ <h3 class="panel-title accordion-toggle">Gender</h3>
           </div>
           <div class="collapse in" id="filter-gender">
             <div class="panel-body">
-              <div class="form-group checkbox">
+              <div class="form-group radio">
                 {% for choice in choices['gender'] %}
                   <label class="form-check">
-                    <input type="checkbox" class="form-check-input" id="gender-{{ choice[0] }}" name="gender" value="{{ choice[0] }}" {% if choice[0] in form_data['gender'] %}checked="checked" {% endif %}/>{{ choice[1] }}
+                    <input type="radio" class="form-check-input" id="gender-{{ choice[0] }}" name="gender" value="{{ choice[0] }}" {% if choice[0] in form_data['gender'] %}checked="checked" {% endif %}/>{{ choice[1] }}
                   </label>
                 {% endfor %}
               </div>
diff --git a/OpenOversight/app/utils.py b/OpenOversight/app/utils.py
index 1ef660408..524cb99b1 100644
--- a/OpenOversight/app/utils.py
+++ b/OpenOversight/app/utils.py
@@ -16,7 +16,7 @@
 from traceback import format_exc
 from distutils.util import strtobool
 
-from sqlalchemy import func
+from sqlalchemy import func, or_
 from sqlalchemy.sql.expression import cast
 import imghdr as imghdr
 from flask import current_app, url_for
@@ -294,9 +294,8 @@ def filter_by_form(form, officer_query, department_id=None):
         officer_query = officer_query.filter(Officer.race.in_(form['race']))
     gender_values = [x for x, _ in GENDER_CHOICES]
     if form.get('gender') and all(gender in gender_values for gender in form['gender']):
-        if 'Not Sure' in form['gender']:
-            form['gender'].append(None)
-        officer_query = officer_query.filter(Officer.gender.in_(form['gender']))
+        if 'Not Sure' not in form['gender']:
+            officer_query = officer_query.filter(or_(Officer.gender.in_(form['gender']), Officer.gender.is_(None)))
 
     if form.get('min_age') and form.get('max_age'):
         current_year = datetime.datetime.now().year
@@ -610,3 +609,20 @@ def prompt_yes_no(prompt, default="no"):
                              "(or 'y' or 'n').\n")
             continue
         return ret
+
+
+def normalize_gender(input_gender):
+    if input_gender is None:
+        return None
+    normalized_genders = {
+        "male": "M",
+        "m": "M",
+        "man": "M",
+        "female": "F",
+        "f": "F",
+        "woman": "F",
+        "nonbinary": "Other",
+        "other": "Other",
+    }
+
+    return normalized_genders.get(input_gender.lower().strip())
diff --git a/OpenOversight/migrations/versions/86eb228e4bc0_refactor_links.py b/OpenOversight/migrations/versions/86eb228e4bc0_refactor_links.py
index 724271e49..5f561fdc3 100644
--- a/OpenOversight/migrations/versions/86eb228e4bc0_refactor_links.py
+++ b/OpenOversight/migrations/versions/86eb228e4bc0_refactor_links.py
@@ -1,7 +1,7 @@
 """Refactor links
 
 Revision ID: 86eb228e4bc0
-Revises: 562bd5f1bc1f
+Revises: 79a14685454f
 Create Date: 2019-03-15 22:40:10.473917
 
 """
diff --git a/OpenOversight/migrations/versions/cd39b33b5360_constrain_officer_gender_options.py b/OpenOversight/migrations/versions/cd39b33b5360_constrain_officer_gender_options.py
new file mode 100644
index 000000000..f3639df70
--- /dev/null
+++ b/OpenOversight/migrations/versions/cd39b33b5360_constrain_officer_gender_options.py
@@ -0,0 +1,76 @@
+"""constrain officer gender options
+
+Revision ID: cd39b33b5360
+Revises: 86eb228e4bc0
+Create Date: 2020-07-13 02:45:07.533549
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'cd39b33b5360'
+down_revision = '86eb228e4bc0'
+branch_labels = None
+depends_on = None
+
+
+def get_update_statement(normalized, options):
+    template = """
+    UPDATE officers
+    SET gender = '{normalized}'
+    WHERE gender::citext in ({options});
+    """
+    options = ', '.join(["'" + o + "'" for o in options])
+    return template.format(normalized=normalized, options=options)
+
+
+def upgrade():
+    conn = op.get_bind()
+    create_extn = "create extension IF NOT EXISTS citext;"
+    conn.execute(create_extn)
+
+    genders = {
+        "M": ('male', 'm', 'man'),
+        "F": ('female', 'f', 'woman'),
+        "Other": ('nonbinary', 'other'),
+    }
+
+    update_statement = ''
+
+    for normalized, options in genders.items():
+        update_statement += get_update_statement(normalized, options)
+
+    conn.execute(update_statement)
+
+    null_query = """
+UPDATE officers
+SET gender = NULL
+WHERE gender not in ('M', 'F', 'Other');
+"""
+    conn.execute(null_query)
+
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('officers', 'gender',
+                    existing_type=sa.VARCHAR(length=120),
+                    type_=sa.VARCHAR(length=5),
+                    existing_nullable=True)
+    # ### end Alembic commands ###
+
+    op.create_check_constraint(
+        'gender_options',
+        'officers',
+        "gender in ('M', 'F', 'Other')"
+    )
+
+
+def downgrade():
+    op.drop_constraint('gender_options', 'officers', type_='check')
+
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('officers', 'gender',
+                    existing_type=sa.VARCHAR(length=5),
+                    type_=sa.VARCHAR(length=120),
+                    existing_nullable=True)
+    # ### end Alembic commands ###
diff --git a/OpenOversight/tests/conftest.py b/OpenOversight/tests/conftest.py
index 25f54cc70..9edb33f74 100644
--- a/OpenOversight/tests/conftest.py
+++ b/OpenOversight/tests/conftest.py
@@ -63,7 +63,7 @@ def pick_race():
 
 
 def pick_gender():
-    return random.choice(['M', 'F', 'Not Sure'])
+    return random.choice(['M', 'F', 'Other', None])
 
 
 def pick_first():
diff --git a/OpenOversight/tests/routes/test_officer_and_department.py b/OpenOversight/tests/routes/test_officer_and_department.py
index 9e7e26166..714ef00c4 100644
--- a/OpenOversight/tests/routes/test_officer_and_department.py
+++ b/OpenOversight/tests/routes/test_officer_and_department.py
@@ -871,7 +871,11 @@ def test_ac_can_add_new_officer_in_their_dept(mockdata, client, session):
             last_name=last_name).one()
         assert officer.first_name == first_name
         assert officer.race == race
-        assert officer.gender == gender
+        if gender == "Not Sure":
+            assert officer.gender is None, "A form input of 'Not Sure' should lead to" \
+                " the officer's gender being saved as NULL/None."
+        else:
+            assert officer.gender == gender
 
 
 def test_ac_can_add_new_officer_with_unit_in_their_dept(mockdata, client, session):
@@ -912,7 +916,11 @@ def test_ac_can_add_new_officer_with_unit_in_their_dept(mockdata, client, sessio
             last_name=last_name).one()
         assert officer.first_name == first_name
         assert officer.race == race
-        assert officer.gender == gender
+        if gender == "Not Sure":
+            assert officer.gender is None, "A form input of 'Not Sure' should lead to" \
+                " the officer's gender being saved as NULL/None."
+        else:
+            assert officer.gender == gender
         assert Assignment.query.filter_by(baseofficer=officer, unit=unit).one()
 
 
diff --git a/OpenOversight/tests/test_commands.py b/OpenOversight/tests/test_commands.py
index 6e6b5bdb3..f866123e7 100644
--- a/OpenOversight/tests/test_commands.py
+++ b/OpenOversight/tests/test_commands.py
@@ -15,6 +15,7 @@
     add_job_title,
     bulk_add_officers,
     advanced_csv_import,
+    create_officer_from_row,
 )
 from OpenOversight.app.models import (
     Assignment,
@@ -702,7 +703,7 @@ def test_bulk_add_officers__no_create_flag(session, department, csv_path):
     # department with one officer
     department_id = department.id
     officer = generate_officer()
-    officer.gender = "Not Sure"
+    officer.gender = None
     officer.department = department
     session.add(officer)
     session.commit()
@@ -1211,3 +1212,30 @@ def test_advanced_csv_import__unit_other_department(
     # command fails because the unit does not belong to the department
     assert result.exception is not None
     assert result.exit_code != 0
+
+
+def test_create_officer_from_row_adds_new_officer_and_normalizes_gender(app, session):
+    with app.app_context():
+        department = Department(name="Cityname Police Department", short_name="CNPD")
+        session.add(department)
+        session.commit()
+        lookup_officer = Officer.query.filter_by(
+            first_name="NewOfficerFromRow").one_or_none()
+        assert lookup_officer is None
+
+        row = {
+            "gender": "Female",
+            "first_name": "NewOfficerFromRow",
+            "last_name": "Jones",
+            "employment_date": "1980-12-01",
+            "unique_internal_identifier": "officer-jones-unique-id",
+        }
+        create_officer_from_row(row, department.id)
+
+        lookup_officer = Officer.query.filter_by(
+            first_name="NewOfficerFromRow").one_or_none()
+
+        # Was an officer created in the database?
+        assert lookup_officer is not None
+        # Was the gender properly normalized?
+        assert lookup_officer.gender == "F"
diff --git a/OpenOversight/tests/test_utils.py b/OpenOversight/tests/test_utils.py
index 64dc3e7fe..3d7d4450d 100644
--- a/OpenOversight/tests/test_utils.py
+++ b/OpenOversight/tests/test_utils.py
@@ -31,13 +31,31 @@ def test_race_filter_select_all_black_officers(mockdata):
         assert element.race in ('BLACK', 'Not Sure')
 
 
-def test_gender_filter_select_all_male_officers(mockdata):
+def test_gender_filter_select_all_male_or_not_sure_officers(mockdata):
     department = OpenOversight.app.models.Department.query.first()
     results = OpenOversight.app.utils.grab_officers(
         {'gender': ['M'], 'dept': department}
     )
+
+    result_genders = [officer.gender for officer in results.all()]
     for element in results.all():
-        assert element.gender in ('M', 'Not Sure')
+        assert element.gender in ('M', None)
+    for gender in ('M', None):
+        assert gender in result_genders
+
+
+def test_gender_filter_include_all_genders_if_not_sure(mockdata):
+    department = OpenOversight.app.models.Department.query.first()
+    results = OpenOversight.app.utils.grab_officers(
+        {'gender': ['Not Sure'], 'dept': department}
+    )
+
+    result_genders = [officer.gender for officer in results.all()]
+    for gender in ('M', 'F', 'Other', None):
+        assert gender in result_genders
+    for officer in results.all():
+        assert officer.department == department
+    assert results.count() == len(department.officers)
 
 
 def test_rank_filter_select_all_commanders(mockdata):

From 1401c6308de5cb0c4e893dc0917694c3108a1eb4 Mon Sep 17 00:00:00 2001
From: abandoned-prototype
 <41744410+abandoned-prototype@users.noreply.github.com>
Date: Mon, 7 Dec 2020 01:31:53 -0600
Subject: [PATCH 06/15] change migration to use LOWER not citext (#855)

---
 .../versions/cd39b33b5360_constrain_officer_gender_options.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/OpenOversight/migrations/versions/cd39b33b5360_constrain_officer_gender_options.py b/OpenOversight/migrations/versions/cd39b33b5360_constrain_officer_gender_options.py
index f3639df70..fbce9e7eb 100644
--- a/OpenOversight/migrations/versions/cd39b33b5360_constrain_officer_gender_options.py
+++ b/OpenOversight/migrations/versions/cd39b33b5360_constrain_officer_gender_options.py
@@ -20,7 +20,7 @@ def get_update_statement(normalized, options):
     template = """
     UPDATE officers
     SET gender = '{normalized}'
-    WHERE gender::citext in ({options});
+    WHERE LOWER(gender) in ({options});
     """
     options = ', '.join(["'" + o + "'" for o in options])
     return template.format(normalized=normalized, options=options)
@@ -28,8 +28,6 @@ def get_update_statement(normalized, options):
 
 def upgrade():
     conn = op.get_bind()
-    create_extn = "create extension IF NOT EXISTS citext;"
-    conn.execute(create_extn)
 
     genders = {
         "M": ('male', 'm', 'man'),

From 5fd10622912fb98c08e6206eec64ba78cae64b89 Mon Sep 17 00:00:00 2001
From: abandoned-prototype
 <41744410+abandoned-prototype@users.noreply.github.com>
Date: Sun, 28 Feb 2021 15:46:00 -0600
Subject: [PATCH 07/15] simplify and streamline officer filtering (#858)

---
 OpenOversight/app/config.py                   |  1 +
 OpenOversight/app/main/views.py               | 19 ++--
 OpenOversight/app/models.py                   | 20 ++---
 OpenOversight/app/templates/list_officer.html |  2 +-
 .../app/templates/tagger_gallery.html         |  4 +-
 OpenOversight/app/utils.py                    | 88 +++++++++----------
 .../routes/test_officer_and_department.py     | 12 +--
 OpenOversight/tests/test_utils.py             |  4 +-
 8 files changed, 77 insertions(+), 73 deletions(-)

diff --git a/OpenOversight/app/config.py b/OpenOversight/app/config.py
index 1e2077b20..8e2636fb5 100644
--- a/OpenOversight/app/config.py
+++ b/OpenOversight/app/config.py
@@ -50,6 +50,7 @@ def init_app(app):
 
 class DevelopmentConfig(BaseConfig):
     DEBUG = True
+    SQLALCHEMY_ECHO = True
     SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI')
     NUM_OFFICERS = 15000
     SITEMAP_URL_SCHEME = 'http'
diff --git a/OpenOversight/app/main/views.py b/OpenOversight/app/main/views.py
index 43fdf0e05..90d95eca6 100644
--- a/OpenOversight/app/main/views.py
+++ b/OpenOversight/app/main/views.py
@@ -5,6 +5,7 @@
 import re
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
+from sqlalchemy.orm import selectinload
 import sys
 from traceback import format_exc
 
@@ -540,15 +541,23 @@ def list_officer(department_id, page=1, race=[], gender=[], rank=[], min_age='16
         form_data['gender'] = request.args.getlist('gender')
 
     unit_choices = [(unit.id, unit.descrip) for unit in Unit.query.filter_by(department_id=department_id).order_by(Unit.descrip.asc()).all()]
-    rank_choices = [jc[0] for jc in db.session.query(Job.job_title, Job.order).filter_by(department_id=department_id, is_sworn_officer=True).order_by(Job.order).all()]
+    rank_choices = [jc[0] for jc in db.session.query(Job.job_title, Job.order).filter_by(department_id=department_id).order_by(Job.order).all()]
     if request.args.get('rank') and all(rank in rank_choices for rank in request.args.getlist('rank')):
         form_data['rank'] = request.args.getlist('rank')
 
-    officers = filter_by_form(form_data, Officer.query, department_id).filter(Officer.department_id == department_id).order_by(Officer.last_name, Officer.first_name, Officer.id).paginate(page, OFFICERS_PER_PAGE, False)
+    officers = filter_by_form(
+        form_data, Officer.query, department_id
+    ).filter(Officer.department_id == department_id)
+    officers = officers.options(selectinload(Officer.face))
+    officers = officers.order_by(Officer.last_name, Officer.first_name, Officer.id)
+    officers = officers.paginate(page, OFFICERS_PER_PAGE, False)
     for officer in officers.items:
-        officer_face = officer.face.order_by(Face.featured.desc()).first()
-        if officer_face:
-            officer.image = officer_face.image.filepath
+        officer_face = sorted(officer.face, key=lambda x: x.featured)
+
+        # could do some extra work to not lazy load images but load them all together
+        # but we would want to ensure to only load the first picture of each officer
+        if officer_face and officer_face[0].image:
+            officer.image = officer_face[0].image.filepath
 
     choices = {
         'race': RACE_CHOICES,
diff --git a/OpenOversight/app/models.py b/OpenOversight/app/models.py
index 67ca37c18..3b1731530 100644
--- a/OpenOversight/app/models.py
+++ b/OpenOversight/app/models.py
@@ -1,4 +1,5 @@
 import re
+from datetime import date
 
 from flask_sqlalchemy import SQLAlchemy
 from flask_sqlalchemy.model import DefaultMeta
@@ -103,15 +104,14 @@ class Officer(BaseModel):
     birth_year = db.Column(db.Integer, index=True, unique=False, nullable=True)
     assignments = db.relationship('Assignment', backref='officer', lazy='dynamic')
     assignments_lazy = db.relationship('Assignment')
-    face = db.relationship('Face', backref='officer', lazy='dynamic')
+    face = db.relationship('Face', backref='officer')
     department_id = db.Column(db.Integer, db.ForeignKey('departments.id'))
     department = db.relationship('Department', backref='officers')
     unique_internal_identifier = db.Column(db.String(50), index=True, unique=True, nullable=True)
-    # we don't expect to pull up officers via link often so we make it lazy.
+
     links = db.relationship(
         'Link',
         secondary=officer_links,
-        lazy='subquery',
         backref=db.backref('officers', lazy=True))
     notes = db.relationship('Note', back_populates='officer', order_by='Note.date_created')
     descriptions = db.relationship('Description', back_populates='officer', order_by='Description.date_created')
@@ -147,18 +147,12 @@ def gender_label(self):
                 return label
 
     def job_title(self):
-        if self.assignments.all():
-            return self.assignments\
-                .order_by(self.assignments[0].__table__.c.star_date.desc())\
-                .first()\
-                .job.job_title
+        if self.assignments_lazy:
+            return max(self.assignments_lazy, key=lambda x: x.star_date or date.min).job.job_title
 
     def badge_number(self):
-        if self.assignments.all():
-            return self.assignments\
-                .order_by(self.assignments[0].__table__.c.star_date.desc())\
-                .first()\
-                .star_no
+        if self.assignments_lazy:
+            return max(self.assignments_lazy, key=lambda x: x.star_date or date.min).star_no
 
     def __repr__(self):
         if self.unique_internal_identifier:
diff --git a/OpenOversight/app/templates/list_officer.html b/OpenOversight/app/templates/list_officer.html
index cbcbf039b..5d11e1a28 100644
--- a/OpenOversight/app/templates/list_officer.html
+++ b/OpenOversight/app/templates/list_officer.html
@@ -163,7 +163,7 @@ <h2>
                       <dt>Gender</dt>
                       <dd>{{ officer.gender_label()|default('Unknown') }}</dd>
                       <dt>Number of Photos</dt>
-                      <dd>{{ officer.face.count() }}</dd>
+                      <dd>{{ officer.face | count }}</dd>
                     </dl>
                   </div>
                 </div>
diff --git a/OpenOversight/app/templates/tagger_gallery.html b/OpenOversight/app/templates/tagger_gallery.html
index c98b2475d..76136b522 100644
--- a/OpenOversight/app/templates/tagger_gallery.html
+++ b/OpenOversight/app/templates/tagger_gallery.html
@@ -23,10 +23,10 @@ <h1 class="page-header">Matching Officers</h1>
         <div class="carousel-inner" role="listbox">
         {% for officer in officers.items %}
 
-        {% if officer.face.first() is none %}
+        {% if officer.face | count == 0 %}
           {% set officer_image = '/static/images/placeholder.png' %}
         {% else %}
-          {% set officer_image = officer.face.first().image.filepath %}
+          {% set officer_image = officer.face[0].image.filepath %}
         {% endif %}
 
         {% if loop.index == 1 %}
diff --git a/OpenOversight/app/utils.py b/OpenOversight/app/utils.py
index 524cb99b1..20ec709f8 100644
--- a/OpenOversight/app/utils.py
+++ b/OpenOversight/app/utils.py
@@ -18,6 +18,7 @@
 
 from sqlalchemy import func, or_
 from sqlalchemy.sql.expression import cast
+from sqlalchemy.orm import selectinload
 import imghdr as imghdr
 from flask import current_app, url_for
 from flask_login import current_user
@@ -251,66 +252,65 @@ def upload_obj_to_s3(file_obj, dest_filename):
     return url
 
 
-def filter_by_form(form, officer_query, department_id=None):
-    # Some SQL acrobatics to left join only the most recent assignment per officer
-    row_num_col = func.row_number().over(
-        partition_by=Assignment.officer_id, order_by=Assignment.star_date.desc()
-    ).label('row_num')
-    subq = db.session.query(
-        Assignment.officer_id,
-        Assignment.job_id,
-        Assignment.star_date,
-        Assignment.star_no,
-        Assignment.unit_id
-    ).add_columns(row_num_col).from_self().filter(row_num_col == 1).subquery()
-    officer_query = officer_query.outerjoin(subq)
-
-    if form.get('name'):
+def filter_by_form(form_data, officer_query, department_id=None):
+    if form_data.get('name'):
         officer_query = officer_query.filter(
-            Officer.last_name.ilike('%%{}%%'.format(form['name']))
+            Officer.last_name.ilike('%%{}%%'.format(form_data['name']))
         )
-    if not department_id and form.get('dept'):
-        department_id = form['dept'].id
+    if not department_id and form_data.get('dept'):
+        department_id = form_data['dept'].id
         officer_query = officer_query.filter(
             Officer.department_id == department_id
         )
-    if form.get('badge'):
-        officer_query = officer_query.filter(
-            subq.c.assignments_star_no.like('%%{}%%'.format(form['badge']))
-        )
-    if form.get('unit'):
-        officer_query = officer_query.filter(
-            subq.c.assignments_unit_id == form['unit']
-        )
 
-    if form.get('unique_internal_identifier'):
+    if form_data.get('unique_internal_identifier'):
         officer_query = officer_query.filter(
-            Officer.unique_internal_identifier.ilike('%%{}%%'.format(form['unique_internal_identifier']))
+            Officer.unique_internal_identifier.ilike('%%{}%%'.format(form_data['unique_internal_identifier']))
         )
+
     race_values = [x for x, _ in RACE_CHOICES]
-    if form.get('race') and all(race in race_values for race in form['race']):
-        if 'Not Sure' in form['race']:
-            form['race'].append(None)
-        officer_query = officer_query.filter(Officer.race.in_(form['race']))
+    if form_data.get('race') and all(race in race_values for race in form_data['race']):
+        if 'Not Sure' in form_data['race']:
+            form_data['race'].append(None)
+        officer_query = officer_query.filter(Officer.race.in_(form_data['race']))
+
     gender_values = [x for x, _ in GENDER_CHOICES]
-    if form.get('gender') and all(gender in gender_values for gender in form['gender']):
-        if 'Not Sure' not in form['gender']:
-            officer_query = officer_query.filter(or_(Officer.gender.in_(form['gender']), Officer.gender.is_(None)))
+    if form_data.get('gender') and all(gender in gender_values for gender in form_data['gender']):
+        if 'Not Sure' not in form_data['gender']:
+            officer_query = officer_query.filter(or_(Officer.gender.in_(form_data['gender']), Officer.gender.is_(None)))
 
-    if form.get('min_age') and form.get('max_age'):
+    if form_data.get('min_age') and form_data.get('max_age'):
         current_year = datetime.datetime.now().year
-        min_birth_year = current_year - int(form['min_age'])
-        max_birth_year = current_year - int(form['max_age'])
+        min_birth_year = current_year - int(form_data['min_age'])
+        max_birth_year = current_year - int(form_data['max_age'])
         officer_query = officer_query.filter(db.or_(db.and_(Officer.birth_year <= min_birth_year,
                                                             Officer.birth_year >= max_birth_year),
                                                     Officer.birth_year == None))  # noqa
 
-    officer_query = officer_query.outerjoin(Job, Assignment.job)
-    rank_values = [x[0] for x in db.session.query(Job.job_title).filter_by(department_id=department_id, is_sworn_officer=True).all()]
-    if form.get('rank') and all(rank in rank_values for rank in form['rank']):
-        if 'Not Sure' in form['rank']:
-            form['rank'].append(None)
-        officer_query = officer_query.filter(Job.job_title.in_(form['rank']))
+    job_ids = []
+    if form_data.get('rank'):
+        job_ids = [job.id for job in Job.query.filter_by(department_id=department_id).filter(Job.job_title.in_(form_data.get("rank"))).all()]
+
+        print(form_data)
+        if 'Not Sure' in form_data['rank']:
+            form_data['rank'].append(None)
+
+    if form_data.get('badge') or form_data.get('unit') or job_ids:
+        officer_query = officer_query.join(Officer.assignments)
+        if form_data.get('badge'):
+            officer_query = officer_query.filter(
+                Assignment.star_no.like('%%{}%%'.format(form_data['badge']))
+            )
+        if form_data.get('unit'):
+            officer_query = officer_query.filter(
+                Assignment.unit_id == form_data['unit']
+            )
+        if job_ids:
+            officer_query = officer_query.filter(Assignment.job_id.in_(job_ids))
+    officer_query = (
+        officer_query
+        .options(selectinload(Officer.assignments_lazy))
+    )
 
     return officer_query
 
diff --git a/OpenOversight/tests/routes/test_officer_and_department.py b/OpenOversight/tests/routes/test_officer_and_department.py
index 714ef00c4..c9bf05a46 100644
--- a/OpenOversight/tests/routes/test_officer_and_department.py
+++ b/OpenOversight/tests/routes/test_officer_and_department.py
@@ -1527,7 +1527,7 @@ def test_admin_can_upload_photos_of_dept_officers(mockdata, client, session, tes
 
         department = Department.query.filter_by(id=AC_DEPT).first()
         officer = department.officers[3]
-        officer_face_count = officer.face.count()
+        officer_face_count = len(officer.face)
 
         crop_mock = MagicMock(return_value=Image.query.first())
         upload_mock = MagicMock(return_value=Image.query.first())
@@ -1541,7 +1541,7 @@ def test_admin_can_upload_photos_of_dept_officers(mockdata, client, session, tes
                 assert rv.status_code == 200
                 assert b'Success' in rv.data
                 # check that Face was added to database
-                assert officer.face.count() == officer_face_count + 1
+                assert len(officer.face) == officer_face_count + 1
 
 
 def test_upload_photo_sends_500_on_s3_error(mockdata, client, session, test_png_BytesIO):
@@ -1553,7 +1553,7 @@ def test_upload_photo_sends_500_on_s3_error(mockdata, client, session, test_png_
         department = Department.query.filter_by(id=AC_DEPT).first()
         mock = MagicMock(return_value=None)
         officer = department.officers[0]
-        officer_face_count = officer.face.count()
+        officer_face_count = len(officer.face)
         with patch('OpenOversight.app.main.views.upload_image_to_s3_and_store_in_db', mock):
             rv = client.post(
                 url_for('main.upload', department_id=department.id, officer_id=officer.id),
@@ -1563,7 +1563,7 @@ def test_upload_photo_sends_500_on_s3_error(mockdata, client, session, test_png_
             assert rv.status_code == 500
             assert b'error' in rv.data
             # check that Face was not added to database
-            assert officer.face.count() == officer_face_count
+            assert len(officer.face) == officer_face_count
 
 
 def test_upload_photo_sends_415_for_bad_file_type(mockdata, client, session):
@@ -1604,7 +1604,7 @@ def test_ac_can_upload_photos_of_dept_officers(mockdata, client, session, test_p
         data = dict(file=(test_png_BytesIO, '204Cat.png'),)
         department = Department.query.filter_by(id=AC_DEPT).first()
         officer = department.officers[4]
-        officer_face_count = officer.face.count()
+        officer_face_count = len(officer.face)
 
         crop_mock = MagicMock(return_value=Image.query.first())
         upload_mock = MagicMock(return_value=Image.query.first())
@@ -1618,7 +1618,7 @@ def test_ac_can_upload_photos_of_dept_officers(mockdata, client, session, test_p
                 assert rv.status_code == 200
                 assert b'Success' in rv.data
                 # check that Face was added to database
-                assert officer.face.count() == officer_face_count + 1
+                assert len(officer.face) == officer_face_count + 1
 
 
 def test_edit_officers_with_blank_uids(mockdata, client, session):
diff --git a/OpenOversight/tests/test_utils.py b/OpenOversight/tests/test_utils.py
index 3d7d4450d..a7d8dfabe 100644
--- a/OpenOversight/tests/test_utils.py
+++ b/OpenOversight/tests/test_utils.py
@@ -110,7 +110,7 @@ def test_filter_by_full_unique_internal_identifier_returns_officers(mockdata):
     department = OpenOversight.app.models.Department.query.first()
     target_unique_internal_id = OpenOversight.app.models.Officer.query.first().unique_internal_identifier
     results = OpenOversight.app.utils.grab_officers(
-        {'race': 'Not Sure', 'gender': 'Not Sure', 'rank': 'Not Sure',
+        {'race': ['Not Sure'], 'gender': ['Not Sure'], 'rank': ['Not Sure'],
          'min_age': 16, 'max_age': 85, 'name': '', 'badge': '',
          'dept': department, 'unique_internal_identifier': target_unique_internal_id}
     )
@@ -124,7 +124,7 @@ def test_filter_by_partial_unique_internal_identifier_returns_officers(mockdata)
     identifier = OpenOversight.app.models.Officer.query.first().unique_internal_identifier
     partial_identifier = identifier[:len(identifier) // 2]
     results = OpenOversight.app.utils.grab_officers(
-        {'race': 'Not Sure', 'gender': 'Not Sure', 'rank': 'Not Sure',
+        {'race': ['Not Sure'], 'gender': ['Not Sure'], 'rank': ['Not Sure'],
          'min_age': 16, 'max_age': 85, 'name': '', 'badge': '',
          'dept': department, 'unique_internal_identifier': partial_identifier}
     )

From 462a3e9358a6e98a7d08d2c70e0af7c89ceaa1d6 Mon Sep 17 00:00:00 2001
From: Madison Swain-Bowden <bowdenm@spu.edu>
Date: Sun, 14 Mar 2021 15:02:49 -0700
Subject: [PATCH 08/15] Update deployment docs to use JSON for CORS policy
 (#860)

---
 DEPLOY.md | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/DEPLOY.md b/DEPLOY.md
index 41d5cbfe2..86759b676 100644
--- a/DEPLOY.md
+++ b/DEPLOY.md
@@ -38,14 +38,20 @@ You'll need to create an AWS account, if you don't already have one. Then, you'l
 For the officer identification UI to work, you'll need to create a CORS policy for the S3 bucket used with OpenOversight. In the AWS UI, this is done by navigating to the listing of buckets, clicking on the name of your bucket, and choosing the Permissions tab, and then "CORS configuration". Since we're not doing anything fancier than making a web browser GET it, we can just use the default policy:
 
 ```
-<CORSConfiguration>
-	<CORSRule>
-		<AllowedOrigin>*</AllowedOrigin>
-		<AllowedMethod>GET</AllowedMethod>
-		<MaxAgeSeconds>3000</MaxAgeSeconds>
-		<AllowedHeader>Authorization</AllowedHeader>
-	</CORSRule>
-</CORSConfiguration>
+[
+    {
+        "AllowedOrigins": [
+            "*"
+        ],
+        "AllowedMethods": [
+            "GET"
+        ],
+        "MaxAgeSeconds": 3000,
+        "AllowedHeaders": [
+            "Authorizations"
+        ]
+    }
+]
 ```
 
 If you don't click "Save" on that policy, however, the policy will not actually be applied.

From 7630c366c64dbde6b166145baff0394e24e15cc9 Mon Sep 17 00:00:00 2001
From: abandoned-prototype
 <41744410+abandoned-prototype@users.noreply.github.com>
Date: Tue, 4 May 2021 22:48:20 -0500
Subject: [PATCH 09/15] initial github action

---
 .github/workflows/main.yml | 48 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 .github/workflows/main.yml

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
new file mode 100644
index 000000000..a1505f01e
--- /dev/null
+++ b/.github/workflows/main.yml
@@ -0,0 +1,48 @@
+name: CI
+
+# Controls when the action will run. 
+on:
+  # Triggers the workflow on push or pull request events but only for the develop branch
+  push:
+    branches: 
+    - develop
+    - main
+  pull_request:
+    branches:
+    - develop
+    - main
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+    env:
+      FLASK_APP: OpenOversight.app
+    strategy:
+      matrix:
+        python-version: [3.5, 3.6, 3.7, 3.8]
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt -r dev-requirements.txt
+
+      # Run flake8
+      - name: Run flake8
+        run: flake8 --ignore=E501,E722,W504,W503
+      # Runs tests
+      - name: Run tests
+        run: FLASK_ENV=testing pytest --doctest-modules -n 4 --dist=loadfile -v OpenOversight/tests/ OpenOversight/app;

From 25aeae3f6e2c0baca49514a25a04e24449dd8cb7 Mon Sep 17 00:00:00 2001
From: abandoned-prototype
 <41744410+abandoned-prototype@users.noreply.github.com>
Date: Tue, 4 May 2021 22:58:13 -0500
Subject: [PATCH 10/15] use FLASK_ENV to set default environment

---
 OpenOversight/app/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/OpenOversight/app/config.py b/OpenOversight/app/config.py
index 8e2636fb5..b5daf90c8 100644
--- a/OpenOversight/app/config.py
+++ b/OpenOversight/app/config.py
@@ -78,5 +78,5 @@ def init_app(cls, app):  # pragma: no cover
     'development': DevelopmentConfig,
     'testing': TestingConfig,
     'production': ProductionConfig,
-    'default': DevelopmentConfig
 }
+config['default'] = config.get(os.environ.get('FLASK_ENV'), DevelopmentConfig)

From 9c057fa46c22cda0f11a145ef9fd9b001f9e45ca Mon Sep 17 00:00:00 2001
From: abandoned-prototype
 <41744410+abandoned-prototype@users.noreply.github.com>
Date: Thu, 20 May 2021 22:01:50 -0500
Subject: [PATCH 11/15] Adhoc/security patch (#873)

* csrf fixes

* fix tests and formatting
---
 OpenOversight/app/__init__.py                 |   3 +
 OpenOversight/app/auth/forms.py               |   6 +-
 OpenOversight/app/auth/views.py               | 199 ++++++------------
 OpenOversight/app/config.py                   |   2 +-
 OpenOversight/app/main/views.py               |  29 +--
 .../auth/email/new_confirmation.html          |   4 +-
 .../templates/auth/email/new_confirmation.txt |   2 +-
 .../auth/email/new_registration.html          |   4 +-
 .../templates/auth/email/new_registration.txt |   2 +-
 OpenOversight/app/templates/auth/user.html    |   2 +-
 .../app/templates/auth/user_delete.html       |   3 +-
 OpenOversight/app/templates/auth/users.html   |  23 +-
 .../app/templates/description_delete.html     |   1 +
 OpenOversight/app/templates/image.html        |   2 +
 .../app/templates/incident_delete.html        |   1 +
 OpenOversight/app/templates/link_delete.html  |   1 +
 OpenOversight/app/templates/note_delete.html  |   1 +
 OpenOversight/app/templates/profile.html      |  12 +-
 OpenOversight/app/templates/sort.html         |   2 +
 OpenOversight/tests/conftest.py               |   5 +-
 OpenOversight/tests/routes/route_helpers.py   |   8 +-
 .../routes/test_officer_and_department.py     |   2 +-
 OpenOversight/tests/routes/test_other.py      |  52 +----
 OpenOversight/tests/routes/test_user_api.py   | 135 ++++++++----
 24 files changed, 214 insertions(+), 287 deletions(-)

diff --git a/OpenOversight/app/__init__.py b/OpenOversight/app/__init__.py
index 2ae49edb0..b609d5cb3 100644
--- a/OpenOversight/app/__init__.py
+++ b/OpenOversight/app/__init__.py
@@ -11,6 +11,7 @@
 from flask_mail import Mail
 from flask_migrate import Migrate
 from flask_sitemap import Sitemap
+from flask_wtf.csrf import CSRFProtect
 
 from .config import config
 
@@ -26,6 +27,7 @@
                   default_limits=["100 per minute", "5 per second"])
 
 sitemap = Sitemap()
+csrf = CSRFProtect()
 
 
 def create_app(config_name='default'):
@@ -40,6 +42,7 @@ def create_app(config_name='default'):
     login_manager.init_app(app)
     limiter.init_app(app)
     sitemap.init_app(app)
+    csrf.init_app(app)
 
     from .main import main as main_blueprint  # noqa
     app.register_blueprint(main_blueprint)
diff --git a/OpenOversight/app/auth/forms.py b/OpenOversight/app/auth/forms.py
index 5a5b2f071..df006c5b3 100644
--- a/OpenOversight/app/auth/forms.py
+++ b/OpenOversight/app/auth/forms.py
@@ -88,7 +88,11 @@ class EditUserForm(Form):
     ac_department = QuerySelectField('Department', validators=[Optional()],
                                      query_factory=dept_choices, get_label='name', allow_blank=True)
     is_administrator = BooleanField('Is administrator?', false_values={'False', 'false', ''})
-    submit = SubmitField(label='Update')
+    is_disabled = BooleanField('Disabled?', false_values={'False', 'false', ''})
+    approved = BooleanField('Approved?', false_values={'False', 'false', ''})
+    submit = SubmitField(label='Update', false_values={'False', 'false', ''})
+    resend = SubmitField(label='Resend', false_values={'False', 'false', ''})
+    delete = SubmitField(label='Delete', false_values={'False', 'false', ''})
 
     def validate(self):
         success = super(EditUserForm, self).validate()
diff --git a/OpenOversight/app/auth/views.py b/OpenOversight/app/auth/views.py
index feb9393d1..d2102c15e 100644
--- a/OpenOversight/app/auth/views.py
+++ b/OpenOversight/app/auth/views.py
@@ -1,7 +1,4 @@
-from future.utils import iteritems
-
 from flask import render_template, redirect, request, url_for, flash, current_app
-from flask.views import MethodView
 from flask_login import login_user, logout_user, login_required, \
     current_user
 from . import auth
@@ -101,7 +98,7 @@ def register():
     return render_template('auth/register.html', form=form, jsloads=jsloads)
 
 
-@auth.route('/confirm/<token>')
+@auth.route('/confirm/<token>', methods=['GET'])
 @login_required
 def confirm(token):
     if current_user.confirmed:
@@ -237,144 +234,76 @@ def change_dept():
     return render_template('auth/change_dept_pref.html', form=form)
 
 
-class UserAPI(MethodView):
-    decorators = [admin_required]
-
-    def get(self, user_id):
-        # isolate the last part of the url
-        end_of_url = request.url.split('/')[-1].split('?')[0]
+@auth.route('/users/', methods=['GET'])
+@admin_required
+def get_users():
+    if request.args.get('page'):
+        page = int(request.args.get('page'))
+    else:
+        page = 1
+    USERS_PER_PAGE = int(current_app.config['USERS_PER_PAGE'])
+    users = User.query.order_by(User.username) \
+        .paginate(page, USERS_PER_PAGE, False)
 
-        if user_id is None:
-            if request.args.get('page'):
-                page = int(request.args.get('page'))
-            else:
-                page = 1
-            USERS_PER_PAGE = int(current_app.config['USERS_PER_PAGE'])
-            users = User.query.order_by(User.email) \
-                .paginate(page, USERS_PER_PAGE, False)
+    return render_template('auth/users.html', objects=users)
 
-            return render_template('auth/users.html', objects=users)
-        else:
-            user = User.query.get(user_id)
-            if not user:
-                return render_template('403.html'), 403
-
-            actions = ['delete', 'enable', 'disable', 'resend', 'approve']
-            if end_of_url in actions:
-                action = getattr(self, end_of_url, None)
-                return action(user)
-            else:
-                form = EditUserForm(
-                    email=user.email,
-                    is_area_coordinator=user.is_area_coordinator,
-                    ac_department=user.ac_department,
-                    is_administrator=user.is_administrator)
-                return render_template('auth/user.html', user=user, form=form)
 
-    def post(self, user_id):
-        # isolate the last part of the url
-        end_of_url = request.url.split('/')[-1].split('?')[0]
+@auth.route('/users/<int:user_id>', methods=['GET', 'POST'])
+@admin_required
+def edit_user(user_id):
+    user = User.query.get(user_id)
+    if not user:
+        return render_template('404.html'), 404
 
-        user = User.query.get(user_id)
+    if request.method == 'GET':
+        form = EditUserForm(obj=user)
+        return render_template('auth/user.html', user=user, form=form)
+    elif request.method == 'POST':
         form = EditUserForm()
+        if form.delete.data:
+            # forward to confirm delete
+            return redirect(url_for('auth.delete_user', user_id=user.id))
+        elif form.resend.data:
+            return admin_resend_confirmation(user)
+        elif form.submit.data:
+            if form.validate_on_submit():
+                # prevent user from removing own admin rights (or disabling account)
+                if user.id == current_user.id:
+                    flash('You cannot edit your own account!')
+                    form = EditUserForm(obj=user)
+                    return render_template('auth/user.html', user=user, form=form)
+                form.populate_obj(user)
+                db.session.add(user)
+                db.session.commit()
+                flash('{} has been updated!'.format(user.username))
+                return redirect(url_for('auth.edit_user', user_id=user.id))
+            else:
+                flash('Invalid entry')
+                return render_template('auth/user.html', user=user, form=form)
 
-        if user and end_of_url and end_of_url == 'delete':
-            return self.delete(user)
-        elif user and form.validate_on_submit():
-            for field, data in iteritems(form.data):
-                setattr(user, field, data)
-
-            db.session.add(user)
-            db.session.commit()
-            flash('{} has been updated!'.format(user.username))
-            return redirect(url_for('auth.user_api'))
-        elif not form.validate_on_submit():
-            flash('Invalid entry')
-            return render_template('auth/user.html', user=user, form=form)
-        else:
-            return render_template('403.html'), 403
-
-    def delete(self, user):
-        if request.method == 'POST':
-            username = user.username
-            db.session.delete(user)
-            db.session.commit()
-            flash('User {} has been deleted!'.format(username))
-            return redirect(url_for('auth.user_api'))
-
-        return render_template('auth/user_delete.html', user=user)
 
-    def enable(self, user):
-        if not user.is_disabled:
-            flash('User {} is already enabled.'.format(user.username))
-        else:
-            user.is_disabled = False
-            db.session.add(user)
-            db.session.commit()
-            flash('User {} has been enabled!'.format(user.username))
-        return redirect(url_for('auth.user_api'))
+@auth.route('/users/<int:user_id>/delete', methods=['GET', 'POST'])
+@admin_required
+def delete_user(user_id):
+    user = User.query.get(user_id)
+    if not user or user.is_administrator:
+        return render_template('403.html'), 403
+    if request.method == 'POST':
+        username = user.username
+        db.session.delete(user)
+        db.session.commit()
+        flash('User {} has been deleted!'.format(username))
+        return redirect(url_for('auth.get_users'))
 
-    def disable(self, user):
-        if user.is_disabled:
-            flash('User {} is already disabled.'.format(user.username))
-        else:
-            user.is_disabled = True
-            db.session.add(user)
-            db.session.commit()
-            flash('User {} has been disabled!'.format(user.username))
-        return redirect(url_for('auth.user_api'))
+    return render_template('auth/user_delete.html', user=user)
 
-    def resend(self, user):
-        if user.confirmed:
-            flash('User {} is already confirmed.'.format(user.username))
-        else:
-            token = user.generate_confirmation_token()
-            send_email(user.email, 'Confirm Your Account',
-                       'auth/email/confirm', user=user, token=token)
-            flash('A new confirmation email has been sent to {}.'.format(user.email))
-        return redirect(url_for('auth.user_api'))
 
-    def approve(self, user):
-        if user.approved:
-            flash('User {} is already approved.'.format(user.username))
-        else:
-            user.approved = True
-            db.session.add(user)
-            db.session.commit()
-            token = user.generate_confirmation_token()
-            send_email(user.email, 'Confirm Your Account',
-                       'auth/email/confirm', user=user, token=token)
-            flash('User {} has been approved!'.format(user.username))
-        return redirect(url_for('auth.user_api'))
-
-
-user_view = UserAPI.as_view('user_api')
-auth.add_url_rule(
-    '/users/',
-    defaults={'user_id': None},
-    view_func=user_view,
-    methods=['GET'])
-auth.add_url_rule(
-    '/users/<int:user_id>',
-    view_func=user_view,
-    methods=['GET', 'POST'])
-auth.add_url_rule(
-    '/users/<int:user_id>/delete',
-    view_func=user_view,
-    methods=['GET', 'POST'])
-auth.add_url_rule(
-    '/users/<int:user_id>/enable',
-    view_func=user_view,
-    methods=['GET'])
-auth.add_url_rule(
-    '/users/<int:user_id>/disable',
-    view_func=user_view,
-    methods=['GET'])
-auth.add_url_rule(
-    '/users/<int:user_id>/resend',
-    view_func=user_view,
-    methods=['GET'])
-auth.add_url_rule(
-    '/users/<int:user_id>/approve',
-    view_func=user_view,
-    methods=['GET'])
+def admin_resend_confirmation(user):
+    if user.confirmed:
+        flash('User {} is already confirmed.'.format(user.username))
+    else:
+        token = user.generate_confirmation_token()
+        send_email(user.email, 'Confirm Your Account',
+                   'auth/email/confirm', user=user, token=token)
+        flash('A new confirmation email has been sent to {}.'.format(user.email))
+    return redirect(url_for('auth.get_users'))
diff --git a/OpenOversight/app/config.py b/OpenOversight/app/config.py
index b5daf90c8..ad62a2971 100644
--- a/OpenOversight/app/config.py
+++ b/OpenOversight/app/config.py
@@ -79,4 +79,4 @@ def init_app(cls, app):  # pragma: no cover
     'testing': TestingConfig,
     'production': ProductionConfig,
 }
-config['default'] = config.get(os.environ.get('FLASK_ENV'), DevelopmentConfig)
+config['default'] = config.get(os.environ.get('FLASK_ENV', ""), DevelopmentConfig)
diff --git a/OpenOversight/app/main/views.py b/OpenOversight/app/main/views.py
index 90d95eca6..4457cc915 100644
--- a/OpenOversight/app/main/views.py
+++ b/OpenOversight/app/main/views.py
@@ -10,7 +10,7 @@
 from traceback import format_exc
 
 from flask import (abort, render_template, request, redirect, url_for,
-                   flash, current_app, jsonify, Response, Markup)
+                   flash, current_app, jsonify, Response)
 from flask_login import current_user, login_required, login_user
 
 from . import main
@@ -331,23 +331,6 @@ def edit_salary(officer_id, salary_id):
     return render_template('add_edit_salary.html', form=form, update=True)
 
 
-@main.route('/user/toggle/<int:uid>', methods=['POST'])
-@login_required
-@admin_required
-def toggle_user(uid):
-    try:
-        user = User.query.filter_by(id=uid).one()
-        if user.is_disabled:
-            user.is_disabled = False
-        elif not user.is_disabled:
-            user.is_disabled = True
-        db.session.commit()
-        flash('Updated user status')
-    except NoResultFound:
-        flash('Unknown error occurred')
-    return redirect(url_for('main.profile', username=user.username))
-
-
 @main.route('/image/<int:image_id>')
 @login_required
 def display_submission(image_id):
@@ -376,6 +359,9 @@ def display_tag(tag_id):
 def classify_submission(image_id, contains_cops):
     try:
         image = Image.query.filter_by(id=image_id).one()
+        if image.contains_cops is not None and not current_user.is_administrator:
+            flash('Only administrator can re-classify image')
+            return redirect(redirect_url())
         image.user_id = current_user.get_id()
         if contains_cops == 1:
             image.contains_cops = True
@@ -473,9 +459,7 @@ def edit_department(department_id):
                         if Assignment.query.filter(Assignment.job_id.in_([rank.id])).count() != 0:
                             failed_deletions.append(rank)
                     for rank in failed_deletions:
-                        formatted_rank = rank.job_title.replace(" ", "+")
-                        link = '/department/{}?name=&badge=&unique_internal_identifier=&rank={}&min_age=16&max_age=100&submit=Submit'.format(department_id, formatted_rank)
-                        flash(Markup('You attempted to delete a rank, {}, that is in use by <a href={}>the linked officers</a>.'.format(rank, link)))
+                        flash('You attempted to delete a rank, {}, that is still in use'.format(rank))
                     return redirect(url_for('main.edit_department', department_id=department_id))
 
             for (new_rank, order) in new_ranks:
@@ -782,6 +766,9 @@ def label_data(department_id=None, image_id=None):
             image = get_random_image(image_query)
 
     if image:
+        if image.is_tagged and not current_user.is_administrator:
+            flash('This image cannot be tagged anymore')
+            return redirect(url_for('main.label_data'))
         proper_path = serve_image(image.filepath)
     else:
         proper_path = None
diff --git a/OpenOversight/app/templates/auth/email/new_confirmation.html b/OpenOversight/app/templates/auth/email/new_confirmation.html
index cdf4fd245..3397904fa 100644
--- a/OpenOversight/app/templates/auth/email/new_confirmation.html
+++ b/OpenOversight/app/templates/auth/email/new_confirmation.html
@@ -4,9 +4,9 @@
   <li>Username: {{ user.username }}</li>
   <li>Email: {{ user.email }}</li>
 </ul>
-<p>To view or delete this user, please <a href="{{ url_for('auth.user_api', _external=True) }}">click here</a>.</p>
+<p>To view or delete this user, please <a href="{{ url_for('auth.edit_user', user_id=user.id, _external=True) }}">click here</a>.</p>
 <p>Alternatively, you can paste the following link in your browser's address bar:</p>
-<p>{{ url_for('auth.user_api', _external=True) }}</p>
+<p>{{ url_for('auth.edit_user', user_id=user.id, _external=True) }}</p>
 <p>Sincerely,</p>
 <p>The OpenOversight Team</p>
 <p><small>Note: replies to this email address are not monitored.</small></p>
diff --git a/OpenOversight/app/templates/auth/email/new_confirmation.txt b/OpenOversight/app/templates/auth/email/new_confirmation.txt
index c81db9693..a6328ac0e 100644
--- a/OpenOversight/app/templates/auth/email/new_confirmation.txt
+++ b/OpenOversight/app/templates/auth/email/new_confirmation.txt
@@ -7,7 +7,7 @@ Email: {{ user.email }}
 
 To view or delete this user, click on the following link:
 
-{{ url_for('auth.user_api', _external=True) }}
+{{ url_for('auth.edit_user', user_id=user.id, _external=True) }}
 
 Sincerely,
 
diff --git a/OpenOversight/app/templates/auth/email/new_registration.html b/OpenOversight/app/templates/auth/email/new_registration.html
index c86e42dc7..120ad3309 100644
--- a/OpenOversight/app/templates/auth/email/new_registration.html
+++ b/OpenOversight/app/templates/auth/email/new_registration.html
@@ -4,9 +4,9 @@
   <li>Username: {{ user.username }}</li>
   <li>Email: {{ user.email }}</li>
 </ul>
-<p>To approve or delete this user, please <a href="{{ url_for('auth.user_api', _external=True) }}">click here</a>.</p>
+<p>To approve or delete this user, please <a href="{{ url_for('auth.edit_user', user_id=user.id, _external=True) }}">click here</a>.</p>
 <p>Alternatively, you can paste the following link in your browser's address bar:</p>
-<p>{{ url_for('auth.user_api', _external=True) }}</p>
+<p>{{ url_for('auth.edit_user', user_id=user.id, _external=True) }}</p>
 <p>Sincerely,</p>
 <p>The OpenOversight Team</p>
 <p><small>Note: replies to this email address are not monitored.</small></p>
diff --git a/OpenOversight/app/templates/auth/email/new_registration.txt b/OpenOversight/app/templates/auth/email/new_registration.txt
index 20f8abf1f..5840085a9 100644
--- a/OpenOversight/app/templates/auth/email/new_registration.txt
+++ b/OpenOversight/app/templates/auth/email/new_registration.txt
@@ -7,7 +7,7 @@ Email: {{ user.email }}
 
 To approve or delete this user, click on the following link:
 
-{{ url_for('auth.user_api', _external=True) }}
+{{ url_for('auth.edit_user', user_id=user.id, _external=True) }}
 
 Sincerely,
 
diff --git a/OpenOversight/app/templates/auth/user.html b/OpenOversight/app/templates/auth/user.html
index 2099964c2..0d5764ab7 100644
--- a/OpenOversight/app/templates/auth/user.html
+++ b/OpenOversight/app/templates/auth/user.html
@@ -13,7 +13,7 @@ <h1>
 	</div>
 	<div class="col-md-6 col-xs-12">
 	    <form class="form" method="post" role="form">
-				{{ wtf.quick_form(form, button_map={'submit':'primary'}) }}
+				{{ wtf.quick_form(form, button_map={"submit":"primary", "delete": "danger"}) }}
 	    </form>
 	    <br>
 	</div>
diff --git a/OpenOversight/app/templates/auth/user_delete.html b/OpenOversight/app/templates/auth/user_delete.html
index afa8db1ed..9e8756a09 100644
--- a/OpenOversight/app/templates/auth/user_delete.html
+++ b/OpenOversight/app/templates/auth/user_delete.html
@@ -11,7 +11,8 @@ <h1>
 		<p class="lead">
 			Are you sure you want to delete this user?
 			This cannot be undone.
-			<form action="{{ '{}/delete'.format(url_for('auth.user_api', user_id=user.id)) }}" method="post">
+			<form action="{{ url_for('auth.delete_user', user_id=user.id) }}" method="post">
+				<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
 				<button class='btn btn-danger' type="submit">Delete</button>
 			</form>
 		</p>
diff --git a/OpenOversight/app/templates/auth/users.html b/OpenOversight/app/templates/auth/users.html
index 8ca8aca26..e32752b9a 100644
--- a/OpenOversight/app/templates/auth/users.html
+++ b/OpenOversight/app/templates/auth/users.html
@@ -10,8 +10,8 @@ <h1 class="page-header">Users</h1>
 <div class="container" role="main">
 	<div class="horizontal-padding">
 		{% with paginate=objects,
-          next_url=url_for('auth.user_api', page=objects.next_num),
-          prev_url=url_for('auth.user_api',  page=objects.prev_num),
+          next_url=url_for('auth.get_users', page=objects.next_num),
+          prev_url=url_for('auth.get_users',  page=objects.prev_num),
           location='top' %}
 	    {% include "partials/paginate_nav.html" %}
 	  {% endwith %}
@@ -29,20 +29,11 @@ <h1 class="page-header">Users</h1>
 				{% for user in objects.items %}
 				<tr id="user-{{user.id}}">
 					<td>
-						<a href="{{user.id}}"><strong>{{ user.username }}</strong></a>
+						<a href="{{ url_for('auth.edit_user', user_id=user.id)}}"><strong>{{ user.username }}</strong></a>
 						<br />
 						<div class="row-actions">
-							<span class="edit"><a href="{{user.id}}">Edit</a> | </span>
-							<span class="delete"><a href="{{user.id}}/delete">Delete</a> | </span>
-							{% if user.is_disabled %}
-							<span class="enable"><a href="{{user.id}}/enable">Enable</a></span>
-							{% elif user.confirmed %}
-							<span class="disable"><a href="{{user.id}}/disable">Disable</a></span>
-							{% elif not user.approved %}
-							<span class="approve"><a href="{{user.id}}/approve">Approve</a></span>
-							{% else %}
-							<span class="resend"><a href="{{user.id}}/resend">Re-send confirmation email</a></span>
-							{% endif %}
+							<span class="edit"><a href="{{ url_for('auth.edit_user', user_id=user.id)}}">Edit user</a> | </span>
+							<span class="profile"><a href="{{ url_for('main.profile', username=user.username)}}">Profile</a></span>
 						</div>
 					</td>
 					<td><a href="mailto:{{user.email}}">{{ user.email }}</a></td>
@@ -80,8 +71,8 @@ <h1 class="page-header">Users</h1>
 			</table>
 		</div>
 		{% with paginate=objects,
-          next_url=url_for('auth.user_api', page=objects.next_num),
-          prev_url=url_for('auth.user_api',  page=objects.prev_num),
+          next_url=url_for('auth.get_users', page=objects.next_num),
+          prev_url=url_for('auth.get_users',  page=objects.prev_num),
           location='bottom' %}
 	    {% include "partials/paginate_nav.html" %}
 	  {% endwith %}
diff --git a/OpenOversight/app/templates/description_delete.html b/OpenOversight/app/templates/description_delete.html
index 1a33ff981..1003a109a 100644
--- a/OpenOversight/app/templates/description_delete.html
+++ b/OpenOversight/app/templates/description_delete.html
@@ -15,6 +15,7 @@ <h1>
 			Are you sure you want to delete this description?
 			This cannot be undone.
 			<form action="{{ '{}/delete'.format(url_for('main.description_api', obj_id=obj.id, officer_id=obj.officer_id)) }}" method="post">
+				<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
 				<button class='btn btn-danger' type="submit">Delete</button>
 			</form>
 		</p>
diff --git a/OpenOversight/app/templates/image.html b/OpenOversight/app/templates/image.html
index 9e8d54ea1..64f777f2a 100644
--- a/OpenOversight/app/templates/image.html
+++ b/OpenOversight/app/templates/image.html
@@ -73,6 +73,7 @@ <h3>Classification</h3>
           <h3>Classify <small>Admin only</small></h3>
           <p>
             <form action="{{ url_for('main.classify_submission', image_id=image.id, contains_cops=0) }}" method="post">
+              <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
               <button type="submit" name="action" class="btn btn-primary">
                 No cops here
               </button>
@@ -80,6 +81,7 @@ <h3>Classify <small>Admin only</small></h3>
           </p>
           <p>
             <form action="{{ url_for('main.classify_submission', image_id=image.id, contains_cops=1) }}" method="post">
+              <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
               <button type="submit" name="action" class="btn btn-primary">
                 Cops here
               </button>
diff --git a/OpenOversight/app/templates/incident_delete.html b/OpenOversight/app/templates/incident_delete.html
index 38f7d1d93..c0166f7f7 100644
--- a/OpenOversight/app/templates/incident_delete.html
+++ b/OpenOversight/app/templates/incident_delete.html
@@ -12,6 +12,7 @@ <h1>
 			Are you sure you want to delete this incident?
 			This cannot be undone.
 			<form action="{{ '{}/delete'.format(url_for('main.incident_api', obj_id=obj.id)) }}" method="post">
+				<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
 				<button class='btn btn-danger' type="submit">Delete</button>
 			</form>
 		</p>
diff --git a/OpenOversight/app/templates/link_delete.html b/OpenOversight/app/templates/link_delete.html
index 4330ae4dd..c516d7fb7 100644
--- a/OpenOversight/app/templates/link_delete.html
+++ b/OpenOversight/app/templates/link_delete.html
@@ -25,6 +25,7 @@ <h1>
 			Are you sure you want to delete this link?
 			This cannot be undone.
 			<form action="{{ url_for('main.link_api_delete', obj_id=obj.id, officer_id=officer_id) }}" method="post">
+				<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
 				<button class='btn btn-danger' type="submit">Delete</button>
 			</form>
 		</p>
diff --git a/OpenOversight/app/templates/note_delete.html b/OpenOversight/app/templates/note_delete.html
index 15bc4d8ba..a4d0d85b3 100644
--- a/OpenOversight/app/templates/note_delete.html
+++ b/OpenOversight/app/templates/note_delete.html
@@ -15,6 +15,7 @@ <h1>
 			Are you sure you want to delete this note?
 			This cannot be undone.
 			<form action="{{ '{}/delete'.format(url_for('main.note_api', obj_id=obj.id, officer_id=obj.officer_id)) }}" method="post">
+				<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
 				<button class='btn btn-danger' type="submit">Delete</button>
 			</form>
 		</p>
diff --git a/OpenOversight/app/templates/profile.html b/OpenOversight/app/templates/profile.html
index 1235f9f7f..f85427f4d 100644
--- a/OpenOversight/app/templates/profile.html
+++ b/OpenOversight/app/templates/profile.html
@@ -37,14 +37,10 @@ <h3>Account Status</h3>
           {% endif %}
 
           {% if current_user.is_administrator and user.is_administrator == False %}
-          <h3>Toggle (Disable/Enable) User <small>Admin only</small></h3>
-          <p>
-            <form action="{{ url_for('main.toggle_user', uid=user.id) }}" method="post">
-              <button type="submit" name="action" class="btn btn-primary">
-                Toggle user
-              </button>
-            </form>
-          </p>
+          <h3><a href="{{ url_for('auth.edit_user', user_id=user.id) }}">Edit user</a> <small>Admin only</small></h3>
+          <!-- <p>
+            <a href="{{ url_for('auth.edit_user', user_id=user.id) }}">Change user access</a>
+          </p> -->
           {% endif %}
 
           {% if current_user.is_administrator %}
diff --git a/OpenOversight/app/templates/sort.html b/OpenOversight/app/templates/sort.html
index 3a1ce8a7c..df43826a5 100644
--- a/OpenOversight/app/templates/sort.html
+++ b/OpenOversight/app/templates/sort.html
@@ -38,6 +38,7 @@ <h1><small>Do you see uniformed law enforcement officers in the photo?</small></
         <div class="col-sm-4 text-center">
           <form action="{{ url_for('main.classify_submission', image_id=image.id, contains_cops=0) }}" method="post">
             <button type="submit" name="action" id="answer-no" class="btn btn-lg btn-danger">
+              <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
               <span class="glyphicon glyphicon glyphicon-remove" aria-hidden="true"></span>
               <u>N</u>o
             </button>
@@ -49,6 +50,7 @@ <h1><small>Do you see uniformed law enforcement officers in the photo?</small></
         </div>
         <div class="col-sm-4 text-center">
           <form action="{{ url_for('main.classify_submission', image_id=image.id, contains_cops=1) }}" method="post">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
             <button type="submit" name="action" id="answer-yes" class="btn btn-lg btn-success">
               <span class="glyphicon glyphicon glyphicon-ok" aria-hidden="true"></span>
               <u>Y</u>es!
diff --git a/OpenOversight/tests/conftest.py b/OpenOversight/tests/conftest.py
index 9edb33f74..3f23f02fd 100644
--- a/OpenOversight/tests/conftest.py
+++ b/OpenOversight/tests/conftest.py
@@ -19,6 +19,7 @@
 from OpenOversight.app import create_app, models
 from OpenOversight.app.utils import merge_dicts
 from OpenOversight.app.models import db as _db, Unit, Job, Officer
+from OpenOversight.tests.routes.route_helpers import ADMIN_EMAIL, ADMIN_PASSWORD
 
 factory = Faker()
 
@@ -340,9 +341,9 @@ def add_mockdata(session):
                             confirmed=True)
     session.add(test_user)
 
-    test_admin = models.User(email='test@example.org',
+    test_admin = models.User(email=ADMIN_EMAIL,
                              username='test_admin',
-                             password='testtest',
+                             password=ADMIN_PASSWORD,
                              confirmed=True,
                              is_administrator=True)
     session.add(test_admin)
diff --git a/OpenOversight/tests/routes/route_helpers.py b/OpenOversight/tests/routes/route_helpers.py
index dd81d2556..4c03ab99d 100644
--- a/OpenOversight/tests/routes/route_helpers.py
+++ b/OpenOversight/tests/routes/route_helpers.py
@@ -3,6 +3,9 @@
 from flask import url_for
 from OpenOversight.app.auth.forms import LoginForm
 
+ADMIN_EMAIL = "test@example.org"
+ADMIN_PASSWORD = "testtest"
+
 
 def login_user(client):
     form = LoginForm(email='jen@example.org',
@@ -17,8 +20,9 @@ def login_user(client):
 
 
 def login_admin(client):
-    form = LoginForm(email='test@example.org',
-                     password='testtest',
+
+    form = LoginForm(email=ADMIN_EMAIL,
+                     password=ADMIN_PASSWORD,
                      remember_me=True)
     rv = client.post(
         url_for('auth.login'),
diff --git a/OpenOversight/tests/routes/test_officer_and_department.py b/OpenOversight/tests/routes/test_officer_and_department.py
index c9bf05a46..e454a1e4c 100644
--- a/OpenOversight/tests/routes/test_officer_and_department.py
+++ b/OpenOversight/tests/routes/test_officer_and_department.py
@@ -567,7 +567,7 @@ def test_admin_cannot_delete_rank_in_use(mockdata, client, session):
         )
 
         updated_ranks = Department.query.filter_by(name='Springfield Police Department').one().jobs
-        assert 'You attempted to delete a rank, Commander, that is in use' in result.data.decode('utf-8')
+        assert 'You attempted to delete a rank, Commander, that is still in use' in result.data.decode('utf-8')
         assert len(updated_ranks) == len(original_ranks)
 
 
diff --git a/OpenOversight/tests/routes/test_other.py b/OpenOversight/tests/routes/test_other.py
index 822fbb3fd..ddb5d0a0a 100644
--- a/OpenOversight/tests/routes/test_other.py
+++ b/OpenOversight/tests/routes/test_other.py
@@ -1,7 +1,7 @@
 # Routing and view tests
 import pytest
 from flask import url_for, current_app
-from .route_helpers import login_user, login_admin, login_ac
+from .route_helpers import login_user
 
 
 @pytest.mark.parametrize("route", [
@@ -32,52 +32,4 @@ def test_user_can_access_profile(mockdata, client, session):
         # User email should not appear
         assert 'User Email' not in rv.data.decode('utf-8')
         # Toggle button should not appear for this non-admin user
-        assert 'Toggle (Disable/Enable) User' not in rv.data.decode('utf-8')
-
-
-def test_admin_sees_toggle_button_on_profiles(mockdata, client, session):
-    with current_app.test_request_context():
-        login_admin(client)
-
-        rv = client.get(
-            url_for('main.profile', username='test_user'),
-            follow_redirects=True
-        )
-        assert 'test_user' in rv.data.decode('utf-8')
-        # User email should appear
-        assert 'User Email' in rv.data.decode('utf-8')
-        # Admin should be able to see the Toggle button
-        assert 'Toggle (Disable/Enable) User' in rv.data.decode('utf-8')
-
-
-def test_admin_can_toggle_user(mockdata, client, session):
-    with current_app.test_request_context():
-        login_admin(client)
-
-        rv = client.post(
-            url_for('main.toggle_user', uid=1),
-            follow_redirects=True
-        )
-        assert 'Disabled' in rv.data.decode('utf-8')
-
-
-def test_ac_cannot_toggle_user(mockdata, client, session):
-    with current_app.test_request_context():
-        login_ac(client)
-
-        rv = client.post(
-            url_for('main.toggle_user', uid=1),
-            follow_redirects=True
-        )
-        assert rv.status_code == 403
-
-
-def test_user_cannot_toggle_user(mockdata, client, session):
-    with current_app.test_request_context():
-        login_user(client)
-
-        rv = client.post(
-            url_for('main.toggle_user', uid=1),
-            follow_redirects=True
-        )
-        assert rv.status_code == 403
+        assert 'Edit User' not in rv.data.decode('utf-8')
diff --git a/OpenOversight/tests/routes/test_user_api.py b/OpenOversight/tests/routes/test_user_api.py
index 4c7080fef..acfe046ca 100644
--- a/OpenOversight/tests/routes/test_user_api.py
+++ b/OpenOversight/tests/routes/test_user_api.py
@@ -2,7 +2,7 @@
 import pytest
 from flask import url_for, current_app
 from ..conftest import AC_DEPT
-from .route_helpers import login_user, login_admin, login_ac
+from .route_helpers import login_user, login_admin, login_ac, ADMIN_EMAIL
 
 
 from OpenOversight.app.auth.forms import EditUserForm, RegistrationForm, LoginForm
@@ -13,10 +13,6 @@
     ('/auth/users/', ['GET']),
     ('/auth/users/1', ['GET', 'POST']),
     ('/auth/users/1/delete', ['GET', 'POST']),
-    ('/auth/users/1/enable', ['GET']),
-    ('/auth/users/1/disable', ['GET']),
-    ('/auth/users/1/resend', ['GET']),
-    ('/auth/users/1/approve', ['GET']),
 ]
 
 
@@ -55,18 +51,6 @@ def test_ac_cannot_access_user_api(route, methods, mockdata, client, session):
             assert rv.status_code == 403
 
 
-@pytest.mark.parametrize("route,methods", routes_methods)
-def test_admin_can_access_user_api(route, methods, mockdata, client, session):
-    with current_app.test_request_context():
-        login_admin(client)
-        if 'GET' in methods:
-            rv = client.get(route)
-            assert rv.status_code != 403
-        if 'POST' in methods:
-            rv = client.post(route)
-            assert rv.status_code != 403
-
-
 def test_admin_can_update_users_to_ac(mockdata, client, session):
     with current_app.test_request_context():
         login_admin(client)
@@ -76,10 +60,11 @@ def test_admin_can_update_users_to_ac(mockdata, client, session):
 
         form = EditUserForm(
             is_area_coordinator=True,
-            ac_department=AC_DEPT)
+            ac_department=AC_DEPT,
+            submit=True)
 
         rv = client.post(
-            url_for('auth.user_api', user_id=user_id),
+            url_for('auth.edit_user', user_id=user_id),
             data=form.data,
             follow_redirects=True
         )
@@ -95,10 +80,10 @@ def test_admin_cannot_update_to_ac_without_department(mockdata, client, session)
         user = User.query.except_(User.query.filter_by(is_administrator=True)).first()
         user_id = user.id
 
-        form = EditUserForm(is_area_coordinator=True)
+        form = EditUserForm(is_area_coordinator=True, submit=True)
 
         rv = client.post(
-            url_for('auth.user_api', user_id=user_id),
+            url_for('auth.edit_user', user_id=user_id),
             data=form.data,
             follow_redirects=True
         )
@@ -116,10 +101,11 @@ def test_admin_can_update_users_to_admin(mockdata, client, session):
 
         form = EditUserForm(
             is_area_coordinator=False,
-            is_administrator=True)
+            is_administrator=True,
+            submit=True)
 
         rv = client.post(
-            url_for('auth.user_api', user_id=user_id),
+            url_for('auth.edit_user', user_id=user_id),
             data=form.data,
             follow_redirects=True
         )
@@ -137,13 +123,13 @@ def test_admin_can_delete_user(mockdata, client, session):
         username = user.username
 
         rv = client.get(
-            url_for('auth.user_api', user_id=user_id) + '/delete',
+            url_for('auth.delete_user', user_id=user_id),
         )
 
         assert b'Are you sure you want to delete this user?' in rv.data
 
         rv = client.post(
-            url_for('auth.user_api', user_id=user_id) + '/delete',
+            url_for('auth.delete_user', user_id=user_id),
             follow_redirects=True
         )
 
@@ -151,46 +137,101 @@ def test_admin_can_delete_user(mockdata, client, session):
         assert not User.query.get(user_id)
 
 
+def test_admin_cannot_delete_other_admin(mockdata, client, session):
+    with current_app.test_request_context():
+        login_admin(client)
+
+        user = User(is_administrator=True, email='another_user@example.org')
+        session.add(user)
+        session.commit()
+        user_id = user.id
+
+        rv = client.post(
+            url_for('auth.delete_user', user_id=user_id),
+            follow_redirects=True
+        )
+
+        assert rv.status_code == 403
+        assert User.query.get(user_id) is not None
+
+
 def test_admin_can_disable_user(mockdata, client, session):
     with current_app.test_request_context():
         login_admin(client)
 
-        user = User.query.first()
+        # just need to make sure to not select the admin user
+        user = User.query.filter_by(is_administrator=False).first()
         user_id = user.id
-        username = user.username
 
         assert not user.is_disabled
 
-        rv = client.get(
-            url_for('auth.user_api', user_id=user_id) + '/disable',
+        form = EditUserForm(
+            is_disabled=True,
+            submit=True,
+        )
+
+        rv = client.post(
+            url_for('auth.edit_user', user_id=user_id),
+            data=form.data,
             follow_redirects=True
         )
 
-        assert 'User {} has been disabled!'.format(username) in rv.data.decode('utf-8')
+        assert 'updated!' in rv.data.decode('utf-8')
 
         user = User.query.get(user_id)
         assert user.is_disabled
 
 
+def test_admin_cannot_disable_self(mockdata, client, session):
+    with current_app.test_request_context():
+        login_admin(client)
+
+        user = User.query.filter_by(email=ADMIN_EMAIL).first()
+        user_id = user.id
+
+        assert not user.is_disabled
+
+        form = EditUserForm(
+            is_disabled=True,
+            submit=True,
+        )
+
+        rv = client.post(
+            url_for('auth.edit_user', user_id=user_id),
+            data=form.data,
+            follow_redirects=True
+        )
+
+        assert "You cannot edit your own account!" in rv.data.decode('utf-8')
+
+        user = User.query.get(user_id)
+        assert not user.is_disabled
+
+
 def test_admin_can_enable_user(mockdata, client, session):
     with current_app.test_request_context():
         login_admin(client)
 
-        user = User.query.first()
+        user = User.query.filter_by(is_administrator=False).first()
         user_id = user.id
-        username = user.username
         user.is_disabled = True
         db.session.commit()
 
         user = User.query.get(user_id)
         assert user.is_disabled
 
-        rv = client.get(
-            url_for('auth.user_api', user_id=user_id) + '/enable',
+        form = EditUserForm(
+            is_disabled=False,
+            submit=True,
+        )
+
+        rv = client.post(
+            url_for('auth.edit_user', user_id=user_id),
+            data=form.data,
             follow_redirects=True
         )
 
-        assert 'User {} has been enabled!'.format(username) in rv.data.decode('utf-8')
+        assert 'updated!' in rv.data.decode('utf-8')
 
         user = User.query.get(user_id)
         assert not user.is_disabled
@@ -204,8 +245,13 @@ def test_admin_can_resend_user_confirmation_email(mockdata, client, session):
         user_id = user.id
         email = user.email
 
-        rv = client.get(
-            url_for('auth.user_api', user_id=user_id) + '/resend',
+        form = EditUserForm(
+            resend=True,
+        )
+
+        rv = client.post(
+            url_for('auth.edit_user', user_id=user_id),
+            data=form.data,
             follow_redirects=True
         )
 
@@ -252,21 +298,26 @@ def test_admin_can_approve_user(mockdata, client, session):
     with current_app.test_request_context():
         login_admin(client)
 
-        user = User.query.first()
+        user = User.query.filter_by(is_administrator=False).first()
         user_id = user.id
-        username = user.username
         user.approved = False
         db.session.commit()
 
         user = User.query.get(user_id)
         assert not user.approved
 
-        rv = client.get(
-            url_for('auth.user_api', user_id=user_id) + '/approve',
+        form = EditUserForm(
+            approved=True,
+            submit=True,
+        )
+
+        rv = client.post(
+            url_for('auth.edit_user', user_id=user_id),
+            data=form.data,
             follow_redirects=True
         )
 
-        assert 'User {} has been approved!'.format(username) in rv.data.decode('utf-8')
+        assert 'updated!' in rv.data.decode('utf-8')
 
         user = User.query.get(user_id)
         assert user.approved

From 46780ce9a37b8a51f68fab780c9da9b62c81f76a Mon Sep 17 00:00:00 2001
From: abandoned-prototype
 <41744410+abandoned-prototype@users.noreply.github.com>
Date: Thu, 20 May 2021 23:22:19 -0500
Subject: [PATCH 12/15] fix image upload

---
 OpenOversight/app/templates/submit_image.html         | 4 ++++
 OpenOversight/app/templates/submit_officer_image.html | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/OpenOversight/app/templates/submit_image.html b/OpenOversight/app/templates/submit_image.html
index 66cd6717c..e8a36cccf 100644
--- a/OpenOversight/app/templates/submit_image.html
+++ b/OpenOversight/app/templates/submit_image.html
@@ -47,6 +47,7 @@ <h3>Drop images here to submit photos of officers:</h3>
    document.getElementById("department").selectedIndex = {{preferred_dept_id}} - 1;
    // Save the preferred department in dept_id
    var dept_id = document.getElementById("department").selectedIndex + 1;
+   var csrf_token = "{{ csrf_token() }}";
 
    // Store changes in drop down list in dept_id variable
    $(function() {
@@ -69,6 +70,9 @@ <h3>Drop images here to submit photos of officers:</h3>
       parallelUploads: 50,
       acceptedFiles: "image/png, image/jpeg, image/gif, image/jpg, image/webp",
       maxFiles: 50,
+      headers: {
+        'X-CSRF-TOKEN': csrf_token
+      },
       init: function() {
          this.on("error", function(file, responseText) {
             file.previewTemplate.appendChild(document.createTextNode(responseText));
diff --git a/OpenOversight/app/templates/submit_officer_image.html b/OpenOversight/app/templates/submit_officer_image.html
index 07131c476..1dda913fd 100644
--- a/OpenOversight/app/templates/submit_officer_image.html
+++ b/OpenOversight/app/templates/submit_officer_image.html
@@ -29,6 +29,7 @@ <h3>Drop images here to submit photos of {{ officer.full_name() }} in {{ officer
   <script src="{{ url_for('static', filename='js/jquery.min.js') }}"></script>
   <script src="{{ url_for('static', filename='js/dropzone.js') }}"></script>
   <script>
+    var csrf_token = "{{ csrf_token() }}";
     Dropzone.autoDiscover = false;
     let myDropzone = new Dropzone("#my-cop-dropzone", {
       url: "/upload/department/{{ officer.department_id }}/officer/{{ officer.id }}",
@@ -37,6 +38,9 @@ <h3>Drop images here to submit photos of {{ officer.full_name() }} in {{ officer
       parallelUploads: 50,
       acceptedFiles: "image/png, image/jpeg, image/gif, image/jpg",
       maxFiles: 50,
+      headers: {
+        'X-CSRF-TOKEN': csrf_token
+      },
       init: function() {
         this.on("error", function(file, responseText) {
           file.previewTemplate.appendChild(document.createTextNode(responseText));

From c0bc1a3690afce7b198c3ae67942d46da53a75c5 Mon Sep 17 00:00:00 2001
From: abandoned-prototype
 <41744410+abandoned-prototype@users.noreply.github.com>
Date: Thu, 20 May 2021 23:31:38 -0500
Subject: [PATCH 13/15] fix csrf on admin tags

---
 OpenOversight/app/templates/tag.html | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/OpenOversight/app/templates/tag.html b/OpenOversight/app/templates/tag.html
index 21276dd97..4922677ab 100644
--- a/OpenOversight/app/templates/tag.html
+++ b/OpenOversight/app/templates/tag.html
@@ -63,6 +63,7 @@ <h3>Officer</h3>
           <h3>Remove tag <small>Admin only</small></h3>
           <p>
             <form action="{{ url_for('main.delete_tag', tag_id=face.id) }}" method="post">
+              <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
               <button type="submit" name="action" class="btn btn-primary">
                 Delete tag
               </button>
@@ -80,6 +81,7 @@ <h3>Add additional tags <small>Admin only</small></h3>
           <h3>Set as featured tag <small>Admin only</small></h3>
           <p>
             <form action="{{ url_for('main.set_featured_tag', tag_id=face.id) }}" method="post">
+              <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
               <button type="submit" name="action" class="btn btn-primary" {% if face.featured %}disabled{% endif %}>
                 Set as featured tag
               </button>

From c1c85f20b4918083b202b75529adc73899a3ddfe Mon Sep 17 00:00:00 2001
From: Shreyas Gandlur <shreyas@rot13x2.xyz>
Date: Wed, 26 May 2021 21:41:01 -0500
Subject: [PATCH 14/15] Fix two small bugs in officer search (#872)

* Change officer search to only pull distinct officers

* Fix featured image selection
---
 OpenOversight/app/main/views.py | 2 +-
 OpenOversight/app/utils.py      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/OpenOversight/app/main/views.py b/OpenOversight/app/main/views.py
index 4457cc915..65dcfed0e 100644
--- a/OpenOversight/app/main/views.py
+++ b/OpenOversight/app/main/views.py
@@ -536,7 +536,7 @@ def list_officer(department_id, page=1, race=[], gender=[], rank=[], min_age='16
     officers = officers.order_by(Officer.last_name, Officer.first_name, Officer.id)
     officers = officers.paginate(page, OFFICERS_PER_PAGE, False)
     for officer in officers.items:
-        officer_face = sorted(officer.face, key=lambda x: x.featured)
+        officer_face = sorted(officer.face, key=lambda x: x.featured, reverse=True)
 
         # could do some extra work to not lazy load images but load them all together
         # but we would want to ensure to only load the first picture of each officer
diff --git a/OpenOversight/app/utils.py b/OpenOversight/app/utils.py
index 20ec709f8..1519887cd 100644
--- a/OpenOversight/app/utils.py
+++ b/OpenOversight/app/utils.py
@@ -309,7 +309,7 @@ def filter_by_form(form_data, officer_query, department_id=None):
             officer_query = officer_query.filter(Assignment.job_id.in_(job_ids))
     officer_query = (
         officer_query
-        .options(selectinload(Officer.assignments_lazy))
+        .options(selectinload(Officer.assignments_lazy)).distinct()
     )
 
     return officer_query

From c9abdaf4704813edb27d76673a24d6cba95ffdcf Mon Sep 17 00:00:00 2001
From: pierwill <19642016+pierwill@users.noreply.github.com>
Date: Wed, 29 Sep 2021 19:05:20 -0500
Subject: [PATCH 15/15] Verify two dependencies (#876)

* build: Verify checksum of geckodriver

* build: verify checksum of node setup script

* fix: install nodejs correctly with apt

Co-authored-by: pierwill <pierwill@users.noreply.github.com>
---
 dockerfiles/web/Dockerfile | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/dockerfiles/web/Dockerfile b/dockerfiles/web/Dockerfile
index 20eee30db..b9f827de6 100644
--- a/dockerfiles/web/Dockerfile
+++ b/dockerfiles/web/Dockerfile
@@ -4,25 +4,42 @@ FROM python:${TRAVIS_PYTHON_VERSION:-3.5}-buster
 
 WORKDIR /usr/src/app
 
+ENV CURL_FLAGS="--proto =https --tlsv1.2 -sSf -L --max-redirs 1 -O"
+
 ENV DEBIAN-FRONTEND noninteractive
 ENV DISPLAY=:1
-ENV GECKODRIVER_VERSION="v0.26.0"
+
+# install apt dependencies
 RUN echo "deb http://deb.debian.org/debian stretch-backports main" > /etc/apt/sources.list.d/backports.list
-RUN wget -O - https://deb.nodesource.com/setup_12.x | bash -
-RUN apt-get update && apt-get install -y xvfb firefox-esr libpq-dev python3-dev nodejs && \
+RUN apt-get update && apt-get install -y xvfb firefox-esr libpq-dev python3-dev && \
     apt-get install -y -t stretch-backports libsqlite3-0 && apt-get clean
 
-RUN wget https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz
+# install node
+ENV NODE_SETUP_SHA=5d07994f59e3edc2904c547e772b818d10abb066f6ff36ab3db5d686b0fe9a73
+RUN curl ${CURL_FLAGS} \
+      https://raw.githubusercontent.com/nodesource/distributions/b8510857fb4ce4b023161be8490b00119884974c/deb/setup_12.x
+RUN echo "${NODE_SETUP_SHA}  setup_12.x" | sha256sum --check -
+RUN bash setup_12.x
+RUN apt-get install -y nodejs
+
+# install geckodriver
+ENV GECKODRIVER_VERSION="v0.26.0"
+ENV GECKODRIVER_SHA=d59ca434d8e41ec1e30dd7707b0c95171dd6d16056fb6db9c978449ad8b93cc0
+ENV GECKODRIVER_BASE_URL="https://github.com/mozilla/geckodriver/releases/download"
+RUN curl ${CURL_FLAGS} \
+      ${GECKODRIVER_BASE_URL}/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz
+RUN echo "${GECKODRIVER_SHA}  geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz" | sha256sum --check -
 RUN mkdir geckodriver
 RUN tar -xzf geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -C geckodriver
 
+# install yarn
 RUN npm install -g yarn
-
 RUN mkdir /var/www ./node_modules /.cache /.yarn /.mozilla
 RUN touch /usr/src/app/yarn-error.log
 COPY yarn.lock /usr/src/app/
 RUN chmod -R 777 /usr/src/app/ /var/lib/xkb /.cache /.yarn /.mozilla
 
+
 COPY requirements.txt dev-requirements.txt /usr/src/app/
 RUN pip3 install --no-cache-dir -r requirements.txt