Officer unit view

CONTRIB.md

@@ -12,6 +12,8 @@ Use [PULL_REQUEST_TEMPLATE.md](/PULL_REQUEST_TEMPLATE.md) to create the descript
 
  `flake8` is a tool for automated linting and style checks. Be sure to run `flake8` and fix any errors before submitting a PR.
 
+ You can run it with `make lint` to execute flake8 from the docker containers.
+
 ## Development Environment
 
 You can use our Docker-compose environment to stand up a development OpenOversight.
@@ -29,6 +31,8 @@ $ docker exec -it openoversight_postgres_1 /bin/bash
 # psql -d openoversight-dev -U openoversight
 ```
 
+or run `make attach`.
+
 Similarly to hop into the web container:
 
 ```

Makefile

@@ -40,10 +40,14 @@ populate: create_db  ## Build and run containers
 .PHONY: test
 test: start  ## Run tests
 	if [ -z "$(name)" ]; \
-	    then FLASK_ENV=testing docker-compose run --rm web pytest -n 4 --dist=loadfile -v tests/; \
-	    else FLASK_ENV=testing docker-compose run --rm web pytest -n 4 --dist=loadfile -v tests/ -k $(name); \
+	    then FLASK_ENV=testing docker-compose run --rm web pytest --doctest-modules -n 4 --dist=loadfile -v tests/ app; \
+	    else FLASK_ENV=testing docker-compose run --rm web pytest --doctest-modules -n 4 --dist=loadfile -v tests app -k $(name); \
 	fi
 
+.PHONY: lint
+lint: start
+	docker-compose run --rm web flake8
+
 .PHONY: cleanassets
 cleanassets:
 	rm -rf ./OpenOversight/app/static/dist/
@@ -71,3 +75,6 @@ help: ## Print this message and exit
 	@awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \
 		| sort \
 		| column -s ':' -t
+
+attach:
+	docker-compose exec postgres psql -h localhost -U openoversight openoversight-dev

OpenOversight/app/commands.py

@@ -227,6 +227,35 @@ def create_officer_from_row(row, department_id):
     process_salary(row, officer, compare=False)
 
 
+def is_equal(a, b):
+    """exhaustive equality checking, originally to compare a sqlalchemy result object of various types to a csv string
+    Note: Stringifying covers object cases (as in the datetime example below)
+    >>> is_equal("1", 1)  # string == int
+    True
+    >>> is_equal("foo", "bar") # string != other string
+    False
+    >>> is_equal(1, "1") # int == string
+    True
+    >>> is_equal(1.0, "1") # float == string
+    True
+    >>> is_equal(datetime(2020, 1, 1), "2020-01-01 00:00:00") # datetime == string
+    True
+    """
+    def try_else_false(comparable):
+        try:
+            return comparable(a, b)
+        except TypeError:
+            return False
+        except ValueError:
+            return False
+
+    return any([
+        try_else_false(lambda _a, _b: str(_a) == str(_b)),
+        try_else_false(lambda _a, _b: int(_a) == int(_b)),
+        try_else_false(lambda _a, _b: float(_a) == float(_b))
+    ])
+
+
 def process_assignment(row, officer, compare=False):
     assignment_fields = {
         'required': ['job_title'],
@@ -252,7 +281,7 @@ def process_assignment(row, officer, compare=False):
                 for fieldname in assignment_fieldnames:
                     current = getattr(assignment, fieldname)
                     # Test if fields match between row and existing assignment
-                    if (current and fieldname in row and row[fieldname] == current) or \
+                    if (current and fieldname in row and is_equal(row[fieldname], current)) or \
                             (not current and (fieldname not in row or not row[fieldname])):
                         i += 1
                 if i == len(assignment_fieldnames):

OpenOversight/app/main/forms.py

@@ -46,6 +46,7 @@ class FindOfficerForm(Form):
     unique_internal_identifier = StringField('unique_internal_identifier', default='', validators=[Regexp(r'\w*'), Length(max=55)])
     dept = QuerySelectField('dept', validators=[DataRequired()],
                             query_factory=dept_choices, get_label='name')
+    unit = SelectField('unit', default='Not Sure')
     rank = StringField('rank', default='Not Sure', validators=[Optional()])  # Gets rewritten by Javascript
     race = SelectField('race', default='Not Sure', choices=RACE_CHOICES,
                        validators=[AnyOf(allowed_values(RACE_CHOICES))])
@@ -384,6 +385,7 @@ class IncidentForm(DateFieldForm):
 class BrowseForm(Form):
     rank = QuerySelectField('rank', validators=[Optional()], get_label='job_title',
                             get_pk=lambda job: job.job_title)  # query set in view function
+    unit = SelectField('unit', default='Not Sure')
     name = StringField('Last name')
     badge = StringField('Badge number')
     unique_internal_identifier = StringField('Unique ID')

OpenOversight/app/main/views.py

@@ -478,7 +478,8 @@ def edit_department(department_id):
 
 
 @main.route('/department/<int:department_id>')
-def list_officer(department_id, page=1, race=[], gender=[], rank=[], min_age='16', max_age='100', name=None, badge=None, unique_internal_identifier=None):
+def list_officer(department_id, page=1, race=[], gender=[], rank=[], min_age='16', max_age='100', name=None,
+                 badge=None, unique_internal_identifier=None, unit=None):
     form = BrowseForm()
     form.rank.query = Job.query.filter_by(department_id=department_id, is_sworn_officer=True).order_by(Job.order.asc()).all()
     form_data = form.data
@@ -489,6 +490,7 @@ def list_officer(department_id, page=1, race=[], gender=[], rank=[], min_age='16
     form_data['max_age'] = max_age
     form_data['name'] = name
     form_data['badge'] = badge
+    form_data['unit'] = unit
     form_data['unique_internal_identifier'] = unique_internal_identifier
 
     OFFICERS_PER_PAGE = int(current_app.config['OFFICERS_PER_PAGE'])
@@ -507,23 +509,28 @@ def list_officer(department_id, page=1, race=[], gender=[], rank=[], min_age='16
         form_data['name'] = request.args.get('name')
     if request.args.get('badge'):
         form_data['badge'] = request.args.get('badge')
+    if request.args.get('unit') and request.args.get('unit') != 'Not Sure':
+        form_data['unit'] = int(request.args.get('unit'))
     if request.args.get('unique_internal_identifier'):
         form_data['unique_internal_identifier'] = request.args.get('unique_internal_identifier')
     if request.args.get('race') and all(race in [rc[0] for rc in RACE_CHOICES] for race in request.args.getlist('race')):
         form_data['race'] = request.args.getlist('race')
     if request.args.get('gender') and all(gender in [gc[0] for gc in GENDER_CHOICES] for gender in request.args.getlist('gender')):
         form_data['gender'] = request.args.getlist('gender')
 
+    unit_choices = [(unit.id, unit.descrip) for unit in Unit.query.filter_by(department_id=department_id).all()]
     rank_choices = [jc[0] for jc in db.session.query(Job.job_title, Job.order).filter_by(department_id=department_id, is_sworn_officer=True).order_by(Job.order).all()]
     if request.args.get('rank') and all(rank in rank_choices for rank in request.args.getlist('rank')):
         form_data['rank'] = request.args.getlist('rank')
 
-    officers = filter_by_form(form_data, Officer.query, department_id).filter(Officer.department_id == department_id).order_by(Officer.last_name).paginate(page, OFFICERS_PER_PAGE, False)
+    officers = filter_by_form(form_data, Officer.query, department_id).filter_by(department_id=department_id)\
+        .order_by(Officer.last_name).paginate(page, OFFICERS_PER_PAGE, False)
 
     choices = {
         'race': RACE_CHOICES,
         'gender': GENDER_CHOICES,
-        'rank': [(rc, rc) for rc in rank_choices]
+        'rank': [(rc, rc) for rc in rank_choices],
+        'unit': [('Not Sure', 'Not Sure')] + unit_choices
     }
 
     return render_template(

OpenOversight/app/templates/list_officer.html

@@ -92,6 +92,22 @@ <h3 class="panel-title accordion-toggle">Rank</h3>
             </div>
           </div>
         </div>
+        <div class="panel">
+          <div class="panel-heading" data-toggle="collapse" data-target="#filter-unit">
+            <h3 class="panel-title accordion-toggle">Unit</h3>
+          </div>
+          <div class="collapse in" id="filter-unit">
+            <div class="panel-body">
+              <div class="form-group">
+                <select class="form-control" name="unit">
+                  {% for choice in choices['unit'] %}
+                  <option id="unit-{{ choice[0] }}" value="{{choice[0]}}" {% if choice[0] == form_data['unit'] %}selected='true'{% endif %}>{{choice[1]}}</option>
+                  {% endfor %}
+                </select>
+              </div>
+            </div>
+          </div>
+        </div>
         <div class="panel">
           <div class="panel-heading" data-toggle="collapse" data-target="#filter-age">
             <h3 class="panel-title accordion-toggle">Age range</h3>
@@ -120,12 +136,12 @@ <h3 class="panel-title accordion-toggle">Age range</h3>
           page=officers.next_num, race=form_data['race'],
           gender=form_data['gender'], rank=form_data['rank'], min_age=form_data['min_age'],
           max_age=form_data['max_age'], name=form_data['name'], badge=form_data['badge'],
-          unique_internal_identifier=form_data['unique_internal_identifier']),
+          unique_internal_identifier=form_data['unique_internal_identifier'], unit=form_data['unit']),
         prev_url=url_for('main.list_officer', department_id=department.id,
           page=officers.prev_num, race=form_data['race'],
           gender=form_data['gender'], rank=form_data['rank'], min_age=form_data['min_age'],
           max_age=form_data['max_age'], name=form_data['name'], badge=form_data['badge'],
-          unique_internal_identifier=form_data['unique_internal_identifier']),
+          unique_internal_identifier=form_data['unique_internal_identifier'], unit=form_data['unit']),
         location='top' %}
       {% include "partials/paginate_nav.html" %}
     {% endwith %}
@@ -179,12 +195,12 @@ <h2>
           page=officers.next_num, race=form_data['race'],
           gender=form_data['gender'], rank=form_data['rank'], min_age=form_data['min_age'],
           max_age=form_data['max_age'], name=form_data['name'], badge=form_data['badge'],
-          unique_internal_identifier=form_data['unique_internal_identifier']),
+          unique_internal_identifier=form_data['unique_internal_identifier'], unit=form_data['unit']),
         prev_url=url_for('main.list_officer', department_id=department.id,
           page=officers.prev_num, race=form_data['race'],
           gender=form_data['gender'], rank=form_data['rank'], min_age=form_data['min_age'],
           max_age=form_data['max_age'], name=form_data['name'], badge=form_data['badge'],
-          unique_internal_identifier=form_data['unique_internal_identifier']),
+          unique_internal_identifier=form_data['unique_internal_identifier'], unit=form_data['unit']),
         location='bottom' %}
       {% include "partials/paginate_nav.html" %}
     {% endwith %}

OpenOversight/app/utils.py

@@ -261,7 +261,8 @@ def filter_by_form(form, officer_query, department_id=None):
         Assignment.officer_id,
         Assignment.job_id,
         Assignment.star_date,
-        Assignment.star_no
+        Assignment.star_no,
+        Assignment.unit_id
     ).add_columns(row_num_col).from_self().filter(row_num_col == 1).subquery()
     officer_query = officer_query.outerjoin(subq)
 
@@ -278,6 +279,11 @@ def filter_by_form(form, officer_query, department_id=None):
         officer_query = officer_query.filter(
             subq.c.assignments_star_no.like('%%{}%%'.format(form['badge']))
         )
+    if form.get('unit'):
+        officer_query = officer_query.filter(
+            subq.c.assignments_unit_id == form['unit']
+        )
+
     if form.get('unique_internal_identifier'):
         officer_query = officer_query.filter(
             Officer.unique_internal_identifier.ilike('%%{}%%'.format(form['unique_internal_identifier']))

OpenOversight/tests/conftest.py

@@ -1,4 +1,6 @@
 import datetime
+from typing import List
+
 from flask import current_app
 from io import BytesIO
 import pytest
@@ -16,7 +18,7 @@
 
 from OpenOversight.app import create_app, models
 from OpenOversight.app.utils import merge_dicts
-from OpenOversight.app.models import db as _db
+from OpenOversight.app.models import db as _db, Unit, Job, Officer
 
 factory = Faker()
 
@@ -39,6 +41,22 @@ def pick_birth_date():
     return random.randint(1950, 2000)
 
 
+def pick_date(seed: bytes = None, start_year=2000, end_year=2020):
+    # source: https://stackoverflow.com/questions/40351791/how-to-hash-strings-into-a-float-in-01
+    # Wanted to deterministically create a date from a seed string (e.g. the hash or uuid on an officer object)
+    from struct import unpack
+    from hashlib import sha256
+
+    def bytes_to_float(b):
+        return float(unpack('L', sha256(b).digest()[:8])[0]) / 2 ** 64
+
+    if seed is None:
+        seed = str(uuid.uuid4()).encode('utf-8')
+
+    return datetime.datetime(start_year, 1, 1, 00, 00, 00) \
+        + datetime.timedelta(days=365 * (end_year - start_year) * bytes_to_float(seed))
+
+
 def pick_race():
     return random.choice(['WHITE', 'BLACK', 'HISPANIC', 'ASIAN',
                           'PACIFIC ISLANDER', 'Not Sure'])
@@ -95,9 +113,11 @@ def generate_officer():
     )
 
 
-def build_assignment(officer, unit, jobs):
+def build_assignment(officer: Officer, units: List[Unit], jobs: Job):
     return models.Assignment(star_no=pick_star(), job_id=random.choice(jobs).id,
-                             officer=officer)
+                             officer=officer, unit_id=random.choice(units).id,
+                             star_date=pick_date(officer.full_name().encode('utf-8')),
+                             resign_date=pick_date(officer.full_name().encode('utf-8')))
 
 
 def build_note(officer, user):
@@ -131,10 +151,11 @@ def build_salary(officer):
 
 def assign_faces(officer, images):
     if random.uniform(0, 1) >= 0.5:
-        for num in range(1, len(images)):
-            return models.Face(officer_id=officer.id,
-                               img_id=num,
-                               original_image_id=random.choice(images).id)
+        # for num in range(1, len(images)):  # <-- bug here, we always return on the first one, the for-loop is unneeded
+        img_id = random.choice(images).id
+        return models.Face(officer_id=officer.id,
+                           img_id=img_id,
+                           original_image_id=img_id)
     else:
         return False
 
@@ -249,7 +270,15 @@ def add_mockdata(session):
     SEED = current_app.config['SEED']
     random.seed(SEED)
 
-    unit1 = models.Unit(descrip="test")
+    test_units = [
+        models.Unit(descrip="test", department_id=1),
+        models.Unit(descrip='District 13', department_id=1),
+        models.Unit(descrip='Donut Devourers', department_id=1),
+        models.Unit(descrip='Bureau of Organized Crime', department_id=2),
+        models.Unit(descrip='Porky\'s BBQ: Rub Division', department_id=2)
+    ]
+    session.add_all(test_units)
+    session.commit()
 
     test_images = [models.Image(filepath='/static/images/test_cop{}.png'.format(x + 1), department_id=1) for x in range(5)] + \
         [models.Image(filepath='/static/images/test_cop{}.png'.format(x + 1), department_id=2) for x in range(5)]
@@ -270,15 +299,14 @@ def add_mockdata(session):
 
     jobs_dept1 = models.Job.query.filter_by(department_id=1).all()
     jobs_dept2 = models.Job.query.filter_by(department_id=2).all()
-    assignments_dept1 = [build_assignment(officer, unit1, jobs_dept1) for officer in officers_dept1]
-    assignments_dept2 = [build_assignment(officer, unit1, jobs_dept2) for officer in officers_dept2]
+    assignments_dept1 = [build_assignment(officer, test_units, jobs_dept1) for officer in officers_dept1]
+    assignments_dept2 = [build_assignment(officer, test_units, jobs_dept2) for officer in officers_dept2]
 
     salaries = [build_salary(officer) for officer in all_officers]
     faces_dept1 = [assign_faces(officer, assigned_images_dept1) for officer in officers_dept1]
     faces_dept2 = [assign_faces(officer, assigned_images_dept2) for officer in officers_dept2]
     faces1 = [f for f in faces_dept1 if f]
     faces2 = [f for f in faces_dept2 if f]
-    session.add(unit1)
     session.commit()
     session.add_all(assignments_dept1)
     session.add_all(assignments_dept2)
@@ -313,12 +341,6 @@ def add_mockdata(session):
     session.add(test_unconfirmed_user)
     session.commit()
 
-    test_units = [models.Unit(descrip='District 13', department_id=1),
-                  models.Unit(descrip='Bureau of Organized Crime',
-                              department_id=1)]
-    session.add_all(test_units)
-    session.commit()
-
     test_addresses = [
         models.Location(
             street_name='Test St',

dockerfiles/web/Dockerfile

@@ -31,6 +31,7 @@ COPY package.json /usr/src/app/
 RUN yarn
 
 COPY create_db.py test_data.py /usr/src/app/
+COPY .flake8 /usr/src/app/
 EXPOSE 3000
 
 ENV PATH="/usr/src/app/geckodriver:${PATH}"