add attach/lint, add pick_date, add is_equal

CONTRIB.md

@@ -12,6 +12,8 @@ Use [PULL_REQUEST_TEMPLATE.md](/PULL_REQUEST_TEMPLATE.md) to create the descript
 
  `flake8` is a tool for automated linting and style checks. Be sure to run `flake8` and fix any errors before submitting a PR.
 
+ You can run it with `make lint` to execute flake8 from the docker containers.
+
 ## Development Environment
 
 You can use our Docker-compose environment to stand up a development OpenOversight.
@@ -29,6 +31,8 @@ $ docker exec -it openoversight_postgres_1 /bin/bash
 # psql -d openoversight-dev -U openoversight
 ```
 
+or run `make attach`.
+
 Similarly to hop into the web container:
 
 ```

Makefile

@@ -40,10 +40,14 @@ populate: create_db  ## Build and run containers
 .PHONY: test
 test: start  ## Run tests
 	if [ -z "$(name)" ]; \
-	    then FLASK_ENV=testing docker-compose run --rm web pytest -n 4 --dist=loadfile -v tests/; \
-	    else FLASK_ENV=testing docker-compose run --rm web pytest -n 4 --dist=loadfile -v tests/ -k $(name); \
+	    then FLASK_ENV=testing docker-compose run --rm web pytest --doctest-modules -n 4 --dist=loadfile -v tests/ app; \
+	    else FLASK_ENV=testing docker-compose run --rm web pytest --doctest-modules -n 4 --dist=loadfile -v tests app -k $(name); \
 	fi
 
+.PHONY: lint
+lint: start
+	docker-compose run --rm web flake8
+
 .PHONY: cleanassets
 cleanassets:
 	rm -rf ./OpenOversight/app/static/dist/
@@ -71,3 +75,6 @@ help: ## Print this message and exit
 	@awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \
 		| sort \
 		| column -s ':' -t
+
+attach:
+	docker-compose exec postgres psql -h localhost -U openoversight openoversight-dev

OpenOversight/app/commands.py

@@ -227,6 +227,35 @@ def create_officer_from_row(row, department_id):
     process_salary(row, officer, compare=False)
 
 
+def is_equal(a, b):
+    """exhaustive equality checking, originally to compare a sqlalchemy result object of various types to a csv string
+    Note: Stringifying covers object cases (as in the datetime example below)
+    >>> is_equal("1", 1)  # string == int
+    True
+    >>> is_equal("foo", "bar") # string != other string
+    False
+    >>> is_equal(1, "1") # int == string
+    True
+    >>> is_equal(1.0, "1") # float == string
+    True
+    >>> is_equal(datetime(2020, 1, 1), "2020-01-01 00:00:00") # datetime == string
+    True
+    """
+    def try_else_false(comparable):
+        try:
+            return comparable(a, b)
+        except TypeError:
+            return False
+        except ValueError:
+            return False
+
+    return any([
+        try_else_false(lambda _a, _b: str(_a) == str(_b)),
+        try_else_false(lambda _a, _b: int(_a) == int(_b)),
+        try_else_false(lambda _a, _b: float(_a) == float(_b))
+    ])
+
+
 def process_assignment(row, officer, compare=False):
     assignment_fields = {
         'required': ['job_title'],
@@ -252,7 +281,7 @@ def process_assignment(row, officer, compare=False):
                 for fieldname in assignment_fieldnames:
                     current = getattr(assignment, fieldname)
                     # Test if fields match between row and existing assignment
-                    if (current and fieldname in row and row[fieldname] == current) or \
+                    if (current and fieldname in row and is_equal(row[fieldname], current)) or \
                             (not current and (fieldname not in row or not row[fieldname])):
                         i += 1
                 if i == len(assignment_fieldnames):

OpenOversight/tests/conftest.py

@@ -16,7 +16,7 @@
 
 from OpenOversight.app import create_app, models
 from OpenOversight.app.utils import merge_dicts
-from OpenOversight.app.models import db as _db
+from OpenOversight.app.models import db as _db, Unit, Job, Officer
 
 factory = Faker()
 
@@ -39,6 +39,22 @@ def pick_birth_date():
     return random.randint(1950, 2000)
 
 
+def pick_date(seed: bytes = None, start_year=2000, end_year=2020):
+    # source: https://stackoverflow.com/questions/40351791/how-to-hash-strings-into-a-float-in-01
+    # Wanted to deterministically create a date from a seed string (e.g. the hash or uuid on an officer object)
+    from struct import unpack
+    from hashlib import sha256
+
+    def bytes_to_float(b):
+        return float(unpack('L', sha256(b).digest()[:8])[0]) / 2 ** 64
+
+    if seed is None:
+        seed = str(uuid.uuid4()).encode('utf-8')
+
+    return datetime.datetime(start_year, 1, 1, 00, 00, 00) \
+        + datetime.timedelta(days=365 * (end_year - start_year) * bytes_to_float(seed))
+
+
 def pick_race():
     return random.choice(['WHITE', 'BLACK', 'HISPANIC', 'ASIAN',
                           'PACIFIC ISLANDER', 'Not Sure'])
@@ -95,9 +111,11 @@ def generate_officer():
     )
 
 
-def build_assignment(officer, unit, jobs):
+def build_assignment(officer: Officer, unit: Unit, jobs: Job):
     return models.Assignment(star_no=pick_star(), job_id=random.choice(jobs).id,
-                             officer=officer)
+                             officer=officer, unit_id=unit.id,
+                             star_date=pick_date(officer.full_name().encode('utf-8')),
+                             resign_date=pick_date(officer.full_name().encode('utf-8')))
 
 
 def build_note(officer, user):
@@ -249,7 +267,8 @@ def add_mockdata(session):
     SEED = current_app.config['SEED']
     random.seed(SEED)
 
-    unit1 = models.Unit(descrip="test")
+    unit1 = models.Unit(descrip="test", department_id=1)
+    session.add(unit1)
 
     test_images = [models.Image(filepath='/static/images/test_cop{}.png'.format(x + 1), department_id=1) for x in range(5)] + \
         [models.Image(filepath='/static/images/test_cop{}.png'.format(x + 1), department_id=2) for x in range(5)]
@@ -278,7 +297,6 @@ def add_mockdata(session):
     faces_dept2 = [assign_faces(officer, assigned_images_dept2) for officer in officers_dept2]
     faces1 = [f for f in faces_dept1 if f]
     faces2 = [f for f in faces_dept2 if f]
-    session.add(unit1)
     session.commit()
     session.add_all(assignments_dept1)
     session.add_all(assignments_dept2)

dockerfiles/web/Dockerfile

@@ -31,6 +31,7 @@ COPY package.json /usr/src/app/
 RUN yarn
 
 COPY create_db.py test_data.py /usr/src/app/
+COPY .flake8 /usr/src/app/
 EXPOSE 3000
 
 ENV PATH="/usr/src/app/geckodriver:${PATH}"