From 5f7ef203a2341218ce743bc7c5770d0e47b438ed Mon Sep 17 00:00:00 2001 From: Marc Wodahl <56242265+mwodahl@users.noreply.github.com> Date: Mon, 9 Dec 2024 13:35:14 -0700 Subject: [PATCH 01/17] Add Road Conditions translator --- Translators/RoadConditions/Dockerfile | 17 +++ Translators/RoadConditions/itis_codes.py | 74 +++++++++++ Translators/RoadConditions/main.py | 139 ++++++++++++++++++++ Translators/RoadConditions/pgquery.py | 86 ++++++++++++ Translators/RoadConditions/tim_generator.py | 32 +++++ 5 files changed, 348 insertions(+) create mode 100644 Translators/RoadConditions/Dockerfile create mode 100644 Translators/RoadConditions/itis_codes.py create mode 100644 Translators/RoadConditions/main.py create mode 100644 Translators/RoadConditions/pgquery.py create mode 100644 Translators/RoadConditions/tim_generator.py diff --git a/Translators/RoadConditions/Dockerfile b/Translators/RoadConditions/Dockerfile new file mode 100644 index 0000000..f1dc42e --- /dev/null +++ b/Translators/RoadConditions/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.10-slim + +ENV PYTHONUNBUFFERED TRUE + +ENV WORKDIR /app + +WORKDIR $WORKDIR + +COPY . . + +ENV FLASK_APP app/main.py + +RUN apt-get update && apt-get install -y libgeos-dev + +RUN pip install -r requirements.txt + +CMD ["python3", "-m", "flask", "run", "--host", "0.0.0.0", "--port", "8082"] \ No newline at end of file diff --git a/Translators/RoadConditions/itis_codes.py b/Translators/RoadConditions/itis_codes.py new file mode 100644 index 0000000..15fc19c --- /dev/null +++ b/Translators/RoadConditions/itis_codes.py @@ -0,0 +1,74 @@ +from enum import Enum + + +class ItisCodes(Enum): + SPEED_LIMIT = '368' + ACCIDENT = '513' + INCIDENT = '531' + HAZARDOUS_MATERIAL_SPILL = '550' + CLOSED = '770' + CLOSED_FOR_SEASON = '774' + REDUCED_ONE_LANE = '777' + AVALANCHE_CONTROL_ACTIVITIES = '1042' + ROAD_CONSTRUCTION = '1025' + HERD_OF_ANIMALS_ON_ROADWAY = '1292' + ROCKFALL = '1309' + LANDSLIDE = '1310' + DELAYS = '1537' + WIDE_LOAD = '2050' + NO_TRAILERS = '2568' + WIDTH_LIMIT = '2573' + HEIGHT_LIMIT = '2574' + WILD_FIRE = '3084' + WEATHER_EMERGENCY = '3201' + MAJOR_EVENT = '3841' + NO_PARKING_SPACES_AVAILABLE = '4103' + FEW_PARKING_SPACES_AVAILABLE = '4104' + SPACES_AVAILABLE = '4105' + NO_PARKING_INFO_AVAILABLE = '4223' + SEVERE_WEATHER = '4865' + SNOW = '4868' + WINTER_STORM = '4871' + RAIN = '4885' + STRONG_WINDS = '5127' + FOG = '5378' + VISIBILITY_REDUCED = '5383' + BLOWING_SNOW = '5985' + BLACK_ICE = '5908' + WET_PAVEMENT = '5895' + ICE = '5906' + ICY_PATCHES = '5907' + SNOW_DRIFTS = '5927' + GRAVEL_ROAD_SURFACE = '5933' + DRY_PAVEMENT = '6011' + DIRT_ROAD_SURFACE = '6016' + MILLED_ROAD_SURFACE = '6017' + SNOW_TIRES_OR_CHAINS_REQUIRED = '6156' + LOOK_OUT_FOR_WORKERS = '6952' + KEEP_TO_RIGHT = '7425' + KEEP_TO_LEFT = '7426' + REDUCE_YOUR_SPEED = '7443' + DRIVE_CAREFUL = '7169' + DRIVE_WITH_EXTREME_CAUTION = '7170' + INCREASE_FOLLOWING_DISTANCE = '7173' + PREPARE_TO_STOP = '7186' + STOP_AT_NEXT_SAFE_PLACE = '7188' + ONLY_TRAVEL_IF_NECESSARY = '7189' + FALLING_ROCKS = '1203' + +ItisCodeExtraKeywords = { + "herd of animals on roadway": "herd of animals on the roadway", + "rockfall": "rock fall", + "wildfire": "wild fire", + "keep to right": "keep right", + "keep to left": "keep left", + "reduce your speed": "reduce speed", + "drive careful": "drive carefully", + "stop at next safe place": "stop at the next safe place", + "only travel if necessary": "only necessary travel", + "falling rocks": "falling rock", + "icy patches": "icy spots", + "snow": "snow packed spots", + "closed for season": "seasonal closure", + "ice": "icy", +} \ No newline at end of file diff --git a/Translators/RoadConditions/main.py b/Translators/RoadConditions/main.py new file mode 100644 index 0000000..269e5a4 --- /dev/null +++ b/Translators/RoadConditions/main.py @@ -0,0 +1,139 @@ +import json +import requests +import copy +import logging +import os +from pgquery import query_db +from tim_generator import get_geometry, get_itis_codes +from flask import request, Flask +from datetime import datetime + +app = Flask(__name__) + +log_level = os.environ.get('LOGGING_LEVEL', 'INFO') +logging.basicConfig(format='%(levelname)s:%(message)s', level=log_level) + +def calculate_direction(coordinates): + try: + long_dif = coordinates[-1][0] - coordinates[0][0] + lat_dif = coordinates[-1][1] - coordinates[0][1] + except ValueError as e: + return "unknown" + except IndexError as e: + return "unknown" + + if abs(long_dif) > abs(lat_dif): + if long_dif > 0: + # eastbound + direction = "I" + else: + # westbound + direction = "D" + elif lat_dif > 0: + # northbound + direction = "I" + else: + # southbound + direction = "D" + return direction + +def translate(rc_geojson): + tims = {"timRcList": []} + + for feature in rc_geojson["features"]: + if (len(feature["geometry"]["coordinates"]) <= 2): + continue + tim_body = {} + tim_body["clientId"] = feature["properties"]["nameId"].replace("_", "-").replace("/", "-") + tim_body["direction"] = calculate_direction(feature['geometry']['coordinates']) + tim_body["segment"] = feature["properties"]["routeSegmentIndex"] + tim_body["route"] = feature["properties"]["routeName"].replace("_", "-") + tim_body["roadCode"] = feature["properties"]["nameId"].replace("_", "-") + tim_body["itisCodes"] = get_itis_codes(feature) + tim_body["geometry"] = get_geometry(feature["geometry"]["coordinates"]) + tim_body["advisory"] = ["3"] + active_tim_record = active_tim(feature, tim_body) + if active_tim_record: + logging.info(f"TIM already active for record: {tim_body['clientId']}") + continue + tims["timRcList"].append(tim_body) + return tims + +def active_tim(feature, tim_body): + tim_id = tim_body["clientId"] + # if TIM has an active TIM holding record that is current & info is the same as the current TIM record, then do not update + active_tim_holding = query_db(f"SELECT * FROM active_tim_holding WHERE client_id LIKE '%{tim_id}%'") + if len(active_tim_holding) > 0: + active_tim_holding = active_tim_holding[0] + return (active_tim_holding["direction"] == tim_body["direction"] and + f"{active_tim_holding['start_latitude']:.8f}" == f"{tim_body['geometry'][0]['latitude']:.8f}" and + f"{active_tim_holding['start_longitude']:.8f}" == f"{tim_body['geometry'][0]['longitude']:.8f}" and + f"{active_tim_holding['end_latitude']:.8f}" == f"{tim_body['geometry'][-1]['latitude']:.8f}" and + f"{active_tim_holding['end_longitude']:.8f}" == f"{tim_body['geometry'][-1]['longitude']:.8f}") + + # if TIM has an active TIM record that is current & info is the same as the current TIM record, then do not update + active_tim = query_db(f"SELECT * FROM active_tim WHERE client_id LIKE '%{tim_id}%' AND tim_type_id = (SELECT tim_type_id FROM tim_type WHERE type = 'RC') AND marked_for_deletion = false") + if len(active_tim) > 0: + active_tim = active_tim[0] + return (active_tim["direction"] == tim_body["direction"] and + f"{active_tim['start_latitude']:.8f}" == f"{tim_body['geometry'][0]['latitude']:.8f}" and + f"{active_tim['start_longitude']:.8f}" == f"{tim_body['geometry'][0]['longitude']:.8f}" and + f"{active_tim['end_latitude']:.8f}" == f"{tim_body['geometry'][-1]['latitude']:.8f}" and + f"{active_tim['end_longitude']:.8f}" == f"{tim_body['geometry'][-1]['longitude']:.8f}") + + +@app.route('/', methods=['POST']) +def entry(): + if request.method == 'OPTIONS': + headers = { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'POST', + 'Access-Control-Allow-Headers': 'Content-Type', + 'Access-Control-Max-Age': '3600' + } + + return ('', 204, headers) + + headers = { + 'Access-Control-Allow-Origin': '*' + } + + result = RC_tim_translator() + logging.info(result) + + return (result, 200, headers) + +def RC_tim_translator(): + logging.info('Road Conditions TIM Translator Timer Called...') + + # Scrape the CDOT endpoint to get current list of road condition features + geoJSON = json.loads(requests.get(f'https://{os.getenv("CDOT_FEED_ENDPOINT")}/api/v1/roadConditions?apiKey={os.getenv("CDOT_FEED_API_KEY")}').content.decode('utf-8')) + + tim_list = translate(geoJSON) + + logging.info('Pushing TIMs to ODE...') + + tim_list_copy = copy.deepcopy(tim_list) + tim_all_clear_list = {"timRcList": [tim for tim in tim_list_copy["timRcList"] if len(tim["itisCodes"]) == 0]} + tim_list["timRcList"] = [tim for tim in tim_list["timRcList"] if len(tim["itisCodes"]) > 0] + + errNo = 0 + return_value = requests.put(f'{os.getenv("ODE_ENDPOINT")}/submit-rc-ac', json=tim_all_clear_list) + print(f"All Clear Return: {return_value.status_code}") + if (return_value.status_code == 200): + logging.info(f'Successfully submitted {len(tim_list["timRcList"])} All Clear TIMs to ODE') + + errNo = 0 + return_value = requests.post(f'{os.getenv("ODE_ENDPOINT")}/create-update-rc-tim', json=tim_list) + if (return_value.status_code == 200): + return f'Successfully pushed {len(tim_list["timRcList"])} TIMs to ODE' + + return f'Error pushing TIMs to ODE: {return_value.content}' + +# Run via flask app if running locally else just run translator directly +if (os.getenv("RUN_LOCAL") == "true"): + if __name__ == '__main__': + app.run() +else: + res_str = RC_tim_translator() + logging.info(res_str) \ No newline at end of file diff --git a/Translators/RoadConditions/pgquery.py b/Translators/RoadConditions/pgquery.py new file mode 100644 index 0000000..150e0c0 --- /dev/null +++ b/Translators/RoadConditions/pgquery.py @@ -0,0 +1,86 @@ +import os +import sqlalchemy +import logging + +db_config = { + # Pool size is the maximum number of permanent connections to keep. + "pool_size": 5, + # Temporarily exceeds the set pool_size if no connections are available. + "max_overflow": 2, + # Maximum number of seconds to wait when retrieving a + # new connection from the pool. After the specified amount of time, an + # exception will be thrown. + "pool_timeout": 30, # 30 seconds + # 'pool_recycle' is the maximum number of seconds a connection can persist. + # Connections that live longer than the specified amount of time will be + # reestablished + "pool_recycle": 60 # 1 minutes +} + +db = None + +def init_tcp_connection_engine(db_user, db_pass, db_name, db_hostname, db_port): + logging.info(f"Creating DB pool") + pool = sqlalchemy.create_engine( + # Equivalent URL: + # postgresql+pg8000://:@:/ + sqlalchemy.engine.url.URL.create( + drivername="postgresql+pg8000", + username=db_user, # e.g. "my-database-user" + password=db_pass, # e.g. "my-database-password" + host=db_hostname, # e.g. "127.0.0.1" + port=db_port, # e.g. 5432 + database=db_name # e.g. "my-database-name" + ), + **db_config + ) + #pool.dialect.description_encoding = None + logging.info("DB pool created!") + return pool + +def init_socket_connection_engine(db_user, db_pass, db_name, unix_query): + logging.info(f"Creating DB pool") + pool = sqlalchemy.create_engine( + # Equivalent URL: + # postgresql+pg8000://:@/?unix_sock=/cloudsql/ + sqlalchemy.engine.url.URL.create( + drivername="postgresql+pg8000", + username=db_user, # e.g. "my-database-user" + password=db_pass, # e.g. "my-database-password" + database=db_name, # e.g. "my-database-name" + query=unix_query + ), + **db_config + ) + logging.info("DB pool created!") + return pool + + +def init_connection_engine(): + db_user = os.environ["DB_USER"] + db_pass = os.environ["DB_PASS"] + db_name = os.environ["DB_NAME"] + if("INSTANCE_CONNECTION_NAME" in os.environ): + instance_connection_name = os.environ["INSTANCE_CONNECTION_NAME"] + unix_query = { + "unix_sock": f"/cloudsql/{instance_connection_name}/.s.PGSQL.5432" + } + return init_socket_connection_engine(db_user, db_pass, db_name, unix_query) + else: + db_host = os.environ["DB_HOST"] + # Extract host and port from db_host + host_args = db_host.split(":") + db_hostname, db_port = host_args[0], int(host_args[1]) + return init_tcp_connection_engine(db_user, db_pass, db_name, db_hostname, db_port) + + +def query_db(query): + global db + if db is None: + db = init_connection_engine() + + logging.info("DB connection starting...") + with db.connect() as conn: + logging.debug("Executing query...") + data = conn.execute(query).fetchall() + return data diff --git a/Translators/RoadConditions/tim_generator.py b/Translators/RoadConditions/tim_generator.py new file mode 100644 index 0000000..b9fb68e --- /dev/null +++ b/Translators/RoadConditions/tim_generator.py @@ -0,0 +1,32 @@ +from itis_codes import ItisCodes, ItisCodeExtraKeywords +from string import digits + +def get_itis_codes(feature): + itisCodes = [] + + # need to iterate over entries & split to check for keywords + for entry in feature["properties"]["currentConditions"]: + itis_codes = entry["conditionDescription"].split(",") + # itis_codes[0] = itis_codes[0].split(" ")[-1] + for code in itis_codes: + code = code.translate({ord(k): None for k in digits}).replace("-", "").strip() + if code == "forecast text included": + continue + for key in ItisCodes: + searchKey = key.name.replace("_", " ").lower() + if searchKey in code.lower() and key.value not in itisCodes: + itisCodes.append(key.value) + elif searchKey in ItisCodeExtraKeywords: + if ( + ItisCodeExtraKeywords[searchKey] in code.lower() + and key.value not in itisCodes + ): + itisCodes.append(key.value) + + return itisCodes + +def get_geometry(geometry): + annotated_geometry = [] + for coord in geometry: + annotated_geometry.append({"latitude": coord[1], "longitude": coord[0]}) + return annotated_geometry From 34e6464cd36b115b1c5ad066c0f3306e4e4a1daa Mon Sep 17 00:00:00 2001 From: Marc Wodahl <56242265+mwodahl@users.noreply.github.com> Date: Mon, 9 Dec 2024 13:35:31 -0700 Subject: [PATCH 02/17] Add README --- Translators/RoadConditions/README.md | 53 ++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 Translators/RoadConditions/README.md diff --git a/Translators/RoadConditions/README.md b/Translators/RoadConditions/README.md new file mode 100644 index 0000000..eae30f8 --- /dev/null +++ b/Translators/RoadConditions/README.md @@ -0,0 +1,53 @@ +# Road Conditions Translator + +## Table of Contents + +- [About](#about) +- [Getting Started](#getting_started) +- [Usage](#usage) +- [Contributing](../CONTRIBUTING.md) + +## About + +Python Road Conditions to TIM message translator that is designed to pull messages from the CDOT Road Conditions feed and translate to TIM messages. + +## Getting Started + +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See [deployment](#deployment) for notes on how to deploy the project on a live system. + +### Prerequisites + +This project supports Python >= 3.5. Packages required to run the translator can be installed via [pip](https://pip.pypa.io/en/stable/) + +```bash +pip install -r requirements.txt +``` + +Alternatively, if you are running VSCode there is a task available to run this on your behalf. This can be accessed under Task Explorer -> vscode -> pipinstall. + +The scripts also require access to the CDOT Postgres database. This can be accessed by setting the environment variables in the .env file. The .env file is not included in the repository for security reasons, however a sample.env file has been provided to show structure required. + +In addition to the environment variables for accessing the CDOT Postgres database, the scripts also require the following environment variables to scrape the Road Conditions endpoint and deposit the resulting TIMs: +
    +
  1. CDOT_FEED_ENDPOINT - the CDOT Data Feed URL
    2. +
  2. CDOT_FEED_API_KEY - the API key for accessing the CDOT Road Conditions data
    3. +
  3. ODE_ENDPOINT - the ODE URL where translated TIMs will be submitted
    4. +
+ +### Testing +Unit tests are ran with the python pytest module. To run the tests, run the following command from the root of the project: + +```bash +python -m pytest ./tests +``` + +Again, if you are running VSCode there is a task available to run this on your behalf. This can be accessed under Task Explorer -> vscode -> python test and coverage. This task includes a unit test coverage report to indicate how much of the code is covered by unit tests. + +## Usage + +### Running the Translator Locally +Using VSCode, a simple launch.json file has been provided to allow debugging the application. This can be accessed under the Run & Debug tab. The default configuration will run the translator using the functions framework. This runs the translator as a REST service accessed on http://localhost:8082. The translator can be tested by sending a POST request to this endpoint. + + +### Running the Translator via Docker +The Road Conditions to TIM translator can also be run locally using Docker. The translator Dockerfile can be found under Translators/RoadConditions/. Additionally, there is a docker-compose file which builds and runs the translator. Setting the environment variable RUN_LOCAL to true will run the translator REST service as a flask application that can be accessed on http://localhost:8082. Alternatively, leaving RUN_LOCAL blank will run the translator one time immediately after the build has finished. \ No newline at end of file From 538e19bb5f17968912e432a21de0652de43f29fe Mon Sep 17 00:00:00 2001 From: Marc Wodahl <56242265+mwodahl@users.noreply.github.com> Date: Mon, 9 Dec 2024 13:35:47 -0700 Subject: [PATCH 03/17] Add RC translator unit tests --- tests/Translators/RoadConditions/__init__.py | 6 + .../RoadConditions/test_pgquery.py | 187 ++++++++++++++++++ .../RoadConditions/test_tim_generator.py | 61 ++++++ 3 files changed, 254 insertions(+) create mode 100644 tests/Translators/RoadConditions/__init__.py create mode 100644 tests/Translators/RoadConditions/test_pgquery.py create mode 100644 tests/Translators/RoadConditions/test_tim_generator.py diff --git a/tests/Translators/RoadConditions/__init__.py b/tests/Translators/RoadConditions/__init__.py new file mode 100644 index 0000000..c44218b --- /dev/null +++ b/tests/Translators/RoadConditions/__init__.py @@ -0,0 +1,6 @@ +import sys +import os + +# Add the WZDx directory to the path so that relative imports work +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.join(SCRIPT_DIR, '..', '..', '..', 'Translators', 'WZDx')) \ No newline at end of file diff --git a/tests/Translators/RoadConditions/test_pgquery.py b/tests/Translators/RoadConditions/test_pgquery.py new file mode 100644 index 0000000..41d54bc --- /dev/null +++ b/tests/Translators/RoadConditions/test_pgquery.py @@ -0,0 +1,187 @@ +from unittest.mock import MagicMock, patch, Mock +from Translators.RoadConditions import pgquery +import sqlalchemy +import os + +# test that init_tcp_connection_engine is calling sqlalchemy.create_engine with expected arguments +@patch('Translators.RoadConditions.pgquery.db_config', new={'pool_size': 5, 'max_overflow': 2, 'pool_timeout': 30, 'pool_recycle': 1800}) +def test_init_tcp_connection_engine(): + # mock return values for function dependencies + sqlalchemy.create_engine = MagicMock( + return_value = "myengine" + ) + sqlalchemy.engine.url.URL.create = MagicMock( + return_value = "myurl" + ) + + # call function + db_user = "user" + db_pass = "pass" + db_name = "mydatabase" + db_hostname = "myhostname" + db_port = 3000 + engine_pool = pgquery.init_tcp_connection_engine(db_user, db_pass, db_name, db_hostname, db_port) + + # check return value + assert(engine_pool == "myengine") + + # check that sqlalchemy.engine.url.URL.create was called with expected arguments + sqlalchemy.engine.url.URL.create.assert_called_once_with( + drivername='postgresql+pg8000', + username=db_user, + password=db_pass, + host=db_hostname, + port=db_port, + database=db_name + ) + + # check that sqlalchemy.create_engine was called with expected arguments + my_db_config = {'pool_size': 5, 'max_overflow': 2, 'pool_timeout': 30, 'pool_recycle': 1800} + sqlalchemy.create_engine.assert_called_once_with("myurl", **my_db_config) + +# test that init_socket_connection_engine is calling sqlalchemy.create_engine with expected arguments +@patch('Translators.RoadConditions.pgquery.db_config', new={'pool_size': 5, 'max_overflow': 2, 'pool_timeout': 30, 'pool_recycle': 1800}) +def test_init_socket_connection_engine(): + # mock return values for function dependencies + sqlalchemy.create_engine = MagicMock( + return_value = "myengine" + ) + sqlalchemy.engine.url.URL.create = MagicMock( + return_value = "myurl" + ) + + # call function + db_user = "user" + db_pass = "pass" + db_name = "mydatabase" + unix_query = {'unix_sock': '/cloudsql/myproject:us-central1:myinstance'} + engine_pool = pgquery.init_socket_connection_engine(db_user, db_pass, db_name, unix_query) + + # check return value + assert(engine_pool == "myengine") + + # check that sqlalchemy.engine.url.URL.create was called with expected arguments + sqlalchemy.engine.url.URL.create.assert_called_once_with( + drivername='postgresql+pg8000', + username=db_user, + password=db_pass, + database=db_name, + query=unix_query + ) + + # check that sqlalchemy.create_engine was called with expected arguments + my_db_config = {'pool_size': 5, 'max_overflow': 2, 'pool_timeout': 30, 'pool_recycle': 1800} + sqlalchemy.create_engine.assert_called_once_with("myurl", **my_db_config) + +# test initializing tcp connection engine based on environment variables +@patch('Translators.RoadConditions.pgquery.db_config', new={'pool_size': 5, 'max_overflow': 2, 'pool_timeout': 30, 'pool_recycle': 1800}) +def test_init_connection_engine_target_tcp(): + # mock return values for function dependencies + pgquery.init_tcp_connection_engine = MagicMock( + return_value = "myengine1" + ) + pgquery.init_socket_connection_engine = MagicMock( + return_value = "myengine2" + ) + + db_user = "user" + db_pass = "pass" + db_name = "mydatabase" + db_hostname = "myhostname:3000" + + # set environment variables + os.environ['DB_USER'] = db_user + os.environ['DB_PASS'] = db_pass + os.environ['DB_NAME'] = db_name + os.environ["DB_HOST"] = db_hostname + + host_args = db_hostname.split(":") + db_hostname, db_port = host_args[0], int(host_args[1]) + + # call function + engine_pool = pgquery.init_connection_engine() + + # check return value + assert(engine_pool == "myengine1") + + # check that init_tcp_connection_engine was called with expected arguments + pgquery.init_tcp_connection_engine.assert_called_once_with(db_user, db_pass, db_name, db_hostname, db_port) + + # check that init_socket_connection_engine was not called + pgquery.init_socket_connection_engine.assert_not_called() + +# test initializing socket connection engine based on environment variables +@patch('Translators.RoadConditions.pgquery.db_config', new={'pool_size': 5, 'max_overflow': 2, 'pool_timeout': 30, 'pool_recycle': 1800}) +@patch('Translators.RoadConditions.pgquery.db', new=None) +def test_init_connection_engine_target_socket(): + # mock return values for function dependencies + pgquery.init_tcp_connection_engine = MagicMock( + return_value = "myengine1" + ) + pgquery.init_socket_connection_engine = MagicMock( + return_value = "myengine2" + ) + + db_user = "user" + db_pass = "pass" + db_name = "mydatabase" + + # set environment variables + os.environ['DB_USER'] = db_user + os.environ['DB_PASS'] = db_pass + os.environ['DB_NAME'] = db_name + os.environ['INSTANCE_CONNECTION_NAME'] = "myproject:us-central1:myinstance" + + unix_query = { + "unix_sock": f"/cloudsql/{os.environ['INSTANCE_CONNECTION_NAME']}/.s.PGSQL.5432" + } + + # call function + engine_pool = pgquery.init_connection_engine() + + # check return value + assert(engine_pool == "myengine2") + + # check that init_socket_connection_engine was called with expected arguments + pgquery.init_socket_connection_engine.assert_called_once_with(db_user, db_pass, db_name, unix_query) + + # check that init_tcp_connection_engine was not called + pgquery.init_tcp_connection_engine.assert_not_called() + + # remove INSTANCE_CONNECTION_NAME from os.environ + del os.environ['INSTANCE_CONNECTION_NAME'] + +# test that query_db is calling engine.connect and connection.execute with expected arguments +@patch('Translators.RoadConditions.pgquery.db_config', new={'pool_size': 5, 'max_overflow': 2, 'pool_timeout': 30, 'pool_recycle': 1800}) +@patch('Translators.RoadConditions.pgquery.db', new=None) +def test_query_db(): + pgquery.init_connection_engine = MagicMock( + return_value = Mock( # return a mock engine + connect = MagicMock( + return_value = Mock( # return a mock connection iterator + __enter__ = MagicMock( + return_value = Mock( # return a mock connection + execute = MagicMock( + return_value = Mock( # return a mock result + fetchall = MagicMock( + return_value = "myresult" + ) + ) + ) + ) + ), + __exit__ = MagicMock() + ) + ) + ) + ) + + # call function + query = "SELECT * FROM mytable" + result = pgquery.query_db(query) + + # check return value + assert(result == "myresult") + + # check that init_connection_engine was called once + pgquery.init_connection_engine.assert_called_once() \ No newline at end of file diff --git a/tests/Translators/RoadConditions/test_tim_generator.py b/tests/Translators/RoadConditions/test_tim_generator.py new file mode 100644 index 0000000..618824b --- /dev/null +++ b/tests/Translators/RoadConditions/test_tim_generator.py @@ -0,0 +1,61 @@ +from Translators.RoadConditions import tim_generator + +############################ getItisCodes ############################ +def test_get_itis_codes_no_conditions(): + feature = { + 'properties': { + 'currentConditions': [] + } + } + itisCodes = tim_generator.get_itis_codes(feature) + assert itisCodes == [] + +def test_get_itis_codes_single_condition(): + feature = { + 'properties': { + 'currentConditions': [ + {'conditionDescription': 'accident on right portion of road'} + ] + } + } + itisCodes = tim_generator.get_itis_codes(feature) + assert itisCodes == ['513'] # ItisCodes.ACCIDENT.value + +def test_get_itis_codes_multiple_conditions(): + feature = { + 'properties': { + 'currentConditions': [ + {'conditionDescription': 'Width limit in effect, accident on right portion of road. Keep to right.'} + ] + } + } + itisCodes = tim_generator.get_itis_codes(feature) + for code in itisCodes: + assert code in ['513', '2573', '7425'] + +def test_get_itis_codes_forecast_text_included(): + feature = { + 'properties': { + 'currentConditions': [ + {'conditionDescription': 'forecast text included, accident on right portion of road'} + ] + } + } + itisCodes = tim_generator.get_itis_codes(feature) + assert itisCodes == ['513'] # ItisCodes.ACCIDENT.value + +############################ getGeometry ############################ +def test_get_geometry_empty(): + geometry = [] + annotated_geometry = tim_generator.get_geometry(geometry) + assert annotated_geometry == [] + +def test_get_geometry_single_coordinate(): + geometry = [[-122.403, 37.795]] + annotated_geometry = tim_generator.get_geometry(geometry) + assert annotated_geometry == [{'latitude': 37.795, 'longitude': -122.403}] + +def test_get_geometry_multiple_coordinates(): + geometry = [[-122.403, 37.795], [-122.403, 37.800]] + annotated_geometry = tim_generator.get_geometry(geometry) + assert annotated_geometry == [{'latitude': 37.795, 'longitude': -122.403}, {'latitude': 37.800, 'longitude': -122.403}] From 6a015afcbea1c276d60a748c7165c0768df20cc0 Mon Sep 17 00:00:00 2001 From: Marc Wodahl <56242265+mwodahl@users.noreply.github.com> Date: Mon, 9 Dec 2024 13:36:15 -0700 Subject: [PATCH 04/17] Add sample.env, requirements.txt --- Translators/RoadConditions/requirements.txt | 11 +++++++++++ Translators/RoadConditions/sample.env | 10 ++++++++++ 2 files changed, 21 insertions(+) create mode 100644 Translators/RoadConditions/requirements.txt create mode 100644 Translators/RoadConditions/sample.env diff --git a/Translators/RoadConditions/requirements.txt b/Translators/RoadConditions/requirements.txt new file mode 100644 index 0000000..14bf813 --- /dev/null +++ b/Translators/RoadConditions/requirements.txt @@ -0,0 +1,11 @@ +functions-framework==3.4.0 +flask==2.0.2 +google-cloud-error-reporting==1.4.1 +pyproj==3.6.1 +requests +shapely==2.0.4 +sqlalchemy==1.4.22 +pg8000==1.29.6 +marshmallow +redis==4.6.0 +werkzeug==2.2.2 \ No newline at end of file diff --git a/Translators/RoadConditions/sample.env b/Translators/RoadConditions/sample.env new file mode 100644 index 0000000..c7d6223 --- /dev/null +++ b/Translators/RoadConditions/sample.env @@ -0,0 +1,10 @@ +dual_carriageway_endpoint=https://dtdapps.coloradodot.info/arcgis/rest/services/LRS/Routes_withDEC/MapServer/exts/CdotLrsAccessRounded +DB_HOST= +DB_NAME= +DB_PASS= +DB_USER= +LOGGING_LEVEL=INFO +CDOT_FEED_ENDPOINT= +CDOT_FEED_API_KEY= +ODE_ENDPOINT= +RUN_LOCAL= \ No newline at end of file From 789eec9d51b56095df9a37e59e98a4fed018e851 Mon Sep 17 00:00:00 2001 From: Marc Wodahl <56242265+mwodahl@users.noreply.github.com> Date: Mon, 9 Dec 2024 13:36:33 -0700 Subject: [PATCH 05/17] Update docker-compose.yml --- docker-compose.yml | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index bce768d..b7f0f36 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '1.0' - services: redis: image: redis-cache:latest @@ -36,6 +34,27 @@ services: options: max-size: '10m' max-file: '5' + Road_conditions_translator: + build: ./Translators/RoadConditions + image: rc_translator:latest + environment: + - dual_carriageway_endpoint=${dual_carriageway_endpoint} + - DB_HOST=${DB_HOST} + - DB_NAME=${DB_NAME} + - DB_PASS=${DB_PASS} + - DB_USER=${DB_USER} + - LOGGING_LEVEL=${LOGGING_LEVEL} + - CDOT_FEED_ENDPOINT=${CDOT_FEED_ENDPOINT} + - CDOT_FEED_API_KEY=${CDOT_FEED_API_KEY} + - DURATION_TIME=${DURATION_TIME} + - ODE_ENDPOINT=${ODE_ENDPOINT} + - RUN_LOCAL=${RUN_LOCAL} + ports: + - '8082:8082' + logging: + options: + max-size: '10m' + max-file: '5' volumes: redis-data: From 55a3d140e15a676736469c29d8009258189e5a95 Mon Sep 17 00:00:00 2001 From: Marc Wodahl <56242265+mwodahl@users.noreply.github.com> Date: Thu, 12 Dec 2024 13:49:07 -0700 Subject: [PATCH 06/17] Remove unnecessary print statement --- Translators/RoadConditions/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Translators/RoadConditions/main.py b/Translators/RoadConditions/main.py index 269e5a4..81974c2 100644 --- a/Translators/RoadConditions/main.py +++ b/Translators/RoadConditions/main.py @@ -119,7 +119,6 @@ def RC_tim_translator(): errNo = 0 return_value = requests.put(f'{os.getenv("ODE_ENDPOINT")}/submit-rc-ac', json=tim_all_clear_list) - print(f"All Clear Return: {return_value.status_code}") if (return_value.status_code == 200): logging.info(f'Successfully submitted {len(tim_list["timRcList"])} All Clear TIMs to ODE') From 3025a44b4dfeda88fbe0c9244652c51483c8577a Mon Sep 17 00:00:00 2001 From: Marc Wodahl <56242265+mwodahl@users.noreply.github.com> Date: Thu, 12 Dec 2024 13:57:53 -0700 Subject: [PATCH 07/17] Remove commented-out code --- Translators/RoadConditions/tim_generator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Translators/RoadConditions/tim_generator.py b/Translators/RoadConditions/tim_generator.py index b9fb68e..6c680cf 100644 --- a/Translators/RoadConditions/tim_generator.py +++ b/Translators/RoadConditions/tim_generator.py @@ -7,7 +7,6 @@ def get_itis_codes(feature): # need to iterate over entries & split to check for keywords for entry in feature["properties"]["currentConditions"]: itis_codes = entry["conditionDescription"].split(",") - # itis_codes[0] = itis_codes[0].split(" ")[-1] for code in itis_codes: code = code.translate({ord(k): None for k in digits}).replace("-", "").strip() if code == "forecast text included": From 6a1a17671030495f97d171970919dc7d27f5d61d Mon Sep 17 00:00:00 2001 From: Marc Wodahl <56242265+mwodahl@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:48:56 -0700 Subject: [PATCH 08/17] Move translate methods into tim_translator.py --- Translators/RoadConditions/main.py | 77 +------------------- Translators/RoadConditions/tim_translator.py | 71 ++++++++++++++++++ 2 files changed, 74 insertions(+), 74 deletions(-) create mode 100644 Translators/RoadConditions/tim_translator.py diff --git a/Translators/RoadConditions/main.py b/Translators/RoadConditions/main.py index 81974c2..bb4e9bc 100644 --- a/Translators/RoadConditions/main.py +++ b/Translators/RoadConditions/main.py @@ -3,85 +3,14 @@ import copy import logging import os -from pgquery import query_db -from tim_generator import get_geometry, get_itis_codes from flask import request, Flask -from datetime import datetime +from tim_translator import translate app = Flask(__name__) log_level = os.environ.get('LOGGING_LEVEL', 'INFO') logging.basicConfig(format='%(levelname)s:%(message)s', level=log_level) -def calculate_direction(coordinates): - try: - long_dif = coordinates[-1][0] - coordinates[0][0] - lat_dif = coordinates[-1][1] - coordinates[0][1] - except ValueError as e: - return "unknown" - except IndexError as e: - return "unknown" - - if abs(long_dif) > abs(lat_dif): - if long_dif > 0: - # eastbound - direction = "I" - else: - # westbound - direction = "D" - elif lat_dif > 0: - # northbound - direction = "I" - else: - # southbound - direction = "D" - return direction - -def translate(rc_geojson): - tims = {"timRcList": []} - - for feature in rc_geojson["features"]: - if (len(feature["geometry"]["coordinates"]) <= 2): - continue - tim_body = {} - tim_body["clientId"] = feature["properties"]["nameId"].replace("_", "-").replace("/", "-") - tim_body["direction"] = calculate_direction(feature['geometry']['coordinates']) - tim_body["segment"] = feature["properties"]["routeSegmentIndex"] - tim_body["route"] = feature["properties"]["routeName"].replace("_", "-") - tim_body["roadCode"] = feature["properties"]["nameId"].replace("_", "-") - tim_body["itisCodes"] = get_itis_codes(feature) - tim_body["geometry"] = get_geometry(feature["geometry"]["coordinates"]) - tim_body["advisory"] = ["3"] - active_tim_record = active_tim(feature, tim_body) - if active_tim_record: - logging.info(f"TIM already active for record: {tim_body['clientId']}") - continue - tims["timRcList"].append(tim_body) - return tims - -def active_tim(feature, tim_body): - tim_id = tim_body["clientId"] - # if TIM has an active TIM holding record that is current & info is the same as the current TIM record, then do not update - active_tim_holding = query_db(f"SELECT * FROM active_tim_holding WHERE client_id LIKE '%{tim_id}%'") - if len(active_tim_holding) > 0: - active_tim_holding = active_tim_holding[0] - return (active_tim_holding["direction"] == tim_body["direction"] and - f"{active_tim_holding['start_latitude']:.8f}" == f"{tim_body['geometry'][0]['latitude']:.8f}" and - f"{active_tim_holding['start_longitude']:.8f}" == f"{tim_body['geometry'][0]['longitude']:.8f}" and - f"{active_tim_holding['end_latitude']:.8f}" == f"{tim_body['geometry'][-1]['latitude']:.8f}" and - f"{active_tim_holding['end_longitude']:.8f}" == f"{tim_body['geometry'][-1]['longitude']:.8f}") - - # if TIM has an active TIM record that is current & info is the same as the current TIM record, then do not update - active_tim = query_db(f"SELECT * FROM active_tim WHERE client_id LIKE '%{tim_id}%' AND tim_type_id = (SELECT tim_type_id FROM tim_type WHERE type = 'RC') AND marked_for_deletion = false") - if len(active_tim) > 0: - active_tim = active_tim[0] - return (active_tim["direction"] == tim_body["direction"] and - f"{active_tim['start_latitude']:.8f}" == f"{tim_body['geometry'][0]['latitude']:.8f}" and - f"{active_tim['start_longitude']:.8f}" == f"{tim_body['geometry'][0]['longitude']:.8f}" and - f"{active_tim['end_latitude']:.8f}" == f"{tim_body['geometry'][-1]['latitude']:.8f}" and - f"{active_tim['end_longitude']:.8f}" == f"{tim_body['geometry'][-1]['longitude']:.8f}") - - @app.route('/', methods=['POST']) def entry(): if request.method == 'OPTIONS': @@ -118,12 +47,12 @@ def RC_tim_translator(): tim_list["timRcList"] = [tim for tim in tim_list["timRcList"] if len(tim["itisCodes"]) > 0] errNo = 0 - return_value = requests.put(f'{os.getenv("ODE_ENDPOINT")}/submit-rc-ac', json=tim_all_clear_list) + return_value = requests.put(f'{os.getenv("TIM_MANAGER_ENDPOINT")}/submit-rc-ac', json=tim_all_clear_list) if (return_value.status_code == 200): logging.info(f'Successfully submitted {len(tim_list["timRcList"])} All Clear TIMs to ODE') errNo = 0 - return_value = requests.post(f'{os.getenv("ODE_ENDPOINT")}/create-update-rc-tim', json=tim_list) + return_value = requests.post(f'{os.getenv("TIM_MANAGER_ENDPOINT")}/create-update-rc-tim', json=tim_list) if (return_value.status_code == 200): return f'Successfully pushed {len(tim_list["timRcList"])} TIMs to ODE' diff --git a/Translators/RoadConditions/tim_translator.py b/Translators/RoadConditions/tim_translator.py new file mode 100644 index 0000000..3bff414 --- /dev/null +++ b/Translators/RoadConditions/tim_translator.py @@ -0,0 +1,71 @@ +import logging +from pgquery import query_db +from tim_generator import get_geometry, get_itis_codes + +def calculate_direction(coordinates): + try: + long_dif = coordinates[-1][0] - coordinates[0][0] + lat_dif = coordinates[-1][1] - coordinates[0][1] + except ValueError as e: + return "unknown" + except IndexError as e: + return "unknown" + + if abs(long_dif) > abs(lat_dif): + if long_dif > 0: + # eastbound + direction = "I" + else: + # westbound + direction = "D" + elif lat_dif > 0: + # northbound + direction = "I" + else: + # southbound + direction = "D" + return direction + +def translate(rc_geojson): + tims = {"timRcList": []} + + for feature in rc_geojson["features"]: + if (len(feature["geometry"]["coordinates"]) <= 2): + continue + tim_body = {} + tim_body["clientId"] = feature["properties"]["nameId"].replace("_", "-").replace("/", "-") + tim_body["direction"] = calculate_direction(feature['geometry']['coordinates']) + tim_body["segment"] = feature["properties"]["routeSegmentIndex"] + tim_body["route"] = feature["properties"]["routeName"].replace("_", "-") + tim_body["roadCode"] = feature["properties"]["nameId"].replace("_", "-") + tim_body["itisCodes"] = get_itis_codes(feature) + tim_body["geometry"] = get_geometry(feature["geometry"]["coordinates"]) + tim_body["advisory"] = ["3"] + active_tim_record = active_tim(feature, tim_body) + if active_tim_record: + logging.info(f"TIM already active for record: {tim_body['clientId']}") + continue + tims["timRcList"].append(tim_body) + return tims + +def active_tim(feature, tim_body): + tim_id = tim_body["clientId"] + # if TIM has an active TIM holding record that is current & info is the same as the current TIM record, then do not update + active_tim_holding = query_db(f"SELECT * FROM active_tim_holding WHERE client_id LIKE '%{tim_id}%'") + if len(active_tim_holding) > 0: + active_tim_holding = active_tim_holding[0] + return (active_tim_holding["direction"] == tim_body["direction"] and + f"{active_tim_holding['start_latitude']:.8f}" == f"{tim_body['geometry'][0]['latitude']:.8f}" and + f"{active_tim_holding['start_longitude']:.8f}" == f"{tim_body['geometry'][0]['longitude']:.8f}" and + f"{active_tim_holding['end_latitude']:.8f}" == f"{tim_body['geometry'][-1]['latitude']:.8f}" and + f"{active_tim_holding['end_longitude']:.8f}" == f"{tim_body['geometry'][-1]['longitude']:.8f}") + + # if TIM has an active TIM record that is current & info is the same as the current TIM record, then do not update + active_tim = query_db(f"SELECT * FROM active_tim WHERE client_id LIKE '%{tim_id}%' AND tim_type_id = (SELECT tim_type_id FROM tim_type WHERE type = 'RC') AND marked_for_deletion = false") + if len(active_tim) > 0: + active_tim = active_tim[0] + return (active_tim["direction"] == tim_body["direction"] and + f"{active_tim['start_latitude']:.8f}" == f"{tim_body['geometry'][0]['latitude']:.8f}" and + f"{active_tim['start_longitude']:.8f}" == f"{tim_body['geometry'][0]['longitude']:.8f}" and + f"{active_tim['end_latitude']:.8f}" == f"{tim_body['geometry'][-1]['latitude']:.8f}" and + f"{active_tim['end_longitude']:.8f}" == f"{tim_body['geometry'][-1]['longitude']:.8f}") From 4e364b82b5ba35ab659bc2b8346c0f9518e4b6fe Mon Sep 17 00:00:00 2001 From: Marc Wodahl <56242265+mwodahl@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:49:16 -0700 Subject: [PATCH 09/17] Add tim_translator unit tests --- tests/Translators/RoadConditions/__init__.py | 4 +- .../RoadConditions/test_tim_translator.py | 60 +++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 tests/Translators/RoadConditions/test_tim_translator.py diff --git a/tests/Translators/RoadConditions/__init__.py b/tests/Translators/RoadConditions/__init__.py index c44218b..24b6775 100644 --- a/tests/Translators/RoadConditions/__init__.py +++ b/tests/Translators/RoadConditions/__init__.py @@ -1,6 +1,6 @@ import sys import os -# Add the WZDx directory to the path so that relative imports work +# Add the RoadConditions directory to the path so that relative imports work SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.join(SCRIPT_DIR, '..', '..', '..', 'Translators', 'WZDx')) \ No newline at end of file +sys.path.append(os.path.join(SCRIPT_DIR, '..', '..', '..', 'Translators', 'RoadConditions')) \ No newline at end of file diff --git a/tests/Translators/RoadConditions/test_tim_translator.py b/tests/Translators/RoadConditions/test_tim_translator.py new file mode 100644 index 0000000..e7c3f28 --- /dev/null +++ b/tests/Translators/RoadConditions/test_tim_translator.py @@ -0,0 +1,60 @@ +from unittest.mock import patch +import Translators.RoadConditions.tim_translator as tim_translator + +def test_calculate_direction_eastbound(): + coordinates = [(0, 0), (1, 0)] + assert tim_translator.calculate_direction(coordinates) == "I" + +def test_calculate_direction_westbound(): + coordinates = [(1, 0), (0, 0)] + assert tim_translator.calculate_direction(coordinates) == "D" + +def test_calculate_direction_northbound(): + coordinates = [(0, 0), (0, 1)] + assert tim_translator.calculate_direction(coordinates) == "I" + +def test_calculate_direction_southbound(): + coordinates = [(0, 1), (0, 0)] + assert tim_translator.calculate_direction(coordinates) == "D" + +@patch('Translators.RoadConditions.tim_translator.get_itis_codes', return_value=[1, 2, 3]) +@patch('Translators.RoadConditions.tim_translator.query_db', return_value=[]) +def test_translate(mock_query_db, mock_get_itis_codes): + rc_geojson = { + "features": [ + { + "geometry": { + "coordinates": [(0, 0), (1, 0), (2, 0)] + }, + "properties": { + "nameId": "test_name", + "routeSegmentIndex": 1, + "routeName": "test_route" + } + } + ] + } + expected_output = { + "timRcList": [ + { + "clientId": "test-name", + "direction": "I", + "segment": 1, + "route": "test-route", + "roadCode": "test-name", + "itisCodes":[1, 2, 3], + "geometry": [{ + "latitude": 0, + "longitude": 0 + }, { + "latitude": 0, + "longitude": 1 + }, { + "latitude": 0, + "longitude": 2 + }], + "advisory": ["3"] + } + ] + } + assert tim_translator.translate(rc_geojson) == expected_output \ No newline at end of file From fa333e466d6c3abedf9a39a49f2a93c7fd55aaa1 Mon Sep 17 00:00:00 2001 From: Marc Wodahl <56242265+mwodahl@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:53:27 -0700 Subject: [PATCH 10/17] Update ODE_ENDPOINT env var to be TIM _MANAGER_ENDPOINT --- Translators/RoadConditions/README.md | 2 +- Translators/RoadConditions/sample.env | 2 +- docker-compose.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Translators/RoadConditions/README.md b/Translators/RoadConditions/README.md index eae30f8..f5644a0 100644 --- a/Translators/RoadConditions/README.md +++ b/Translators/RoadConditions/README.md @@ -31,7 +31,7 @@ In addition to the environment variables for accessing the CDOT Postgres databas
  1. CDOT_FEED_ENDPOINT - the CDOT Data Feed URL
  2. CDOT_FEED_API_KEY - the API key for accessing the CDOT Road Conditions data
    3. -
  3. ODE_ENDPOINT - the ODE URL where translated TIMs will be submitted
    4. +
  4. TIM_MANAGER_ENDPOINT - the TIM Manager URL where translated TIMs will be submitted
### Testing diff --git a/Translators/RoadConditions/sample.env b/Translators/RoadConditions/sample.env index c7d6223..4822fe3 100644 --- a/Translators/RoadConditions/sample.env +++ b/Translators/RoadConditions/sample.env @@ -6,5 +6,5 @@ DB_USER= LOGGING_LEVEL=INFO CDOT_FEED_ENDPOINT= CDOT_FEED_API_KEY= -ODE_ENDPOINT= +TIM_MANAGER_ENDPOINT= RUN_LOCAL= \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index b7f0f36..e1f8563 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,7 +26,7 @@ services: - REDIS_PORT=${REDIS_PORT} - REDIS_PASS=${REDIS_PASS} - DURATION_TIME=${DURATION_TIME} - - ODE_ENDPOINT=${ODE_ENDPOINT} + - TIM_MANAGER_ENDPOINT=${TIM_MANAGER_ENDPOINT} - RUN_LOCAL=${RUN_LOCAL} ports: - '8081:8081' @@ -47,7 +47,7 @@ services: - CDOT_FEED_ENDPOINT=${CDOT_FEED_ENDPOINT} - CDOT_FEED_API_KEY=${CDOT_FEED_API_KEY} - DURATION_TIME=${DURATION_TIME} - - ODE_ENDPOINT=${ODE_ENDPOINT} + - TIM_MANAGER_ENDPOINT=${TIM_MANAGER_ENDPOINT} - RUN_LOCAL=${RUN_LOCAL} ports: - '8082:8082' From 78721295dd1ba508a16f6f611cc5ad69835d08aa Mon Sep 17 00:00:00 2001 From: Marc Wodahl <56242265+mwodahl@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:53:46 -0700 Subject: [PATCH 11/17] Reorder Dockerfile steps --- Translators/RoadConditions/Dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Translators/RoadConditions/Dockerfile b/Translators/RoadConditions/Dockerfile index f1dc42e..9b14df8 100644 --- a/Translators/RoadConditions/Dockerfile +++ b/Translators/RoadConditions/Dockerfile @@ -6,12 +6,14 @@ ENV WORKDIR /app WORKDIR $WORKDIR -COPY . . - -ENV FLASK_APP app/main.py - RUN apt-get update && apt-get install -y libgeos-dev +COPY ./requirements.txt . + RUN pip install -r requirements.txt +COPY . . + +ENV FLASK_APP app/main.py + CMD ["python3", "-m", "flask", "run", "--host", "0.0.0.0", "--port", "8082"] \ No newline at end of file From 779287909163ad152f6de3669ab0eb44dbf9e88c Mon Sep 17 00:00:00 2001 From: Marc Wodahl <56242265+mwodahl@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:53:55 -0700 Subject: [PATCH 12/17] Remove references to ODE in main --- Translators/RoadConditions/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Translators/RoadConditions/main.py b/Translators/RoadConditions/main.py index bb4e9bc..73e77f7 100644 --- a/Translators/RoadConditions/main.py +++ b/Translators/RoadConditions/main.py @@ -40,7 +40,7 @@ def RC_tim_translator(): tim_list = translate(geoJSON) - logging.info('Pushing TIMs to ODE...') + logging.info('Pushing TIMs to TIM Manager...') tim_list_copy = copy.deepcopy(tim_list) tim_all_clear_list = {"timRcList": [tim for tim in tim_list_copy["timRcList"] if len(tim["itisCodes"]) == 0]} @@ -49,14 +49,14 @@ def RC_tim_translator(): errNo = 0 return_value = requests.put(f'{os.getenv("TIM_MANAGER_ENDPOINT")}/submit-rc-ac', json=tim_all_clear_list) if (return_value.status_code == 200): - logging.info(f'Successfully submitted {len(tim_list["timRcList"])} All Clear TIMs to ODE') + logging.info(f'Successfully submitted {len(tim_list["timRcList"])} All Clear TIMs to TIM Manager') errNo = 0 return_value = requests.post(f'{os.getenv("TIM_MANAGER_ENDPOINT")}/create-update-rc-tim', json=tim_list) if (return_value.status_code == 200): - return f'Successfully pushed {len(tim_list["timRcList"])} TIMs to ODE' + return f'Successfully pushed {len(tim_list["timRcList"])} TIMs to TIM Manager' - return f'Error pushing TIMs to ODE: {return_value.content}' + return f'Error pushing TIMs to TIM Manager: {return_value.content}' # Run via flask app if running locally else just run translator directly if (os.getenv("RUN_LOCAL") == "true"): From e77cd3b3c7b4790d81af5a86da717adbb6bdc1f8 Mon Sep 17 00:00:00 2001 From: Marc Wodahl <56242265+mwodahl@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:26:04 -0700 Subject: [PATCH 13/17] Remove unused variable, fix logging statement --- Translators/RoadConditions/main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Translators/RoadConditions/main.py b/Translators/RoadConditions/main.py index 73e77f7..18d11e3 100644 --- a/Translators/RoadConditions/main.py +++ b/Translators/RoadConditions/main.py @@ -46,12 +46,10 @@ def RC_tim_translator(): tim_all_clear_list = {"timRcList": [tim for tim in tim_list_copy["timRcList"] if len(tim["itisCodes"]) == 0]} tim_list["timRcList"] = [tim for tim in tim_list["timRcList"] if len(tim["itisCodes"]) > 0] - errNo = 0 return_value = requests.put(f'{os.getenv("TIM_MANAGER_ENDPOINT")}/submit-rc-ac', json=tim_all_clear_list) if (return_value.status_code == 200): - logging.info(f'Successfully submitted {len(tim_list["timRcList"])} All Clear TIMs to TIM Manager') + logging.info(f'Successfully submitted {len(tim_all_clear_list["timRcList"])} All Clear TIMs to TIM Manager') - errNo = 0 return_value = requests.post(f'{os.getenv("TIM_MANAGER_ENDPOINT")}/create-update-rc-tim', json=tim_list) if (return_value.status_code == 200): return f'Successfully pushed {len(tim_list["timRcList"])} TIMs to TIM Manager' From 992b6761e98cd36696eed8c0c7b50d21c6492bca Mon Sep 17 00:00:00 2001 From: Marc Wodahl <56242265+mwodahl@users.noreply.github.com> Date: Tue, 17 Dec 2024 14:26:27 -0700 Subject: [PATCH 14/17] Update testing ITIS codes to use enum instead of string values --- .../Translators/RoadConditions/test_tim_generator.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/Translators/RoadConditions/test_tim_generator.py b/tests/Translators/RoadConditions/test_tim_generator.py index 618824b..75dc47d 100644 --- a/tests/Translators/RoadConditions/test_tim_generator.py +++ b/tests/Translators/RoadConditions/test_tim_generator.py @@ -1,4 +1,5 @@ from Translators.RoadConditions import tim_generator +from Translators.RoadConditions import itis_codes ############################ getItisCodes ############################ def test_get_itis_codes_no_conditions(): @@ -19,7 +20,7 @@ def test_get_itis_codes_single_condition(): } } itisCodes = tim_generator.get_itis_codes(feature) - assert itisCodes == ['513'] # ItisCodes.ACCIDENT.value + assert itisCodes == [itis_codes.ItisCodes.ACCIDENT.value] def test_get_itis_codes_multiple_conditions(): feature = { @@ -31,7 +32,11 @@ def test_get_itis_codes_multiple_conditions(): } itisCodes = tim_generator.get_itis_codes(feature) for code in itisCodes: - assert code in ['513', '2573', '7425'] + assert code in [ + itis_codes.ItisCodes.ACCIDENT.value, + itis_codes.ItisCodes.WIDTH_LIMIT.value, + itis_codes.ItisCodes.KEEP_TO_RIGHT.value + ] def test_get_itis_codes_forecast_text_included(): feature = { @@ -42,7 +47,7 @@ def test_get_itis_codes_forecast_text_included(): } } itisCodes = tim_generator.get_itis_codes(feature) - assert itisCodes == ['513'] # ItisCodes.ACCIDENT.value + assert itisCodes == [itis_codes.ItisCodes.ACCIDENT.value] ############################ getGeometry ############################ def test_get_geometry_empty(): From 0f9d94c15119f30c57a2d8f46f448055f80e6161 Mon Sep 17 00:00:00 2001 From: Marc Wodahl <56242265+mwodahl@users.noreply.github.com> Date: Thu, 19 Dec 2024 11:56:31 -0700 Subject: [PATCH 15/17] Add RCFeature class --- Translators/RoadConditions/tim_generator.py | 2 +- Translators/RoadConditions/tim_translator.py | 47 ++++++++++++++++---- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/Translators/RoadConditions/tim_generator.py b/Translators/RoadConditions/tim_generator.py index 6c680cf..a8b7c58 100644 --- a/Translators/RoadConditions/tim_generator.py +++ b/Translators/RoadConditions/tim_generator.py @@ -5,7 +5,7 @@ def get_itis_codes(feature): itisCodes = [] # need to iterate over entries & split to check for keywords - for entry in feature["properties"]["currentConditions"]: + for entry in feature.get_current_conditions(): itis_codes = entry["conditionDescription"].split(",") for code in itis_codes: code = code.translate({ord(k): None for k in digits}).replace("-", "").strip() diff --git a/Translators/RoadConditions/tim_translator.py b/Translators/RoadConditions/tim_translator.py index 3bff414..cd7b64b 100644 --- a/Translators/RoadConditions/tim_translator.py +++ b/Translators/RoadConditions/tim_translator.py @@ -1,7 +1,34 @@ +from datetime import datetime import logging from pgquery import query_db from tim_generator import get_geometry, get_itis_codes +class RCFeature: + def __init__(self, properties, geometry): + self.name_id = properties["nameId"].replace("_", "-") + self.route = properties["routeName"].replace("_", "-") + self.segment = properties["routeSegmentIndex"] + self.current_conditions = properties["currentConditions"] + self.geometry = geometry + + def get_name_id(self): + return self.name_id + + def get_client_id(self): + return self.name_id.replace("/", "-") + + def get_route(self): + return self.route + + def get_route_segment(self): + return self.segment + + def get_current_conditions(self): + return self.current_conditions + + def get_geometry(self): + return self.geometry + def calculate_direction(coordinates): try: long_dif = coordinates[-1][0] - coordinates[0][0] @@ -30,25 +57,27 @@ def translate(rc_geojson): tims = {"timRcList": []} for feature in rc_geojson["features"]: - if (len(feature["geometry"]["coordinates"]) <= 2): + feature = RCFeature(feature["properties"], feature["geometry"]["coordinates"]) + if (len(feature.get_geometry()) <= 2): continue tim_body = {} - tim_body["clientId"] = feature["properties"]["nameId"].replace("_", "-").replace("/", "-") - tim_body["direction"] = calculate_direction(feature['geometry']['coordinates']) - tim_body["segment"] = feature["properties"]["routeSegmentIndex"] - tim_body["route"] = feature["properties"]["routeName"].replace("_", "-") - tim_body["roadCode"] = feature["properties"]["nameId"].replace("_", "-") + tim_body["clientId"] = feature.get_client_id() + tim_body["direction"] = calculate_direction(feature.get_geometry()) + tim_body["segment"] = feature.get_route_segment() + tim_body["route"] = feature.get_route() + tim_body["roadCode"] = feature.get_name_id() tim_body["itisCodes"] = get_itis_codes(feature) - tim_body["geometry"] = get_geometry(feature["geometry"]["coordinates"]) + tim_body["geometry"] = get_geometry(feature.get_geometry()) tim_body["advisory"] = ["3"] - active_tim_record = active_tim(feature, tim_body) + active_tim_record = active_tim(tim_body) if active_tim_record: logging.info(f"TIM already active for record: {tim_body['clientId']}") continue tims["timRcList"].append(tim_body) + print(tims) return tims -def active_tim(feature, tim_body): +def active_tim(tim_body): tim_id = tim_body["clientId"] # if TIM has an active TIM holding record that is current & info is the same as the current TIM record, then do not update active_tim_holding = query_db(f"SELECT * FROM active_tim_holding WHERE client_id LIKE '%{tim_id}%'") From 87ab2182610e1d7bd6afdcf568c60878ea5dd695 Mon Sep 17 00:00:00 2001 From: Marc Wodahl <56242265+mwodahl@users.noreply.github.com> Date: Thu, 19 Dec 2024 11:57:01 -0700 Subject: [PATCH 16/17] RCFeature unit testing updates --- .../RoadConditions/test_tim_generator.py | 69 ++++++++++++------- .../RoadConditions/test_tim_translator.py | 3 +- 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/tests/Translators/RoadConditions/test_tim_generator.py b/tests/Translators/RoadConditions/test_tim_generator.py index 75dc47d..db1a95b 100644 --- a/tests/Translators/RoadConditions/test_tim_generator.py +++ b/tests/Translators/RoadConditions/test_tim_generator.py @@ -1,35 +1,51 @@ -from Translators.RoadConditions import tim_generator -from Translators.RoadConditions import itis_codes +from Translators.RoadConditions import tim_generator, itis_codes +from Translators.RoadConditions.tim_translator import RCFeature ############################ getItisCodes ############################ def test_get_itis_codes_no_conditions(): - feature = { - 'properties': { - 'currentConditions': [] - } + properties = { + 'currentConditions': [], + 'nameId': "I-80", + 'routeSegmentIndex': 1, + "routeName": "I-80" } + geometry = { + 'coordinates': [[-122.403, 37.795], [-122.403, 37.800]] + } + feature = RCFeature(properties, geometry) itisCodes = tim_generator.get_itis_codes(feature) assert itisCodes == [] def test_get_itis_codes_single_condition(): - feature = { - 'properties': { - 'currentConditions': [ - {'conditionDescription': 'accident on right portion of road'} - ] - } + properties = { + 'currentConditions': [ + {'conditionDescription': 'accident on right portion of road'} + ], + 'nameId': "I-80", + 'routeSegmentIndex': 1, + "routeName": "I-80" } + geometry = { + 'coordinates': [[-122.403, 37.795], [-122.403, 37.800]] + } + feature = RCFeature(properties, geometry) itisCodes = tim_generator.get_itis_codes(feature) assert itisCodes == [itis_codes.ItisCodes.ACCIDENT.value] def test_get_itis_codes_multiple_conditions(): - feature = { - 'properties': { - 'currentConditions': [ - {'conditionDescription': 'Width limit in effect, accident on right portion of road. Keep to right.'} - ] - } + properties = { + 'currentConditions': [ + {'conditionDescription': 'Width limit in effect, accident on right portion of road. Keep to right.'} + ], + 'nameId': "I-80", + 'routeSegmentIndex': 1, + "routeName": "I-80" } + geometry = { + 'coordinates': [[-122.403, 37.795], [-122.403, 37.800]] + } + feature = RCFeature(properties, geometry) + itisCodes = tim_generator.get_itis_codes(feature) for code in itisCodes: assert code in [ @@ -39,13 +55,18 @@ def test_get_itis_codes_multiple_conditions(): ] def test_get_itis_codes_forecast_text_included(): - feature = { - 'properties': { - 'currentConditions': [ - {'conditionDescription': 'forecast text included, accident on right portion of road'} - ] - } + properties = { + 'currentConditions': [ + {'conditionDescription': 'forecast text included, accident on right portion of road'} + ], + 'nameId': "I-80", + 'routeSegmentIndex': 1, + "routeName": "I-80" + } + geometry = { + 'coordinates': [[-122.403, 37.795], [-122.403, 37.800]] } + feature = RCFeature(properties, geometry) itisCodes = tim_generator.get_itis_codes(feature) assert itisCodes == [itis_codes.ItisCodes.ACCIDENT.value] diff --git a/tests/Translators/RoadConditions/test_tim_translator.py b/tests/Translators/RoadConditions/test_tim_translator.py index e7c3f28..a3217f4 100644 --- a/tests/Translators/RoadConditions/test_tim_translator.py +++ b/tests/Translators/RoadConditions/test_tim_translator.py @@ -29,7 +29,8 @@ def test_translate(mock_query_db, mock_get_itis_codes): "properties": { "nameId": "test_name", "routeSegmentIndex": 1, - "routeName": "test_route" + "routeName": "test_route", + "currentConditions": [] } } ] From 8d059ebd834eb8d6421e26ec5c45b131cc7eb4cc Mon Sep 17 00:00:00 2001 From: Marc Wodahl <56242265+mwodahl@users.noreply.github.com> Date: Fri, 3 Jan 2025 12:25:57 -0700 Subject: [PATCH 17/17] Remove unnecessary print statement --- Translators/RoadConditions/tim_translator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Translators/RoadConditions/tim_translator.py b/Translators/RoadConditions/tim_translator.py index cd7b64b..122edaa 100644 --- a/Translators/RoadConditions/tim_translator.py +++ b/Translators/RoadConditions/tim_translator.py @@ -74,7 +74,6 @@ def translate(rc_geojson): logging.info(f"TIM already active for record: {tim_body['clientId']}") continue tims["timRcList"].append(tim_body) - print(tims) return tims def active_tim(tim_body):