From 36d4fd30c548128efaf0d560f94fd81b7e57f8ec Mon Sep 17 00:00:00 2001 From: Guillaume BISCH Date: Fri, 27 Mar 2020 19:56:01 +0100 Subject: [PATCH] add db rollback and try blocks for database update in tasks --- lazyblacksmith/__init__.py | 2 +- lbtasks/tasks/blueprint/character.py | 6 ++- lbtasks/tasks/blueprint/corporation.py | 4 +- lbtasks/tasks/character/skills.py | 13 +++++-- lbtasks/tasks/schedule/purge.py | 47 +++++++++++++---------- lbtasks/tasks/universe/adjusted_prices.py | 13 +++++-- lbtasks/tasks/universe/indexes.py | 19 +++++---- lbtasks/tasks/universe/market_order.py | 5 +-- 8 files changed, 68 insertions(+), 41 deletions(-) diff --git a/lazyblacksmith/__init__.py b/lazyblacksmith/__init__.py index 48566a8..a8e7668 100644 --- a/lazyblacksmith/__init__.py +++ b/lazyblacksmith/__init__.py @@ -1,2 +1,2 @@ # -*- encoding: utf-8 -*- -__version__ = 1.0 +__version__ = 1.0.2 diff --git a/lbtasks/tasks/blueprint/character.py b/lbtasks/tasks/blueprint/character.py index 50c3c85..c469236 100644 --- a/lbtasks/tasks/blueprint/character.py +++ b/lbtasks/tasks/blueprint/character.py @@ -111,7 +111,8 @@ def task_update_character_blueprints(character_id): db.session.add(blueprints[key]) db.session.commit() except SQLAlchemyError: - logger.exception( + db.session.rollback() + logger.error( "Error while trying to add blueprint id: %d", item_id ) @@ -127,7 +128,8 @@ def task_update_character_blueprints(character_id): try: db.session.commit() except SQLAlchemyError: - logger.exception( + db.session.rollback() + logger.error( "Error while trying to delete unused blueprints" ) diff --git a/lbtasks/tasks/blueprint/corporation.py b/lbtasks/tasks/blueprint/corporation.py index ab85bd7..79d7d98 100644 --- a/lbtasks/tasks/blueprint/corporation.py +++ b/lbtasks/tasks/blueprint/corporation.py @@ -130,7 +130,8 @@ def task_update_corporation_blueprints(character_id): db.session.add(blueprints[key]) db.session.commit() except SQLAlchemyError: - logger.exception( + db.session.rollback() + logger.error( "Error while trying to add blueprint id: %d", item_id ) @@ -145,6 +146,7 @@ def task_update_corporation_blueprints(character_id): try: db.session.commit() except SQLAlchemyError: + db.session.rollback() logger.exception( "Error while trying to delete unused blueprints" ) diff --git a/lbtasks/tasks/character/skills.py b/lbtasks/tasks/character/skills.py index b1249c4..9cc0b3e 100644 --- a/lbtasks/tasks/character/skills.py +++ b/lbtasks/tasks/character/skills.py @@ -1,5 +1,6 @@ # -*- encoding: utf-8 -*- """ Update character skills """ +from sqlalchemy.exc import SQLAlchemyError from lazyblacksmith.extension.esipy import esiclient from lazyblacksmith.extension.esipy.operations import get_characters_skills from lazyblacksmith.models import Skill, TokenScope, User, db @@ -7,7 +8,7 @@ inc_fail_token_scope, update_token_state) -from ... import celery_app +from ... import celery_app, logger @celery_app.task(name="update_character_skill") @@ -45,8 +46,14 @@ def task_update_character_skills(character_id): ) db.session.merge(skill) - db.session.commit() - update_token_state(token, character_skills.header['Expires'][0]) + try: + db.session.commit() + update_token_state(token, character_skills.header['Expires'][0]) + except SQLAlchemyError: + db.session.rollback() + logger.exception( + "Error while to commit skills" + ) else: inc_fail_token_scope(token, character_skills.status) diff --git a/lbtasks/tasks/schedule/purge.py b/lbtasks/tasks/schedule/purge.py index f8257ad..9cb9518 100644 --- a/lbtasks/tasks/schedule/purge.py +++ b/lbtasks/tasks/schedule/purge.py @@ -1,36 +1,43 @@ # -*- encoding: utf-8 -*- """ Purging task """ import datetime +from sqlalchemy.exc import SQLAlchemyError import config from lazyblacksmith.models import ItemPrice, TokenScope, db from lazyblacksmith.utils.time import utcnow -from ... import celery_app +from ... import celery_app, logger @celery_app.task(name="schedule.purge") def task_purge(): """ Purge all old stuff everywhere. """ # purge all tokens people never updated - purge_date_limit = ( - utcnow() - datetime.timedelta(days=config.PURGE_INVALID_TOKENS) - ) - TokenScope.query.filter_by( - valid=False - ).filter( - ( - (TokenScope.last_update.is_(None)) & - (TokenScope.updated_at <= purge_date_limit) - ) | ( - TokenScope.last_update < purge_date_limit + + try: + purge_date_limit = ( + utcnow() - datetime.timedelta(days=config.PURGE_INVALID_TOKENS) + ) + TokenScope.query.filter_by( + valid=False + ).filter( + ( + (TokenScope.last_update.is_(None)) & + (TokenScope.updated_at <= purge_date_limit) + ) | ( + TokenScope.last_update < purge_date_limit + ) + ).delete() + db.session.commit() + + # purge old market data + purge_date_limit = ( + utcnow() - datetime.timedelta(days=config.PURGE_OLD_PRICES) ) - ).delete() - db.session.commit() + ItemPrice.query.filter(ItemPrice.updated_at < purge_date_limit).delete() + db.session.commit() - # purge old market data - purge_date_limit = ( - utcnow() - datetime.timedelta(days=config.PURGE_OLD_PRICES) - ) - ItemPrice.query.filter(ItemPrice.updated_at < purge_date_limit).delete() - db.session.commit() + except SQLAlchemyError: + db.session.rollback() + logger.exception("Error while trying to purge data") diff --git a/lbtasks/tasks/universe/adjusted_prices.py b/lbtasks/tasks/universe/adjusted_prices.py index 456660d..007c6d2 100644 --- a/lbtasks/tasks/universe/adjusted_prices.py +++ b/lbtasks/tasks/universe/adjusted_prices.py @@ -1,11 +1,12 @@ # -*- encoding: utf-8 -*- """ Market adjusted prices tasks """ +from sqlalchemy.exc import SQLAlchemyError from lazyblacksmith.extension.esipy import esiclient from lazyblacksmith.extension.esipy.operations import get_markets_prices from lazyblacksmith.models import Item, ItemAdjustedPrice, db from lazyblacksmith.models.enums import ActivityEnum -from ... import celery_app +from ... import celery_app, logger @celery_app.task(name="universe.adjusted_price") @@ -13,9 +14,13 @@ def task_adjusted_price_base_cost(): """Task that update the adjusted prices from the API then calculate the base cost for every blueprints. """ - prices = update_adjusted_price() - if prices is not None: - update_base_costs(prices) + try: + prices = update_adjusted_price() + if prices is not None: + update_base_costs(prices) + except SQLAlchemyError: + db.session.rollback() + logger.exception("Error while trying to update adjusted prices") def update_adjusted_price(): diff --git a/lbtasks/tasks/universe/indexes.py b/lbtasks/tasks/universe/indexes.py index 073d8f8..ce06e81 100644 --- a/lbtasks/tasks/universe/indexes.py +++ b/lbtasks/tasks/universe/indexes.py @@ -1,11 +1,12 @@ # -*- encoding: utf-8 -*- """ Market indexes task """ +from sqlalchemy.exc import SQLAlchemyError from lazyblacksmith.extension.esipy import esiclient from lazyblacksmith.extension.esipy.operations import get_industry_systems from lazyblacksmith.models import IndustryIndex from lazyblacksmith.models import db -from ... import celery_app +from ... import celery_app, logger @celery_app.task(name="universe.industry_indexes") @@ -27,9 +28,13 @@ def task_industry_indexes(): row['cost_index'] = activity_index.cost_index insert_index_list.append(row) - db.engine.execute("TRUNCATE TABLE %s" % IndustryIndex.__tablename__) - db.engine.execute( - IndustryIndex.__table__.insert(), - insert_index_list - ) - db.session.commit() + try: + db.engine.execute("TRUNCATE TABLE %s" % IndustryIndex.__tablename__) + db.engine.execute( + IndustryIndex.__table__.insert(), + insert_index_list + ) + db.session.commit() + except SQLAlchemyError: + db.session.rollback() + logger.exception("Error while updating indexes") diff --git a/lbtasks/tasks/universe/market_order.py b/lbtasks/tasks/universe/market_order.py index 33b0bd4..ab05bb2 100644 --- a/lbtasks/tasks/universe/market_order.py +++ b/lbtasks/tasks/universe/market_order.py @@ -108,9 +108,8 @@ def task_update_region_order_price(region_id, item_id_list): save_item_prices(item_list) except SQLAlchemyError as exc: - logger.error( - 'Something went wrong while trying to insert/update data: %s', - str(exc) + logger.exception( + 'Something went wrong while trying to insert/update data:', ) db.session.rollback()