Merge pull request #78 from Kyria/develop

add db rollback and try blocks for database update in tasks

lazyblacksmith/__init__.py

@@ -1,2 +1,2 @@
 # -*- encoding: utf-8 -*-
-__version__ = 1.0
+__version__ = 1.0.2

lbtasks/tasks/blueprint/character.py

@@ -111,7 +111,8 @@ def task_update_character_blueprints(character_id):
                     db.session.add(blueprints[key])
                     db.session.commit()
                 except SQLAlchemyError:
-                    logger.exception(
+                    db.session.rollback()
+                    logger.error(
                         "Error while trying to add blueprint id: %d",
                         item_id
                     )
@@ -127,7 +128,8 @@ def task_update_character_blueprints(character_id):
     try:
         db.session.commit()
     except SQLAlchemyError:
-        logger.exception(
+        db.session.rollback()
+        logger.error(
             "Error while trying to delete unused blueprints"
         )
 

lbtasks/tasks/blueprint/corporation.py

@@ -130,7 +130,8 @@ def task_update_corporation_blueprints(character_id):
                     db.session.add(blueprints[key])
                     db.session.commit()
                 except SQLAlchemyError:
-                    logger.exception(
+                    db.session.rollback()
+                    logger.error(
                         "Error while trying to add blueprint id: %d",
                         item_id
                     )
@@ -145,6 +146,7 @@ def task_update_corporation_blueprints(character_id):
     try:
         db.session.commit()
     except SQLAlchemyError:
+        db.session.rollback()
         logger.exception(
             "Error while trying to delete unused blueprints"
         )

lbtasks/tasks/character/skills.py

@@ -1,13 +1,14 @@
 # -*- encoding: utf-8 -*-
 """ Update character skills """
+from sqlalchemy.exc import SQLAlchemyError
 from lazyblacksmith.extension.esipy import esiclient
 from lazyblacksmith.extension.esipy.operations import get_characters_skills
 from lazyblacksmith.models import Skill, TokenScope, User, db
 from lazyblacksmith.utils.models import (get_token_update_esipy,
                                          inc_fail_token_scope,
                                          update_token_state)
 
-from ... import celery_app
+from ... import celery_app, logger
 
 
 @celery_app.task(name="update_character_skill")
@@ -45,8 +46,14 @@ def task_update_character_skills(character_id):
                 )
                 db.session.merge(skill)
 
-        db.session.commit()
-        update_token_state(token, character_skills.header['Expires'][0])
+        try:
+            db.session.commit()
+            update_token_state(token, character_skills.header['Expires'][0])
+        except SQLAlchemyError:
+            db.session.rollback()
+            logger.exception(
+                "Error while to commit skills"
+            )
 
     else:
         inc_fail_token_scope(token, character_skills.status)

lbtasks/tasks/schedule/purge.py

@@ -1,36 +1,43 @@
 # -*- encoding: utf-8 -*-
 """ Purging task """
 import datetime
+from sqlalchemy.exc import SQLAlchemyError
 
 import config
 from lazyblacksmith.models import ItemPrice, TokenScope, db
 from lazyblacksmith.utils.time import utcnow
 
-from ... import celery_app
+from ... import celery_app, logger
 
 
 @celery_app.task(name="schedule.purge")
 def task_purge():
     """ Purge all old stuff everywhere. """
     # purge all tokens people never updated
-    purge_date_limit = (
-        utcnow() - datetime.timedelta(days=config.PURGE_INVALID_TOKENS)
-    )
-    TokenScope.query.filter_by(
-        valid=False
-    ).filter(
-        (
-            (TokenScope.last_update.is_(None)) &
-            (TokenScope.updated_at <= purge_date_limit)
-        ) | (
-            TokenScope.last_update < purge_date_limit
+
+    try:
+        purge_date_limit = (
+            utcnow() - datetime.timedelta(days=config.PURGE_INVALID_TOKENS)
+        )
+        TokenScope.query.filter_by(
+            valid=False
+        ).filter(
+            (
+                (TokenScope.last_update.is_(None)) &
+                (TokenScope.updated_at <= purge_date_limit)
+            ) | (
+                TokenScope.last_update < purge_date_limit
+            )
+        ).delete()
+        db.session.commit()
+
+        # purge old market data
+        purge_date_limit = (
+            utcnow() - datetime.timedelta(days=config.PURGE_OLD_PRICES)
         )
-    ).delete()
-    db.session.commit()
+        ItemPrice.query.filter(ItemPrice.updated_at < purge_date_limit).delete()
+        db.session.commit()
 
-    # purge old market data
-    purge_date_limit = (
-        utcnow() - datetime.timedelta(days=config.PURGE_OLD_PRICES)
-    )
-    ItemPrice.query.filter(ItemPrice.updated_at < purge_date_limit).delete()
-    db.session.commit()
+    except SQLAlchemyError:
+        db.session.rollback()
+        logger.exception("Error while trying to purge data")

lbtasks/tasks/universe/adjusted_prices.py

@@ -1,21 +1,26 @@
 # -*- encoding: utf-8 -*-
 """ Market adjusted prices tasks """
+from sqlalchemy.exc import SQLAlchemyError
 from lazyblacksmith.extension.esipy import esiclient
 from lazyblacksmith.extension.esipy.operations import get_markets_prices
 from lazyblacksmith.models import Item, ItemAdjustedPrice, db
 from lazyblacksmith.models.enums import ActivityEnum
 
-from ... import celery_app
+from ... import celery_app, logger
 
 
 @celery_app.task(name="universe.adjusted_price")
 def task_adjusted_price_base_cost():
     """Task that update the adjusted prices from the API then calculate the
     base cost for every blueprints.
     """
-    prices = update_adjusted_price()
-    if prices is not None:
-        update_base_costs(prices)
+    try:
+        prices = update_adjusted_price()
+        if prices is not None:
+            update_base_costs(prices)
+    except SQLAlchemyError:
+        db.session.rollback()
+        logger.exception("Error while trying to update adjusted prices")
 
 
 def update_adjusted_price():

lbtasks/tasks/universe/indexes.py

@@ -1,11 +1,12 @@
 # -*- encoding: utf-8 -*-
 """ Market indexes task """
+from sqlalchemy.exc import SQLAlchemyError
 from lazyblacksmith.extension.esipy import esiclient
 from lazyblacksmith.extension.esipy.operations import get_industry_systems
 from lazyblacksmith.models import IndustryIndex
 from lazyblacksmith.models import db
 
-from ... import celery_app
+from ... import celery_app, logger
 
 
 @celery_app.task(name="universe.industry_indexes")
@@ -27,9 +28,13 @@ def task_industry_indexes():
                 row['cost_index'] = activity_index.cost_index
                 insert_index_list.append(row)
 
-        db.engine.execute("TRUNCATE TABLE %s" % IndustryIndex.__tablename__)
-        db.engine.execute(
-            IndustryIndex.__table__.insert(),
-            insert_index_list
-        )
-        db.session.commit()
+        try:
+            db.engine.execute("TRUNCATE TABLE %s" % IndustryIndex.__tablename__)
+            db.engine.execute(
+                IndustryIndex.__table__.insert(),
+                insert_index_list
+            )
+            db.session.commit()
+        except SQLAlchemyError:
+            db.session.rollback()
+            logger.exception("Error while updating indexes")

lbtasks/tasks/universe/market_order.py

@@ -108,9 +108,8 @@ def task_update_region_order_price(region_id, item_id_list):
         save_item_prices(item_list)
 
     except SQLAlchemyError as exc:
-        logger.error(
-            'Something went wrong while trying to insert/update data: %s',
-            str(exc)
+        logger.exception(
+            'Something went wrong while trying to insert/update data:',
         )
         db.session.rollback()