From ca77abf7facb535dbb2146d5cb798db763cbc402 Mon Sep 17 00:00:00 2001 From: Jeremy Farbota Date: Tue, 15 May 2018 11:36:44 -0700 Subject: [PATCH 1/2] added spot bid price as an option --- dataduct/pipeline/ec2_resource.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dataduct/pipeline/ec2_resource.py b/dataduct/pipeline/ec2_resource.py index 19b0c43..5f7f5c4 100644 --- a/dataduct/pipeline/ec2_resource.py +++ b/dataduct/pipeline/ec2_resource.py @@ -17,6 +17,7 @@ ETL_AMI = config.ec2.get('ETL_AMI', const.NONE) SECURITY_GROUP = config.ec2.get('SECURITY_GROUP', const.NONE) SECURITY_GROUP_ID = config.ec2.get('SECURITY_GROUP_ID', const.NONE) +SPOT_BID_PRICE = config.ec2.get('SPOT_BID_PRICE', const.NONE) SUBNET_ID = config.ec2.get('SUBNET_ID', const.NONE) KEY_PAIR = config.etl.get('KEY_PAIR', const.NONE) RETRY_DELAY = config.etl.get('RETRY_DELAY', const.DEFAULT_DELAY) @@ -35,6 +36,7 @@ def __init__(self, ami=ETL_AMI, security_group=SECURITY_GROUP, security_group_id=SECURITY_GROUP_ID, + spot_bid_price=SPOT_BID_PRICE, subnet_id=SUBNET_ID, **kwargs): """Constructor for the Ec2Resource class @@ -47,6 +49,7 @@ def __init__(self, instance_type(str): machine type to be used eg. m1.large ami(str): ami id for the ec2 resource retry_delay(str): time delay between step retries + spot_bid_price(str): max spot bid price for resource e.g 0.04 **kwargs(optional): Keyword arguments directly passed to base class """ @@ -72,5 +75,6 @@ def __init__(self, retryDelay=RETRY_DELAY, securityGroups=security_group, securityGroupIds=security_group_id, + spotBidPrice=spot_bid_price, subnetId=subnet_id ) From a6ab112165d939bf2b7115950b7bb878f49759b5 Mon Sep 17 00:00:00 2001 From: Jeremy Farbota Date: Tue, 15 May 2018 11:45:48 -0700 Subject: [PATCH 2/2] fixed version of psycopg2 --- requirements.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index b5315fe..513be8e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,12 +2,14 @@ boto>=2.34.0 Sphinx>=1.2.3 sphinx-rtd-theme>=0.1.6 pandas>=0.14.1 -psycopg2 +psycopg2==2.7.1 MySQL-python PyYAML coverage -pyparsing>=2 +pyparsing==1.5.6 pygraphviz testfixtures>=4.1.1 mock pytimeparse +pyprind +requests