diff --git a/tap_postgres/db.py b/tap_postgres/db.py index 60479b0..bfec6eb 100644 --- a/tap_postgres/db.py +++ b/tap_postgres/db.py @@ -4,6 +4,7 @@ import psycopg2 import psycopg2.extras import singer +import pytz LOGGER = singer.get_logger() cursor_iter_size = 20000 @@ -57,11 +58,11 @@ def selected_value_to_singer_value_impl(elem, sql_datatype): cleaned_elem = elem elif isinstance(elem, datetime.datetime): if sql_datatype == 'timestamp with time zone': - cleaned_elem = elem.isoformat() + cleaned_elem = singer.utils.strftime(elem.astimezone(tz=pytz.UTC)) else: #timestamp WITH OUT time zone - cleaned_elem = elem.isoformat() + '+00:00' + cleaned_elem = singer.utils.strftime(elem.replace(tzinfo=pytz.UTC)) elif isinstance(elem, datetime.date): - cleaned_elem = elem.isoformat() + 'T00:00:00+00:00' + cleaned_elem = elem.isoformat() + 'T00:00:00Z' elif sql_datatype == 'bit': cleaned_elem = elem == '1' elif sql_datatype == 'boolean': diff --git a/tap_postgres/sync_strategies/logical_replication.py b/tap_postgres/sync_strategies/logical_replication.py index d19e401..6b9e60e 100644 --- a/tap_postgres/sync_strategies/logical_replication.py +++ b/tap_postgres/sync_strategies/logical_replication.py @@ -136,18 +136,24 @@ def selected_value_to_singer_value_impl(elem, og_sql_datatype, conn_info): if elem is None: return elem if sql_datatype == 'timestamp without time zone': - return parse(elem).isoformat() + '+00:00' + return singer.utils.strftime(singer.utils.strptime_to_utc(elem)) if sql_datatype == 'timestamp with time zone': if isinstance(elem, datetime.datetime): - return elem.isoformat() + return singer.utils.strftime(elem.astimezone(tz=pytz.UTC)) - return parse(elem).isoformat() + return singer.utils.strftime(singer.utils.strptime_to_utc(elem)) if sql_datatype == 'date': if isinstance(elem, datetime.date): #logical replication gives us dates as strings UNLESS they from an array - return elem.isoformat() + 'T00:00:00+00:00' - return parse(elem).isoformat() + "+00:00" + return elem.isoformat() + 'T00:00:00Z' + + return parse(elem).isoformat() + "Z" if sql_datatype == 'time with time zone': + #NB> to be consistent, we should probably being using the latter method + #>>> dateutil.parser.parse(t).isoformat().split('T')[1] + # '05:30:10+00:00' + #>>> singer.utils.strftime(singer.utils.strptime_to_utc(t)).split('T')[1] + #'05:30:10.000000Z' return parse(elem).isoformat().split('T')[1] if sql_datatype == 'bit': #for arrays, elem will == True