diff --git a/tap_postgres/__init__.py b/tap_postgres/__init__.py index 6831f82..691e6b2 100644 --- a/tap_postgres/__init__.py +++ b/tap_postgres/__init__.py @@ -235,7 +235,6 @@ def produce_table_info(conn): with conn.cursor(cursor_factory=psycopg2.extras.DictCursor, name='stitch_cursor') as cur: cur.itersize = post_db.cursor_iter_size table_info = {} - # SELECT CASE WHEN $2.typtype = 'd' THEN $2.typbasetype ELSE $1.atttypid END cur.execute(""" SELECT pg_class.reltuples::BIGINT AS approximate_row_count, @@ -244,7 +243,7 @@ def produce_table_info(conn): pg_class.relname AS table_name, attname AS column_name, i.indisprimary AS primary_key, - format_type(a.atttypid, NULL::integer) AS data_type, + format_type(CASE WHEN pgt.typtype = 'd' THEN pgt.typbasetype ELSE a.atttypid END, NULL::integer) AS data_type, information_schema._pg_char_max_length(CASE WHEN COALESCE(subpgt.typtype, pgt.typtype) = 'd' THEN COALESCE(subpgt.typbasetype, pgt.typbasetype) ELSE COALESCE(subpgt.oid, pgt.oid) END, diff --git a/tests/test_postgres_full_table_replication.py b/tests/test_postgres_full_table_replication.py index 36fadc3..a607c7f 100644 --- a/tests/test_postgres_full_table_replication.py +++ b/tests/test_postgres_full_table_replication.py @@ -61,6 +61,7 @@ 'our_inet': {'type': ['null', 'string']}, 'our_mac': {'type': ['null', 'string']}, 'our_alignment_enum': {'type': ['null', 'string']}, + 'our_text_domain': {'type': ['null', 'string']}, 'our_money': {'type': ['null', 'string']} }}} @@ -123,6 +124,9 @@ def setUp(self): cur.execute(""" DROP TYPE IF EXISTS ALIGNMENT CASCADE """) cur.execute(""" CREATE TYPE ALIGNMENT AS ENUM ('good', 'bad', 'ugly') """) + cur.execute(""" DROP DOMAIN IF EXISTS text_domain_type CASCADE """) + cur.execute(""" CREATE DOMAIN text_domain_type text""") + create_table_sql = """ CREATE TABLE {} (id SERIAL PRIMARY KEY, our_varchar VARCHAR, @@ -150,6 +154,7 @@ def setUp(self): our_cidr cidr, our_mac macaddr, our_alignment_enum ALIGNMENT, + our_text_domain domain_type, our_money money) """.format(canonicalized_table_name(test_schema_name, test_table_name, cur), NUMERIC_PRECISION, NUMERIC_SCALE) @@ -188,6 +193,7 @@ def setUp(self): 'our_inet': '192.168.100.128/24', 'our_mac' : '08:00:2b:01:02:03', 'our_alignment_enum': 'good', + 'our_text_domain': 'hello, world!', 'our_money': '100.1122', } @@ -338,6 +344,7 @@ def test_run(self): ('properties', 'our_cidr'): {'inclusion': 'available', 'selected-by-default': True, 'sql-datatype': 'cidr'}, ('properties', 'our_mac'): {'inclusion': 'available', 'selected-by-default': True, 'sql-datatype': 'macaddr'}, ('properties', 'our_alignment_enum'): {'inclusion': 'available', 'selected-by-default': True, 'sql-datatype': 'alignment'}, + ('properties', 'our_text_domain'): {'inclusion': 'available', 'selected-by-default': True, 'sql-datatype': 'text'}, ('properties', 'our_money'): {'inclusion': 'available', 'selected-by-default': True, 'sql-datatype': 'money'}}, metadata.to_map(md)) @@ -414,6 +421,7 @@ def test_run(self): 'our_cidr' : self.rec_1['our_cidr'], 'our_mac' : self.rec_1['our_mac'], 'our_alignment_enum' : self.rec_1['our_alignment_enum'], + 'our_text_domain' : self.rec_1['our_text_domain'], 'our_money' : '$100.11' } diff --git a/tests/test_postgres_logical_replication.py b/tests/test_postgres_logical_replication.py index 7e355be..9a00446 100644 --- a/tests/test_postgres_logical_replication.py +++ b/tests/test_postgres_logical_replication.py @@ -59,6 +59,7 @@ 'our_inet': {'type': ['null', 'string']}, 'our_mac': {'type': ['null', 'string']}, 'our_alignment_enum': {'type': ['null', 'string']}, + 'our_text_domain': {'type': ['null', 'string']}, 'our_money': {'type': ['null', 'string']}}}} @@ -139,6 +140,8 @@ def setUp(self): cur.execute(""" DROP TYPE IF EXISTS ALIGNMENT CASCADE """) cur.execute(""" CREATE TYPE ALIGNMENT AS ENUM ('good', 'bad', 'ugly') """) + cur.execute(""" DROP DOMAIN IF EXISTS text_domain_type CASCADE """) + cur.execute(""" CREATE DOMAIN text_domain_type text""") create_table_sql = """ CREATE TABLE {} (id SERIAL PRIMARY KEY, @@ -168,6 +171,7 @@ def setUp(self): our_inet inet, our_mac macaddr, our_alignment_enum ALIGNMENT, + our_text_domain text_domain_type, our_money money) """.format(canonicalized_table_name(test_schema_name, test_table_name, cur)) @@ -206,7 +210,8 @@ def setUp(self): 'our_cidr' : '192.168.100.128/25', 'our_inet': '192.168.100.128/24', 'our_mac' : '08:00:2b:01:02:03', - 'our_alignment_enum': 'bad'} + 'our_alignment_enum': 'bad', + 'our_text_domain': 'hello, world!'} insert_record(cur, test_table_name, self.rec_1) @@ -472,6 +477,7 @@ def test_run(self): 'our_inet': '192.168.102.128', 'our_mac': self.rec_3['our_mac'], 'our_alignment_enum' : None, + 'our_text_domain': None, 'our_money' :'$412.12' } self.assertEqual(set(actual_record_1.keys()), set(expected_inserted_record.keys()), @@ -628,6 +634,7 @@ def test_run(self): 'our_inet': self.rec_1['our_inet'], 'our_mac': self.rec_1['our_mac'], 'our_alignment_enum' : 'bad', + 'our_text_domain': 'hello, world!', 'our_money' : '$56.81' } diff --git a/tests/unittests/test_discovery.py b/tests/unittests/test_discovery.py index 8bf5578..9d29ddc 100644 --- a/tests/unittests/test_discovery.py +++ b/tests/unittests/test_discovery.py @@ -453,7 +453,94 @@ def test_catalog(self): 'type': 'object', 'definitions' : tap_postgres.BASE_RECURSIVE_SCHEMAS}, stream_dict.get('schema')) +class TestDomainsTable(unittest.TestCase): + maxDiff = None + table_name = "CHICKEN TIMES" + + def setUp(self): + table_spec = { + "columns": [ + { + "name": "our_test_domain_pk", + "type": "test_domain", + "primary_key": True, + }, + {"name": "our_test_domain", "type": "test_domain"}, + {"name": "our_test_integer_domain", "type": "test_integer_domain"}, + ], + "name": TestHStoreTable.table_name, + } + with get_test_connection() as conn: + cur = conn.cursor() + cur.execute(""" DROP DOMAIN IF EXISTS test_domain CASCADE """) + cur.execute(""" CREATE DOMAIN test_domain AS text; """) + cur.execute(""" DROP DOMAIN IF EXISTS test_integer_domain CASCADE """) + cur.execute(""" CREATE DOMAIN test_integer_domain AS integer; """) + + ensure_test_table(table_spec) + + def test_catalog(self): + conn_config = get_test_connection_config() + streams = tap_postgres.do_discovery(conn_config) + chicken_streams = [ + s for s in streams if s["tap_stream_id"] == "public-CHICKEN TIMES" + ] + self.assertEqual(len(chicken_streams), 1) + stream_dict = chicken_streams[0] + stream_dict.get("metadata").sort(key=lambda md: md["breadcrumb"]) + + with get_test_connection() as conn: + with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur: + cur.execute( + """INSERT INTO "CHICKEN TIMES" (our_test_domain_pk, our_test_domain, our_test_integer_domain) VALUES ('sad', 'happy', 3)""" + ) + cur.execute("""SELECT * FROM "CHICKEN TIMES" """) + self.assertEqual( + metadata.to_map(stream_dict.get("metadata")), + { + (): { + "table-key-properties": ["our_test_domain_pk"], + "database-name": "postgres", + "schema-name": "public", + "is-view": False, + "row-count": 0, + }, + ("properties", "our_test_domain_pk"): { + "inclusion": "automatic", + "sql-datatype": "text", + "selected-by-default": True, + }, + ("properties", "our_test_domain"): { + "inclusion": "available", + "sql-datatype": "text", + "selected-by-default": True, + }, + ("properties", "our_test_integer_domain"): { + "inclusion": "available", + "sql-datatype": "integer", + "selected-by-default": True, + }, + }, + ) + + self.assertEqual( + { + "properties": { + "our_test_domain_pk": {"type": ["string"]}, + "our_test_domain": {"type": ["null", "string"]}, + "our_test_integer_domain": { + "minimum": -2147483648, + "maximum": 2147483647, + "type": ["null", "integer"], + }, + }, + "type": "object", + "definitions": BASE_RECURSIVE_SCHEMAS, + }, + stream_dict.get("schema"), + ) + class TestArraysTable(unittest.TestCase): maxDiff = None table_name = 'CHICKEN TIMES'