Add support for custom fields (lookup fields)

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 2.2.0
+  * Adds Support for lookup fields [#124](https://github.com/singer-io/tap-zendesk/pull/124)
+
 ## 2.1.0
   * Adds new streams `talk_phone_numbers` and `ticket_metric_events` [#111](https://github.com/singer-io/tap-zendesk/pull/111)
 ## 2.0.1

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup
 
 setup(name='tap-zendesk',
-      version='2.1.0',
+      version='2.2.0',
       description='Singer.io tap for extracting data from the Zendesk API',
       author='Stitch',
       url='https://singer.io',

tap_zendesk/streams.py

@@ -29,28 +29,23 @@
     'integer': 'integer',
     'decimal': 'number',
     'checkbox': 'boolean',
+    'lookup': 'integer',
 }
 
 def get_abs_path(path):
     return os.path.join(os.path.dirname(os.path.realpath(__file__)), path)
 
 def process_custom_field(field):
     """ Take a custom field description and return a schema for it. """
-    zendesk_type = field.type
-    json_type = CUSTOM_TYPES.get(zendesk_type)
-    if json_type is None:
-        raise Exception("Discovered unsupported type for custom field {} (key: {}): {}"
-                        .format(field.title,
-                                field.key,
-                                zendesk_type))
-    field_schema = {'type': [
-        json_type,
-        'null'
-    ]}
-
-    if zendesk_type == 'date':
+    if field.type not in CUSTOM_TYPES:
+        LOGGER.critical("Discovered unsupported type for custom field %s (key: %s): %s",
+                        field.title, field.key, field.type)
+
+    json_type = CUSTOM_TYPES.get(field.type, "string")
+    field_schema = {'type': [json_type, 'null']}
+    if field.type == 'date':
         field_schema['format'] = 'datetime'
-    if zendesk_type == 'dropdown':
+    if field.type == 'dropdown':
         field_schema['enum'] = [o.value for o in field.custom_field_options]
 
     return field_schema

test/base.py

@@ -8,10 +8,6 @@
 
 from tap_tester import connections, menagerie, runner, LOGGER
 from tap_tester.base_case import BaseCase as tt_base
-from tap_tester.jira_client import JiraClient as jira_client
-from tap_tester.jira_client import CONFIGURATION_ENVIRONMENT as jira_config
-
-JIRA_CLIENT = jira_client({ **jira_config })
 
 
 # BUG https://jira.talendforge.org/browse/TDL-19985
@@ -446,11 +442,3 @@ def max_bookmarks_by_stream(self, sync_records):
                        if bk_value > max_bookmarks[stream][stream_bookmark_key]:
                            max_bookmarks[stream][stream_bookmark_key] = bk_value
         return max_bookmarks
-
-    def skipUntilDone(jira_ticket):
-
-        def wrap(test_method):
-            is_done = JIRA_CLIENT.get_jira_issue_status(jira_ticket) == "Done"
-            return tt_base.skipUnless(is_done, jira_ticket)(test_method)
-
-        return wrap

test/test_all_fields.py

@@ -8,7 +8,6 @@ class ZendeskAllFields(ZendeskTest):
     def name(self):
         return "zendesk_all_fields"
 
-    @ZendeskTest.skipUntilDone("TDL-20862")
     def test_run(self):
         """
         • Verify no unexpected streams were replicated

test/test_all_streams.py

@@ -111,7 +111,6 @@ def rate_tickets(self, records):
             #zenpy_client.tickets.rate(id, rating) # example rating {'score': 'good'}
 
 
-    @ZendeskTest.skipUntilDone("TDL-20862")
     def test_run(self):
         # Default test setup
         # Create the connection for Zendesk

test/test_automatic_fields.py

@@ -14,7 +14,6 @@ class ZendeskAutomaticFields(ZendeskTest):
     def name(self):
         return "zendesk_automatic_fields"
 
-    @ZendeskTest.skipUntilDone("TDL-20862")
     def test_run(self):
         """
         Verify we can deselect all fields except when inclusion=automatic, which is handled by base.py methods

test/test_bookmarks.py

@@ -41,7 +41,6 @@ def tearDown(self):
             self.client.users.delete(self.created_user)
 
 
-    @ZendeskTest.skipUntilDone("TDL-20862")
     def test_run(self):
         # Default test setup
         # Create the connection for Zendesk

test/test_custom_fields_discover.py

@@ -6,7 +6,7 @@ class ZendeskCustomFieldsDiscover(ZendeskTest):
     def name(self):
         return "tap_tester_zendesk_custom_fields_discover"
 
-    @ZendeskTest.skipUntilDone("TDL-20862")
+
     def test_run(self):
         # Default test setup
         # Create the connection for Zendesk

test/test_discovery.py

@@ -23,7 +23,7 @@ class ZendeskDiscover(ZendeskTest):
     def name(self):
         return "zendesk_discover_test"
 
-    @ZendeskTest.skipUntilDone("TDL-20862")
+
     def test_run(self):
         streams_to_test = self.expected_check_streams()
 

test/test_lookup_fields.py

@@ -0,0 +1,87 @@
+from base import ZendeskTest
+from tap_tester import connections, menagerie, runner
+
+
+class ZendeskAllFields(ZendeskTest):
+    """Ensure that when all streams and fields are selected, all fields are replicated."""
+
+    def name(self):
+        return "zendesk_lookup_fields"
+
+
+    def test_run(self):
+        """
+        • Verify no unexpected streams were replicated
+        • Verify that data for configured lookup_fields is replicated
+        """
+
+        # Streams to verify all fields tests
+        expected_streams = {"organizations", "users"}
+
+        lookup_fields_map = {
+            "organizations":["lookup_org_1", "lookup_org_2", "lookup_org_3", "test_new_ref"],
+            "users": ["lookup1", "lookup2", "lookup3", "manager_user"]
+        }
+
+        expected_automatic_fields = self.expected_automatic_fields()
+        conn_id = connections.ensure_connection(self)
+
+        found_catalogs = self.run_and_verify_check_mode(conn_id)
+
+        # table and field selection
+        test_catalogs_all_fields = [catalog for catalog in found_catalogs
+                                    if catalog.get('tap_stream_id') in expected_streams]
+
+        self.perform_and_verify_table_and_field_selection(
+            conn_id, test_catalogs_all_fields)
+
+        # grab metadata after performing table-and-field selection to set expectations
+        # used for asserting all fields are replicated
+        stream_to_all_catalog_fields = dict()
+        for catalog in test_catalogs_all_fields:
+            stream_id, stream_name = catalog['stream_id'], catalog['stream_name']
+            catalog_entry = menagerie.get_annotated_schema(conn_id, stream_id)
+            fields_from_field_level_md = [md_entry['breadcrumb'][1]
+                                          for md_entry in catalog_entry['metadata']
+                                          if md_entry['breadcrumb'] != []]
+            fields_from_field_level_md += lookup_fields_map[stream_name]
+            if stream_name == "users":
+                fields_from_field_level_md.remove("chat_only")
+            stream_to_all_catalog_fields[stream_name] = set(fields_from_field_level_md)
+
+        self.run_and_verify_sync(conn_id)
+
+        synced_records = runner.get_records_from_target_output()
+
+        # Verify no unexpected streams were replicated
+        synced_stream_names = set(synced_records.keys())
+        self.assertSetEqual(expected_streams, synced_stream_names)
+
+        for stream in expected_streams:
+            with self.subTest(stream=stream):
+
+                # expected values
+                expected_all_keys = stream_to_all_catalog_fields[stream]
+                expected_automatic_keys = expected_automatic_fields.get(
+                    stream, set())
+
+                # Verify that more than just the automatic fields are replicated for each stream.
+                self.assertTrue(expected_automatic_keys.issubset(
+                    expected_all_keys), msg='{} is not in "expected_all_keys"'.format(expected_automatic_keys-expected_all_keys))
+
+                messages = synced_records.get(stream)
+                # collect actual values
+                actual_all_keys = set()
+                for message in messages['messages']:
+                    if message['action'] == 'upsert':
+                        actual_all_keys.update(message['data'].keys())
+                    if stream == "organizations":
+                        for key in message['data']["organization_fields"].keys():
+                            if key in lookup_fields_map[stream]:
+                                actual_all_keys.add(key)
+                    if stream == "users":
+                        for key in message['data']["user_fields"].keys():
+                            if key in lookup_fields_map[stream]:
+                                actual_all_keys.add(key)
+                # verify all fields for each stream are replicated
+                self.assertSetEqual(expected_all_keys, actual_all_keys)

test/test_minimal_selection.py

@@ -19,7 +19,7 @@ def expected_pks(self):
             'users': {'id'}
         }
 
-    @ZendeskTest.skipUntilDone("TDL-20862")
+
     def test_run(self):
         # Default test setup
         # Create the connection for Zendesk

test/test_pagination.py

@@ -10,7 +10,7 @@ class ZendeskPagination(ZendeskTest):
     def name(self):
         return "zendesk_pagination_test"
 
-    @ZendeskTest.skipUntilDone("TDL-20862")
+
     def test_run(self):
         """
         • Verify that for each stream you can get multiple pages of data.

test/test_standard_bookmark.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 
 from base import ZendeskTest
-from tap_tester import connections, menagerie, runner
+from tap_tester import connections, menagerie, runner, LOGGER
 
 
 class ZendeskBookMark(ZendeskTest):
@@ -10,7 +10,7 @@ class ZendeskBookMark(ZendeskTest):
     def name(self):
         return "zendesk_bookmark_test"
 
-    @ZendeskTest.skipUntilDone("TDL-20862")
+
     def test_run(self):
         """
         Verify that for each stream you can do a sync which records bookmarks.

test/test_start_date.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 
 from base import ZendeskTest
-from tap_tester import connections, runner
+from tap_tester import connections, runner, LOGGER
 
 
 class ZendeskStartDate(ZendeskTest):
@@ -16,7 +16,7 @@ class ZendeskStartDate(ZendeskTest):
     def name(self):
         return "zendesk_start_date_test"
 
-    @ZendeskTest.skipUntilDone("TDL-20862")
+
     def test_run(self):
         """
         Test that the start_date configuration is respected

test/unittests/test_customfield_schema_type.py

@@ -0,0 +1,76 @@
+import types
+import unittest
+from tap_zendesk.streams import process_custom_field
+
+
+class TestCustomFieldSchemaDiscovery(unittest.TestCase):
+
+    """
+    Validates that all zendesk types are validated and converted to correct singer type formats
+    """
+
+    def get_z_field_obj(self, *params):
+        field = types.SimpleNamespace()
+        field.title, field.key, field.type = params
+        return field
+
+    def test_return_field_type_lookup(self):
+        expected_singer_type = {"type" : ["integer", "null"]}
+        z_field = self.get_z_field_obj("test_lookup_type", "user_lookup_1", "lookup")
+        actual_singer_type = process_custom_field(z_field)
+        self.assertEqual(actual_singer_type, expected_singer_type)
+
+    def test_return_field_type_text(self):
+        expected_singer_type = {"type" : ["string", "null"]}
+        z_field = self.get_z_field_obj("user_name", "name_field", "text")
+        actual_singer_type = process_custom_field(z_field)
+        self.assertEqual(actual_singer_type, expected_singer_type)
+
+    def test_return_field_type_textarea(self):
+        expected_singer_type = {"type" : ["string", "null"]}
+        z_field = self.get_z_field_obj("field_title_textarea", "field_key_textarea", "textarea")
+        actual_singer_type = process_custom_field(z_field)
+        self.assertEqual(actual_singer_type, expected_singer_type)
+
+    def test_return_field_type_regexp(self):
+        expected_singer_type = {"type" : ["string", "null"]}
+        z_field = self.get_z_field_obj("field_title_regexp", "field_key_regexp", "regexp")
+        actual_singer_type = process_custom_field(z_field)
+        self.assertEqual(actual_singer_type, expected_singer_type)
+
+    def test_return_field_type_dropdown(self):
+        expected_singer_type = {"type" : ["string", "null"]}
+        z_field = self.get_z_field_obj("field_title_dropdown", "field_key_dropdown", "regexp")
+        actual_singer_type = process_custom_field(z_field)
+        self.assertEqual(actual_singer_type, expected_singer_type)
+
+    def test_return_field_type_decimal(self):
+        expected_singer_type = {"type" : ["number", "null"]}
+        z_field = self.get_z_field_obj("field_title_decimal", "field_key_decimal", "decimal")
+        actual_singer_type = process_custom_field(z_field)
+        self.assertEqual(actual_singer_type, expected_singer_type)
+
+    def test_return_field_type_integer(self):
+        expected_singer_type = {"type" : ["integer", "null"]}
+        z_field = self.get_z_field_obj("field_title_integer", "field_key_integer", "integer")
+        actual_singer_type = process_custom_field(z_field)
+        self.assertEqual(actual_singer_type, expected_singer_type)
+
+    def test_return_field_type_checkbox(self):
+        expected_singer_type = {"type" : ["boolean", "null"]}
+        z_field = self.get_z_field_obj("field_title_checkbox", "field_key_checkbox", "checkbox")
+        actual_singer_type = process_custom_field(z_field)
+        self.assertEqual(actual_singer_type, expected_singer_type)
+
+
+    def test_return_field_type_date(self):
+        expected_singer_type = {"type" : ["string", "null"], 'format': 'datetime'}
+        z_field = self.get_z_field_obj("field_title_date", "field_key_date", "date")
+        actual_singer_type = process_custom_field(z_field)
+        self.assertEqual(actual_singer_type, expected_singer_type)
+
+    def test_return_field_type_unidentified(self):
+        expected_singer_type = {"type" : ["string", "null"]}
+        z_field = self.get_z_field_obj("test_UNINDETIFIED_TYPE", "UNINDETIFIED_TYPE", "UNINDETIFIED_TYPE")
+        actual_singer_type = process_custom_field(z_field)
+        self.assertEqual(actual_singer_type, expected_singer_type)