From 7fd3fb0727188d4d80c12ece627a44eae587c6c9 Mon Sep 17 00:00:00 2001 From: Kethan Cherukuri <105211331+kethan1122@users.noreply.github.com> Date: Tue, 14 Mar 2023 18:17:42 +0530 Subject: [PATCH] Fix GitHub community issues and pagination issue (#4) * fix community issues * fix pagination issue * update query params for calls stream * updated replication_key and query_param for calls stream * remove replication_keys for full_table stream accounts * changelog and version bump * update changelog --- CHANGELOG.md | 8 ++++++++ setup.py | 2 +- tap_twilio/schemas/usage_triggers.json | 3 ++- tap_twilio/streams.py | 23 ++++++++++++++--------- tap_twilio/sync.py | 21 +++++++++++++++------ 5 files changed, 40 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e914416..d3257d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.0.3 + * Fixes following issues [#4](https://github.com/singer-io/tap-twilio/pull/4) + * Fixes pagination issue + * Fixes GitHub community issue [#1](https://github.com/singer-io/tap-twilio/issues/1) + * Modifies query_params for `alerts` and `usage_records` streams + * Updates data_type for `current_value` field of `usage_triggers` stream to `number` + * Updates replication_key for `calls` and `messages` stream as `date_updated` + ## 0.0.2 * Automates field selection for replication keys [#5](https://github.com/singer-io/tap-twilio/pull/5) diff --git a/setup.py b/setup.py index 3569f9f..d7c4682 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages setup(name='tap-twilio', - version='0.0.2', + version='0.0.3', description='Singer.io tap for extracting data from the Twilio API', author='jeff.huth@bytecode.io', classifiers=['Programming Language :: Python :: 3 :: Only'], diff --git a/tap_twilio/schemas/usage_triggers.json b/tap_twilio/schemas/usage_triggers.json index a02e02b..7f18dd4 100644 --- a/tap_twilio/schemas/usage_triggers.json +++ b/tap_twilio/schemas/usage_triggers.json @@ -32,7 +32,8 @@ "type": ["null", "string"] }, "current_value": { - "type": ["null", "integer"] + "type": ["null", "number"], + "multipleOf": 1e-8 }, "date_created": { "type": ["null", "string"], diff --git a/tap_twilio/streams.py b/tap_twilio/streams.py index 1a46388..4fe6fa7 100644 --- a/tap_twilio/streams.py +++ b/tap_twilio/streams.py @@ -22,7 +22,6 @@ 'data_key': 'accounts', 'key_properties': ['sid'], 'replication_method': 'FULL_TABLE', # Fetch ALL, filter results - 'replication_keys': ['date_updated'], 'params': {}, 'pagingation': 'root', 'children': { @@ -44,7 +43,7 @@ 'dependent_phone_numbers': { 'api_url': 'https://api.twilio.com', 'api_version': '2010-04-01', - 'path': 'Accounts/{ParentId}/Addresses/{ParentId}/DependentPhoneNumbers.json', + 'path': 'Accounts/{AccountSid}/Addresses/{ParentId}/DependentPhoneNumbers.json', 'data_key': 'dependent_phone_numbers', 'key_properties': ['sid'], 'replication_method': 'FULL_TABLE', # ALL for parent Address @@ -74,6 +73,7 @@ 'api_version': '2010-04-01', 'path': 'Accounts/{ParentId}/AvailablePhoneNumbers.json', 'data_key': 'countries', + 'sub_resource_key': 'available_phone_numbers', 'key_properties': ['country_code'], 'replication_method': 'FULL_TABLE', 'params': {}, @@ -86,6 +86,7 @@ 'api_version': '2010-04-01', 'path': 'Accounts/{AccountSid}/AvailablePhoneNumbers/{ParentId}/Local.json', 'data_key': 'available_phone_numbers', + 'sub_resource_key': 'local', 'key_properties': ['iso_country', 'phone_number'], 'replication_method': 'FULL_TABLE', # ALL for parent Address 'params': {}, @@ -99,6 +100,7 @@ 'api_version': '2010-04-01', 'path': 'Accounts/{AccountSid}/AvailablePhoneNumbers/{ParentId}/Mobile.json', 'data_key': 'available_phone_numbers', + 'sub_resource_key': 'mobile', 'key_properties': ['iso_country', 'phone_number'], 'replication_method': 'FULL_TABLE', # ALL for parent Address 'params': {}, @@ -112,6 +114,7 @@ 'api_version': '2010-04-01', 'path': 'Accounts/{AccountSid}/AvailablePhoneNumbers/{ParentId}/TollFree.json', 'data_key': 'available_phone_numbers', + 'sub_resource_key': 'toll_free', 'key_properties': ['iso_country', 'phone_number'], 'replication_method': 'FULL_TABLE', # ALL for parent Address 'params': {}, @@ -154,7 +157,7 @@ 'data_key': 'calls', 'key_properties': ['sid'], 'replication_method': 'INCREMENTAL', # Filter query - 'replication_keys': ['end_time'], + 'replication_keys': ['date_updated'], 'bookmark_query_field_from': 'EndTime>', # Daily 'bookmark_query_field_to': 'EndTime<', 'params': {}, @@ -180,8 +183,9 @@ 'conference_participants': { 'api_url': 'https://api.twilio.com', 'api_version': '2010-04-01', - 'path': 'Accounts/{ParentId}/Conferences/{ParentId}/Participants.json', + 'path': 'Accounts/{AccountSid}/Conferences/{ParentId}/Participants.json', 'data_key': 'participants', + 'sub_resource_key': 'participants', 'key_properties': ['uri'], 'replication_method': 'FULL_TABLE', # ALL for parent Conference 'params': {}, @@ -252,7 +256,7 @@ 'data_key': 'messages', 'key_properties': ['sid'], 'replication_method': 'INCREMENTAL', # Filter query - 'replication_keys': ['date_sent'], + 'replication_keys': ['date_updated'], 'bookmark_query_field_from': 'DateSent>', # Daily 'bookmark_query_field_to': 'DateSent<', 'params': {}, @@ -265,6 +269,7 @@ 'api_version': '2010-04-01', 'path': 'Accounts/{AccountSid}/Messages/{ParentId}/Media.json', 'data_key': 'media_list', + 'sub_resource_key': 'media', 'key_properties': ['sid'], 'replication_method': 'FULL_TABLE', # ALL for parent Address 'params': {}, @@ -282,8 +287,8 @@ 'key_properties': ['account_sid', 'category', 'start_date'], 'replication_method': 'INCREMENTAL', # Filter query 'replication_keys': ['end_date'], - 'bookmark_query_field_from': 'StartDate', # Daily - 'bookmark_query_field_to': 'EndDate', + 'bookmark_query_field_from': 'start_date', # Daily + 'bookmark_query_field_to': 'end_date', 'params': {}, 'pagingation': 'root' }, @@ -313,8 +318,8 @@ 'key_properties': ['sid'], 'replication_method': 'INCREMENTAL', # Filter query 'replication_keys': ['date_updated'], - 'bookmark_query_field_from': 'StartDate', # Bookmark - 'bookmark_query_field_to': 'EndDate', # Current Date + 'bookmark_query_field_from': 'start_date', # Bookmark + 'bookmark_query_field_to': 'end_date', # Current Date 'max_days_ago': 30, 'params': {}, 'pagingation': 'meta' diff --git a/tap_twilio/sync.py b/tap_twilio/sync.py index 0efe8f9..e1bddec 100644 --- a/tap_twilio/sync.py +++ b/tap_twilio/sync.py @@ -248,13 +248,12 @@ def sync_endpoint( break # No data results # Get pagination details - pagination = data.get('meta', {}).get('pagination', {}) - api_total = int(pagination.get('total_results', 0)) - if pagination.get('next', {}).get('rel') == 'next': - next_url = pagination.get('next', {}).get('href') + if data.get("next_page_uri"): + next_url = endpoint_config.get("api_url") + data["next_page_uri"] else: next_url = None + api_total = len(data.get(endpoint_config.get("data_key"), [])) if not data or data is None: total_records = 0 break # No data results @@ -327,8 +326,18 @@ def sync_endpoint( # their endpoints will be in the results, # this will grab the child path for the child stream we're syncing, # if we're syncing it. If it doesn't exist we just skip it below. - child_path = record.get('_subresource_uris', {})\ - .get(child_stream_name, None) + child_path = None + if child_stream_name in ("usage_records", "usage_triggers"): + if 'usage' in record.get('_subresource_uris', {}): + child_path = child_endpoint_config.get('path').format( + ParentId=parent_id) + elif child_stream_name == 'dependent_phone_numbers': + child_path = child_endpoint_config.get('path').format( + ParentId=parent_id, AccountSid=config.get('account_sid')) + else: + child_path = record.get('_subresource_uris', {}).get( + child_endpoint_config.get('sub_resource_key', + child_stream_name)) child_bookmark_field = next(iter(child_endpoint_config.get( 'replication_keys', [])), None)