Fix GitHub community issues and pagination issue (#4)

* fix community issues * fix pagination issue * update query params for calls stream * updated replication_key and query_param for calls stream * remove replication_keys for full_table stream accounts * changelog and version bump * update changelog
singer-io · Mar 14, 2023 · 7fd3fb0 · 7fd3fb0
1 parent e06dbee
commit 7fd3fb0
Show file tree

Hide file tree

Showing 5 changed files with 40 additions and 17 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.0.3
+  * Fixes following issues [#4](https://github.com/singer-io/tap-twilio/pull/4)
+      * Fixes pagination issue
+      * Fixes GitHub community issue [#1](https://github.com/singer-io/tap-twilio/issues/1)
+      * Modifies query_params for `alerts` and `usage_records` streams
+      * Updates data_type for `current_value` field of `usage_triggers` stream to `number`
+      * Updates replication_key for `calls` and `messages` stream as `date_updated`
+
 ## 0.0.2
   * Automates field selection for replication keys [#5](https://github.com/singer-io/tap-twilio/pull/5)
 

diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='tap-twilio',
-      version='0.0.2',
+      version='0.0.3',
       description='Singer.io tap for extracting data from the Twilio API',
       author='[email protected]',
       classifiers=['Programming Language :: Python :: 3 :: Only'],

diff --git a/tap_twilio/schemas/usage_triggers.json b/tap_twilio/schemas/usage_triggers.json
@@ -32,7 +32,8 @@
       "type": ["null", "string"]
     },
     "current_value": {
-      "type": ["null", "integer"]
+      "type": ["null", "number"],
+      "multipleOf": 1e-8
     },
     "date_created": {
       "type": ["null", "string"],

diff --git a/tap_twilio/streams.py b/tap_twilio/streams.py
@@ -22,7 +22,6 @@
         'data_key': 'accounts',
         'key_properties': ['sid'],
         'replication_method': 'FULL_TABLE',  # Fetch ALL, filter results
-        'replication_keys': ['date_updated'],
         'params': {},
         'pagingation': 'root',
         'children': {
@@ -44,7 +43,7 @@
                     'dependent_phone_numbers': {
                         'api_url': 'https://api.twilio.com',
                         'api_version': '2010-04-01',
-                        'path': 'Accounts/{ParentId}/Addresses/{ParentId}/DependentPhoneNumbers.json',
+                        'path': 'Accounts/{AccountSid}/Addresses/{ParentId}/DependentPhoneNumbers.json',
                         'data_key': 'dependent_phone_numbers',
                         'key_properties': ['sid'],
                         'replication_method': 'FULL_TABLE',  # ALL for parent Address
@@ -74,6 +73,7 @@
                 'api_version': '2010-04-01',
                 'path': 'Accounts/{ParentId}/AvailablePhoneNumbers.json',
                 'data_key': 'countries',
+                'sub_resource_key': 'available_phone_numbers',
                 'key_properties': ['country_code'],
                 'replication_method': 'FULL_TABLE',
                 'params': {},
@@ -86,6 +86,7 @@
                         'api_version': '2010-04-01',
                         'path': 'Accounts/{AccountSid}/AvailablePhoneNumbers/{ParentId}/Local.json',
                         'data_key': 'available_phone_numbers',
+                        'sub_resource_key': 'local',
                         'key_properties': ['iso_country', 'phone_number'],
                         'replication_method': 'FULL_TABLE',  # ALL for parent Address
                         'params': {},
@@ -99,6 +100,7 @@
                         'api_version': '2010-04-01',
                         'path': 'Accounts/{AccountSid}/AvailablePhoneNumbers/{ParentId}/Mobile.json',
                         'data_key': 'available_phone_numbers',
+                        'sub_resource_key': 'mobile',
                         'key_properties': ['iso_country', 'phone_number'],
                         'replication_method': 'FULL_TABLE',  # ALL for parent Address
                         'params': {},
@@ -112,6 +114,7 @@
                         'api_version': '2010-04-01',
                         'path': 'Accounts/{AccountSid}/AvailablePhoneNumbers/{ParentId}/TollFree.json',
                         'data_key': 'available_phone_numbers',
+                        'sub_resource_key': 'toll_free',
                         'key_properties': ['iso_country', 'phone_number'],
                         'replication_method': 'FULL_TABLE',  # ALL for parent Address
                         'params': {},
@@ -154,7 +157,7 @@
                 'data_key': 'calls',
                 'key_properties': ['sid'],
                 'replication_method': 'INCREMENTAL',  # Filter query
-                'replication_keys': ['end_time'],
+                'replication_keys': ['date_updated'],
                 'bookmark_query_field_from': 'EndTime>',  # Daily
                 'bookmark_query_field_to': 'EndTime<',
                 'params': {},
@@ -180,8 +183,9 @@
                     'conference_participants': {
                         'api_url': 'https://api.twilio.com',
                         'api_version': '2010-04-01',
-                        'path': 'Accounts/{ParentId}/Conferences/{ParentId}/Participants.json',
+                        'path': 'Accounts/{AccountSid}/Conferences/{ParentId}/Participants.json',
                         'data_key': 'participants',
+                        'sub_resource_key': 'participants',
                         'key_properties': ['uri'],
                         'replication_method': 'FULL_TABLE',  # ALL for parent Conference
                         'params': {},
@@ -252,7 +256,7 @@
                 'data_key': 'messages',
                 'key_properties': ['sid'],
                 'replication_method': 'INCREMENTAL',  # Filter query
-                'replication_keys': ['date_sent'],
+                'replication_keys': ['date_updated'],
                 'bookmark_query_field_from': 'DateSent>',  # Daily
                 'bookmark_query_field_to': 'DateSent<',
                 'params': {},
@@ -265,6 +269,7 @@
                         'api_version': '2010-04-01',
                         'path': 'Accounts/{AccountSid}/Messages/{ParentId}/Media.json',
                         'data_key': 'media_list',
+                        'sub_resource_key': 'media',
                         'key_properties': ['sid'],
                         'replication_method': 'FULL_TABLE',  # ALL for parent Address
                         'params': {},
@@ -282,8 +287,8 @@
                 'key_properties': ['account_sid', 'category', 'start_date'],
                 'replication_method': 'INCREMENTAL',  # Filter query
                 'replication_keys': ['end_date'],
-                'bookmark_query_field_from': 'StartDate',  # Daily
-                'bookmark_query_field_to': 'EndDate',
+                'bookmark_query_field_from': 'start_date',  # Daily
+                'bookmark_query_field_to': 'end_date',
                 'params': {},
                 'pagingation': 'root'
             },
@@ -313,8 +318,8 @@
         'key_properties': ['sid'],
         'replication_method': 'INCREMENTAL',  # Filter query
         'replication_keys': ['date_updated'],
-        'bookmark_query_field_from': 'StartDate',  # Bookmark
-        'bookmark_query_field_to': 'EndDate',  # Current Date
+        'bookmark_query_field_from': 'start_date',  # Bookmark
+        'bookmark_query_field_to': 'end_date',  # Current Date
         'max_days_ago': 30,
         'params': {},
         'pagingation': 'meta'

diff --git a/tap_twilio/sync.py b/tap_twilio/sync.py
@@ -248,13 +248,12 @@ def sync_endpoint(
                 break  # No data results
 
             # Get pagination details
-            pagination = data.get('meta', {}).get('pagination', {})
-            api_total = int(pagination.get('total_results', 0))
-            if pagination.get('next', {}).get('rel') == 'next':
-                next_url = pagination.get('next', {}).get('href')
+            if data.get("next_page_uri"):
+                next_url = endpoint_config.get("api_url") + data["next_page_uri"]
             else:
                 next_url = None
 
+            api_total = len(data.get(endpoint_config.get("data_key"), []))
             if not data or data is None:
                 total_records = 0
                 break  # No data results
@@ -327,8 +326,18 @@ def sync_endpoint(
                             # their endpoints will be in the results,
                             # this will grab the child path for the child stream we're syncing,
                             # if we're syncing it. If it doesn't exist we just skip it below.
-                            child_path = record.get('_subresource_uris', {})\
-                                .get(child_stream_name, None)
+                            child_path = None
+                            if child_stream_name in ("usage_records", "usage_triggers"):
+                                if 'usage' in record.get('_subresource_uris', {}):
+                                    child_path = child_endpoint_config.get('path').format(
+                                        ParentId=parent_id)
+                            elif child_stream_name == 'dependent_phone_numbers':
+                                child_path = child_endpoint_config.get('path').format(
+                                    ParentId=parent_id, AccountSid=config.get('account_sid'))
+                            else:
+                                child_path = record.get('_subresource_uris', {}).get(
+                                    child_endpoint_config.get('sub_resource_key',
+                                                              child_stream_name))
                             child_bookmark_field = next(iter(child_endpoint_config.get(
                                 'replication_keys', [])), None)