From 01d315db324d05dcd04b9340a732902df9c85e98 Mon Sep 17 00:00:00 2001 From: shantanu73 Date: Sat, 16 Mar 2024 17:57:13 +0000 Subject: [PATCH] Changes: 1) Changed number of threads from 5 to default 20 for gl_journal_entries stream. 2) Added date window implementation for gl_journal_entries stream. --- .../gl_journal_entries_generator.py | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tap_mambu/tap_generators/gl_journal_entries_generator.py b/tap_mambu/tap_generators/gl_journal_entries_generator.py index 6b54305..151492c 100644 --- a/tap_mambu/tap_generators/gl_journal_entries_generator.py +++ b/tap_mambu/tap_generators/gl_journal_entries_generator.py @@ -1,12 +1,12 @@ from .multithreaded_bookmark_generator import MultithreadedBookmarkGenerator -from ..helpers import get_bookmark -from ..helpers.datetime_utils import datetime_to_utc_str, str_to_localized_datetime, utc_now +from ..helpers.datetime_utils import datetime_to_utc_str +from datetime import datetime class GlJournalEntriesGenerator(MultithreadedBookmarkGenerator): - def _init_config(self): - super()._init_config() - self.max_threads = 5 + def __init__(self, stream_name, client, config, state, sub_type): + super(GlJournalEntriesGenerator, self).__init__(stream_name, client, config, state, sub_type) + self.date_windowing = True def _init_endpoint_config(self): super(GlJournalEntriesGenerator, self)._init_endpoint_config() @@ -16,13 +16,18 @@ def _init_endpoint_config(self): "field": "entryId", "order": "ASC" } - self.endpoint_filter_criteria = [ + + def modify_request_params(self, start, end): + self.endpoint_body['filterCriteria'] = [ + { + "field": "creationDate", + "operator": "AFTER", + "value": datetime.strftime(start, '%Y-%m-%dT00:00:00.000000Z') + }, { "field": "creationDate", - "operator": "BETWEEN", - "value": datetime_to_utc_str(str_to_localized_datetime( - get_bookmark(self.state, self.stream_name, self.sub_type, self.start_date))), - "secondValue": datetime_to_utc_str(utc_now()) + "operator": "BEFORE", + "value": datetime.strftime(end, '%Y-%m-%dT23:59:59.000000Z') } ]