Skip to content

Commit

Permalink
Changes:
Browse files Browse the repository at this point in the history
1) Changed number of threads from 5 to default 20 for gl_journal_entries stream.
2) Added date window implementation for gl_journal_entries stream.
  • Loading branch information
shantanu73 committed Mar 16, 2024
1 parent db922e5 commit 01d315d
Showing 1 changed file with 15 additions and 10 deletions.
25 changes: 15 additions & 10 deletions tap_mambu/tap_generators/gl_journal_entries_generator.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
from .multithreaded_bookmark_generator import MultithreadedBookmarkGenerator
from ..helpers import get_bookmark
from ..helpers.datetime_utils import datetime_to_utc_str, str_to_localized_datetime, utc_now
from ..helpers.datetime_utils import datetime_to_utc_str
from datetime import datetime


class GlJournalEntriesGenerator(MultithreadedBookmarkGenerator):
def _init_config(self):
super()._init_config()
self.max_threads = 5
def __init__(self, stream_name, client, config, state, sub_type):
super(GlJournalEntriesGenerator, self).__init__(stream_name, client, config, state, sub_type)
self.date_windowing = True

def _init_endpoint_config(self):
super(GlJournalEntriesGenerator, self)._init_endpoint_config()
Expand All @@ -16,13 +16,18 @@ def _init_endpoint_config(self):
"field": "entryId",
"order": "ASC"
}
self.endpoint_filter_criteria = [

def modify_request_params(self, start, end):
self.endpoint_body['filterCriteria'] = [
{
"field": "creationDate",
"operator": "AFTER",
"value": datetime.strftime(start, '%Y-%m-%dT00:00:00.000000Z')
},
{
"field": "creationDate",
"operator": "BETWEEN",
"value": datetime_to_utc_str(str_to_localized_datetime(
get_bookmark(self.state, self.stream_name, self.sub_type, self.start_date))),
"secondValue": datetime_to_utc_str(utc_now())
"operator": "BEFORE",
"value": datetime.strftime(end, '%Y-%m-%dT23:59:59.000000Z')
}
]

Expand Down

0 comments on commit 01d315d

Please sign in to comment.