From 72a6f10737053028ef10b2b6211af95a10167c66 Mon Sep 17 00:00:00 2001 From: Benjamin Dornel Date: Wed, 15 Jan 2025 22:54:48 +0800 Subject: [PATCH 1/2] chore(banks): declare negative symbol explicitly in boa --- src/monopoly/constants/statement.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/monopoly/constants/statement.py b/src/monopoly/constants/statement.py index dbf468b4..8538a3fb 100644 --- a/src/monopoly/constants/statement.py +++ b/src/monopoly/constants/statement.py @@ -57,7 +57,7 @@ class SharedPatterns(StrEnum): OPTIONAL_NEGATIVE_SYMBOL = r"(?:-)?" DEBIT_CREDIT_SUFFIX = r"(?PCR\b|DR\b|\+|\-)?\s*" - AMOUNT = rf"(?P{OPTIONAL_NEGATIVE_SYMBOL}{COMMA_FORMAT}|{ENCLOSED_COMMA_FORMAT}\s*" + AMOUNT = rf"(?P{COMMA_FORMAT}|{ENCLOSED_COMMA_FORMAT}\s*" AMOUNT_EXTENDED_WITHOUT_EOL = AMOUNT + DEBIT_CREDIT_SUFFIX AMOUNT_EXTENDED = AMOUNT_EXTENDED_WITHOUT_EOL + r"$" @@ -109,7 +109,8 @@ class CreditTransactionPatterns(RegexEnum): BANK_OF_AMERICA = ( rf"(?P{ISO8601.MM_DD_YY})\s+" + SharedPatterns.DESCRIPTION - + SharedPatterns.AMOUNT_EXTENDED + + r"(?P\-)?" + + SharedPatterns.AMOUNT ) DBS = ( rf"(?P{ISO8601.DD_MMM})\s+" From e6710463fdb78f7c4e9570fea7d18964f260633b Mon Sep 17 00:00:00 2001 From: Benjamin Dornel Date: Wed, 15 Jan 2025 23:00:19 +0800 Subject: [PATCH 2/2] refactor: rename suffix -> polarity --- src/monopoly/config.py | 2 +- src/monopoly/constants/statement.py | 12 ++++++------ src/monopoly/statements/debit_statement.py | 11 ++++++----- src/monopoly/statements/transaction.py | 20 ++++++++++---------- tests/unit/test_cli.py | 2 +- tests/unit/test_credit_statement.py | 7 +++++-- tests/unit/test_safety_check.py | 12 ++++++------ tests/unit/test_statement_process_line.py | 12 ++++++------ tests/unit/test_statement_refund.py | 4 ++-- 9 files changed, 43 insertions(+), 39 deletions(-) diff --git a/src/monopoly/config.py b/src/monopoly/config.py index e4bc4496..e1083e9e 100644 --- a/src/monopoly/config.py +++ b/src/monopoly/config.py @@ -54,7 +54,7 @@ class StatementConfig: "01 NOV BALANCE B/F 190.77" (will be ignored) "01 NOV YA KUN KAYA TOAST 12.00 " (will be kept) - `transaction_auto_polarity` controls whether transaction amounts are set as negative. - or positive if they have 'CR' or '+' as a suffix. Enabled by default. + or positive if they have 'CR' or '+' as a polarity identifier. Enabled by default. If enabled, only 'CR' or '+' will make a transaction positive. Disabled by default. - `safety_check` controls whether the safety check for banks. Use for banks that don't provide total amount (or total debit/credit) diff --git a/src/monopoly/constants/statement.py b/src/monopoly/constants/statement.py index 8538a3fb..1b5ef9fc 100644 --- a/src/monopoly/constants/statement.py +++ b/src/monopoly/constants/statement.py @@ -37,7 +37,7 @@ class Columns(AutoEnum): AMOUNT = auto() DATE = auto() DESCRIPTION = auto() - SUFFIX = auto() + POLARITY = auto() TRANSACTION_DATE = auto() @@ -55,10 +55,10 @@ class SharedPatterns(StrEnum): COMMA_FORMAT = r"\d{1,3}(,\d{3})*\.\d*" ENCLOSED_COMMA_FORMAT = rf"\({COMMA_FORMAT}\s{{0,1}}\))" OPTIONAL_NEGATIVE_SYMBOL = r"(?:-)?" - DEBIT_CREDIT_SUFFIX = r"(?PCR\b|DR\b|\+|\-)?\s*" + POLARITY = r"(?PCR\b|DR\b|\+|\-)?\s*" AMOUNT = rf"(?P{COMMA_FORMAT}|{ENCLOSED_COMMA_FORMAT}\s*" - AMOUNT_EXTENDED_WITHOUT_EOL = AMOUNT + DEBIT_CREDIT_SUFFIX + AMOUNT_EXTENDED_WITHOUT_EOL = AMOUNT + POLARITY AMOUNT_EXTENDED = AMOUNT_EXTENDED_WITHOUT_EOL + r"$" BALANCE = rf"(?P{COMMA_FORMAT})?$" @@ -109,7 +109,7 @@ class CreditTransactionPatterns(RegexEnum): BANK_OF_AMERICA = ( rf"(?P{ISO8601.MM_DD_YY})\s+" + SharedPatterns.DESCRIPTION - + r"(?P\-)?" + + r"(?P\-)?" + SharedPatterns.AMOUNT ) DBS = ( @@ -160,7 +160,7 @@ class CreditTransactionPatterns(RegexEnum): TRUST = ( rf"(?P{ISO8601.DD_MMM})\s+" + r"(?P(?:(?!Total outstanding balance).)*?)" - + r"(?P\+)?" + + r"(?P\+)?" + SharedPatterns.AMOUNT + "$" # necessary to ignore FCY ) @@ -182,7 +182,7 @@ class DebitTransactionPatterns(RegexEnum): + SharedPatterns.DESCRIPTION # remove *\s + SharedPatterns.AMOUNT[:-3] - + r"(?P\-|\+)\s+" + + r"(?P\-|\+)\s+" + SharedPatterns.BALANCE ) OCBC = ( diff --git a/src/monopoly/statements/debit_statement.py b/src/monopoly/statements/debit_statement.py index c15ec540..b93bb53f 100644 --- a/src/monopoly/statements/debit_statement.py +++ b/src/monopoly/statements/debit_statement.py @@ -21,17 +21,18 @@ def pre_process_match( self, transaction_match: TransactionMatch ) -> TransactionMatch: """ - Pre-processes transactions by adding a debit or credit suffix to the group dict + Pre-processes transactions by adding a debit or credit + polarity identifier to the group dict """ if self.config.statement_type == EntryType.DEBIT: - transaction_match.groupdict.suffix = self.get_debit_suffix( + transaction_match.groupdict.polarity = self.get_debit_polarity( transaction_match ) return transaction_match - def get_debit_suffix(self, transaction_match: TransactionMatch) -> str | None: + def get_debit_polarity(self, transaction_match: TransactionMatch) -> str | None: """ - Gets the accounting suffix for debit card statements + Gets the accounting polarity for debit card statements Attempts to identify whether a transaction is a debit or credit entry based on the distance from the withdrawal @@ -52,7 +53,7 @@ def get_debit_suffix(self, transaction_match: TransactionMatch) -> str | None: if withdrawal_diff > deposit_diff: return "CR" return "DR" - return transaction_match.groupdict.suffix + return transaction_match.groupdict.polarity @lru_cache def get_withdrawal_pos(self, page_number: int) -> int | None: diff --git a/src/monopoly/statements/transaction.py b/src/monopoly/statements/transaction.py index 7034cf4a..869a5d28 100644 --- a/src/monopoly/statements/transaction.py +++ b/src/monopoly/statements/transaction.py @@ -29,13 +29,13 @@ def __init__( description: str, amount: str, transaction_date: Optional[str] = None, - suffix: Optional[str] = None, + polarity: Optional[str] = None, **_, ): self.transaction_date = transaction_date self.amount = amount self.description = description - self.suffix = suffix + self.polarity = polarity def __getitem__(self, x): return self.__dict__[x] @@ -84,20 +84,20 @@ class Transaction: description: str amount: float date: str = Field(alias="transaction_date") - suffix: Optional[str] = None + polarity: Optional[str] = None # avoid storing config logic, since the Transaction object is used to create # a single unique hash which should not change auto_polarity: bool = Field(default=True, init=True, repr=False) - def as_raw_dict(self, show_suffix=False): + def as_raw_dict(self, show_polarity=False): """Returns stringified dictionary version of the transaction""" items = { Columns.DATE.value: self.date, Columns.DESCRIPTION.value: self.description, Columns.AMOUNT.value: str(self.amount), } - if show_suffix: - items[Columns.SUFFIX] = self.suffix + if show_polarity: + items[Columns.POLARITY] = self.polarity return items @field_validator("description", mode="after") @@ -129,13 +129,13 @@ def treat_parenthesis_enclosure_as_credit(self: ArgsKwargs | Any) -> "ArgsKwargs amount: str = self.kwargs[Columns.AMOUNT] if isinstance(amount, str): if amount.startswith("(") and amount.endswith(")"): - self.kwargs[Columns.SUFFIX] = "CR" + self.kwargs[Columns.POLARITY] = "CR" return self @model_validator(mode="after") def convert_credit_amount_to_negative(self: "Transaction") -> "Transaction": """ - Converts transactions with a suffix of "CR" or "+" to positive + Converts transactions with a polarity of "CR" or "+" to positive """ # avoid negative zero if self.amount == 0: @@ -144,7 +144,7 @@ def convert_credit_amount_to_negative(self: "Transaction") -> "Transaction": if not self.auto_polarity: return self - if self.suffix in ("CR", "+"): + if self.polarity in ("CR", "+"): self.amount = abs(self.amount) else: @@ -152,4 +152,4 @@ def convert_credit_amount_to_negative(self: "Transaction") -> "Transaction": return self def __str__(self): - return json.dumps(self.as_raw_dict(show_suffix=True)) + return json.dumps(self.as_raw_dict(show_polarity=True)) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 9313cc88..2f54e6c2 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -93,7 +93,7 @@ def test_monopoly_output(cli_runner: CliRunner): assert result.exit_code == 0 assert "1 statement(s) processed" in result.output - assert "input.pdf -> example-credit-2023-07-ae15d6.csv\n" in result.output + assert "input.pdf -> example-credit-2023-07-74498f.csv\n" in result.output def test_monopoly_no_pdf(cli_runner: CliRunner): diff --git a/tests/unit/test_credit_statement.py b/tests/unit/test_credit_statement.py index bebb1699..fc44df35 100644 --- a/tests/unit/test_credit_statement.py +++ b/tests/unit/test_credit_statement.py @@ -81,10 +81,13 @@ def test_inject_prev_month_balance(credit_statement): transaction_date="2024-01-01", description="bar", amount=-123.12, - suffix=None, + polarity=None, ), Transaction( - transaction_date="2024-01-01", description="foo", amount=-99.99, suffix=None + transaction_date="2024-01-01", + description="foo", + amount=-99.99, + polarity=None, ), ] assert result[0] in expected diff --git a/tests/unit/test_safety_check.py b/tests/unit/test_safety_check.py index 9063fd29..bec2cd81 100644 --- a/tests/unit/test_safety_check.py +++ b/tests/unit/test_safety_check.py @@ -40,13 +40,13 @@ def test_debit_safety_check(debit_statement: DebitStatement): debit_statement.transactions = [ Transaction( - transaction_date="23/01", description="foo", amount=10.0, suffix="CR" + transaction_date="23/01", description="foo", amount=10.0, polarity="CR" ), Transaction( - transaction_date="24/01", description="bar", amount=20.0, suffix="CR" + transaction_date="24/01", description="bar", amount=20.0, polarity="CR" ), Transaction( - transaction_date="25/01", description="baz", amount=-2.5, suffix="DR" + transaction_date="25/01", description="baz", amount=-2.5, polarity="DR" ), ] @@ -66,13 +66,13 @@ def test_debit_safety_check_failure(debit_statement: DebitStatement): debit_statement.document = document debit_statement.transactions = [ Transaction( - transaction_date="23/01", description="foo", amount=10.0, suffix="CR" + transaction_date="23/01", description="foo", amount=10.0, polarity="CR" ), Transaction( - transaction_date="24/01", description="bar", amount=20.0, suffix="CR" + transaction_date="24/01", description="bar", amount=20.0, polarity="CR" ), Transaction( - transaction_date="25/01", description="baz", amount=-2.5, suffix="DR" + transaction_date="25/01", description="baz", amount=-2.5, polarity="DR" ), ] diff --git a/tests/unit/test_statement_process_line.py b/tests/unit/test_statement_process_line.py index 06fcbef9..2adcdb9d 100644 --- a/tests/unit/test_statement_process_line.py +++ b/tests/unit/test_statement_process_line.py @@ -25,13 +25,13 @@ def test_get_transactions(statement: BaseStatement): transaction_date="19/06", description="YA KUN KAYA TOAST", amount=-3.2, - suffix=None, + polarity=None, ), Transaction( transaction_date="20/06", description="FAIRPRICE FINEST", amount=-9.9, - suffix=None, + polarity=None, ), ] assert transactions == expected @@ -51,7 +51,7 @@ def test_check_bound(statement: BaseStatement): transaction_date="19/06", description="YA KUN KAYA TOAST", amount=-3.2, - suffix=None, + polarity=None, ), ] statement.config.transaction_bound = None @@ -75,7 +75,7 @@ def test_get_multiline_transactions(statement: BaseStatement): transaction_date="02 Aug", description="SHOPEE CCY FEE 1.25 SINGAPORE SG", amount=-3.2, - suffix=None, + polarity=None, ) ] assert transactions == expected @@ -90,7 +90,7 @@ def test_process_match_multiline_description(statement: BaseStatement): "transaction_date": "04 Aug", "description": "SHOPEE", "amount": "3.20", - "suffix": None, + "polarity": None, } match = TransactionMatch( match=re.search("foo", "foo"), @@ -105,7 +105,7 @@ def test_process_match_multiline_description(statement: BaseStatement): "transaction_date": "04 Aug", "amount": "3.20", "description": "SHOPEE", - "suffix": None, + "polarity": None, } context = MatchContext(line=line, lines=lines, idx=0, description="SHOPEE") match = statement.process_match(match, context) diff --git a/tests/unit/test_statement_refund.py b/tests/unit/test_statement_refund.py index c66bf1fb..8e27ea1a 100644 --- a/tests/unit/test_statement_refund.py +++ b/tests/unit/test_statement_refund.py @@ -18,13 +18,13 @@ def test_statement_process_refund(statement: BaseStatement): transaction_date="08 SEP", description="AIRBNB * FOO123 456 GB", amount=343.01, - suffix="CR", + polarity="CR", ), Transaction( transaction_date="14 AUG", description="AIRBNB * FOO123 456 GB", amount=-343.01, - suffix=None, + polarity=None, ), ] assert statement.transactions == expected_transactions