Fix xtb parsing & warnings

investd/reports/__init__.py

@@ -1,6 +1,9 @@
+import os
 from datetime import datetime
 from pathlib import Path
 
+os.environ["JUPYTER_PLATFORM_DIRS"] = "1"
+
 import jupytext
 from nbconvert.exporters import HTMLExporter, export
 from nbconvert.preprocessors import ExecutePreprocessor

investd/reports/overview.py

@@ -86,11 +86,11 @@ def highlight_total_row(row: pd.Series) -> list[str]:
 # ### Invested amount over time
 
 # %%
-df = views.amount_over_time(df_tx, period="Y")
+df = views.amount_over_time(df_tx, period="YE")
 display(df)
 
 # %%
-df = views.amount_over_time(df_tx, period="M").iloc[-12:]
+df = views.amount_over_time(df_tx, period="ME").iloc[-12:]
 display(df)
 
 # %%

investd/sources/xtb.py

@@ -16,7 +16,7 @@
 COL_TYPE = "Type"
 
 INPUT_COLUMNS = [COL_AMOUNT, COL_COMMENT, COL_ID, COL_SYMBOL, COL_TIME, COL_TYPE]
-SUPPORTED_TYPES = {"Stocks/ETF purchase", "Stocks/ETF sale"}
+SUPPORTED_TYPES = {"Stocks/ETF purchase": Action.BUY, "Stocks/ETF sale": Action.SELL}
 
 
 class XTB(SourceBase):
@@ -47,10 +47,10 @@ def parse_source_file(self, path: Path) -> Iterator[Transaction]:
         return map(lambda i_row: self._convert(i_row[1]), df.iterrows())
 
     def _convert(self, record: pd.Series) -> Transaction:
-        action, quantity, price = XTB.parse_comment(record[COL_COMMENT])
+        quantity, price = XTB.parse_comment(record[COL_COMMENT])
         return Transaction(
             id=str(record[COL_ID]),
-            timestamp=pd.to_datetime(record[COL_TIME]),
+            timestamp=record[COL_TIME],
             symbol=record[COL_SYMBOL],
             type=AssetType.ETF,
             platform=self.source_name,
@@ -60,16 +60,16 @@ def _convert(self, record: pd.Series) -> Transaction:
             price=price,
             exchange_rate=abs(record[COL_AMOUNT]) / price / quantity,
             amount_ref_currency=abs(record[COL_AMOUNT]),
-            action=Action(action.upper()),
+            action=Action(SUPPORTED_TYPES[record[COL_TYPE]]),
         )
 
     @staticmethod
-    def parse_comment(comment: str) -> tuple[str, float, float]:
+    def parse_comment(comment: str) -> tuple[float, float]:
         for match in re.finditer(
-            r"(?P<action>BUY|SELL) (?P<quantity>[\d.]+)(/\d+)? @ (?P<price>[\d.]+)",
+            r"(?P<quantity>[\d.]+)(/\d+)? @ (?P<price>[\d.]+)",
             comment,
         ):
-            return match["action"], float(match["quantity"]), float(match["price"])
+            return float(match["quantity"]), float(match["price"])
         raise ValueError(
             f"No matches found for comment pattern in Comment: '{comment}'"
         )

investd/views.py

@@ -42,7 +42,7 @@ def total_invested_ref_currency(df_tx: pd.DataFrame) -> float:
 def to_nice_df(ndf: pd.DataFrame | pd.Series, columns: Iterable[str]) -> pd.DataFrame:
     """Transforms series into a nice looking dataframe in a notebook."""
     df = pd.DataFrame(ndf)
-    df = df.applymap(round, ndigits=2)  # type: ignore
+    df = df.map(round, ndigits=2)  # type: ignore
     df.columns = pd.Index(columns).map(str)
     df.index.set_names(None, inplace=True)
     return df
@@ -61,7 +61,7 @@ def invested_ref_amount_by_col(df_tx: pd.DataFrame, col: str) -> pd.DataFrame:
     Generates a Series with invested amount by asset type in the reference currency.
     """
     df_tx = _add_signed_cols(df_tx)
-    grouped = df_tx.groupby(col)["amount_ref_currency_signed"].sum()
+    grouped = df_tx.groupby(col, observed=True)["amount_ref_currency_signed"].sum()
     ref_currency = config.INVESTD_REF_CURRENCY
     df = to_nice_df(grouped, columns=[str(ref_currency)])
     df = add_pct_col(df, based_on_col=str(ref_currency))
@@ -74,7 +74,7 @@ def amounts_by_currency(df_tx: pd.DataFrame) -> pd.DataFrame:
     original currency and reference currency.
     """
     df_tx = _add_signed_cols(df_tx)
-    df_cur = df_tx.groupby("currency")[
+    df_cur = df_tx.groupby("currency", observed=True)[
         ["amount_signed", "amount_ref_currency_signed"]
     ].sum()
     ref_currency = config.INVESTD_REF_CURRENCY
@@ -83,15 +83,13 @@ def amounts_by_currency(df_tx: pd.DataFrame) -> pd.DataFrame:
     return df_cur
 
 
-def invested_amount_original_cur_by_col(
-    df_tx: pd.DataFrame, col: str
-) -> pd.DataFrame | pd.Series:
+def invested_amount_original_cur_by_col(df_tx: pd.DataFrame, col: str) -> pd.Series:
     """
     Aggregate invested amount by currency.
     """
     df_tx = _add_signed_cols(df_tx)
     grouping: list[str] | str = ["currency", col] if col != "currency" else col
-    return df_tx.groupby(grouping)["amount_signed"].sum()
+    return df_tx.groupby(grouping, observed=True)["amount_signed"].sum()
 
 
 def amount_over_time(df_tx: pd.DataFrame, period: str) -> pd.DataFrame:
@@ -137,7 +135,7 @@ def portfolio_value(
     exchange_rate_to_ref_cur = (
         df_portfolio["currency"]
         .map(
-            lambda cur: quotes.get(f"{cur}{ref_currency}=X")
+            lambda cur: quotes.get(f"{cur}{ref_currency}=X")  # type: ignore
             if cur != ref_currency
             else 1
         )

tests/resources/data/sources/xtb/xtb-statement.csv

@@ -1,8 +1,10 @@
-ID;Type;Time;Symbol;Comment;Amount
-130876160;Stocks/ETF purchase;09.12.2022 12:09:37;IBC5.DE;OPEN BUY 5/7 @ 5.2080;-26.04
-130820511;Stocks/ETF purchase;09.12.2022 10:00:25;IBC5.DE;OPEN BUY 2/7 @ 5.2080;-10.42
-130801242;Stocks/ETF purchase;09.12.2022 09:14:24;SXR8.DE;OPEN BUY 1 @ 388.90;-388.9
-130658625;Deposit;08.12.2022 19:15:04;;Pekao S.A. deposit, Pekao S.A. provider transaction id=NL9S2567123361Z, Pekao S.A. merchant reference id=65165, id=1123;426.46
-120055884;Stocks/ETF purchase;09.11.2022 14:57:11;IBC5.DE;OPEN BUY 7 @ 5.0220;-35.15
-120045385;Stocks/ETF purchase;09.11.2022 14:37:19;SXR8.DE;OPEN BUY 1 @ 392.20;-392.2
-207157984;Dividend;29.12.2021 12:00:01;VHYD.UK;VHYD.UK USD 0.5074/ SHR;12.34
+ID;Type;Time;Symbol;Comment;Amount;Source
+130876160;Stocks/ETF purchase;09.12.2022 12:09:37;IBC5.DE;OPEN BUY 5/7 @ 5.2080;-26.04;My trades
+130820511;Stocks/ETF purchase;09.12.2022 10:00:25;IBC5.DE;OPEN BUY 2/7 @ 5.2080;-10.42;My trades
+130801242;Stocks/ETF purchase;09.12.2022 09:14:24;SXR8.DE;OPEN BUY 1 @ 388.90;-388.9;My trades
+130658625;Deposit;08.12.2022 19:15:04;;Pekao S.A. deposit, Pekao S.A. provider transaction id=NL9S2567123361Z, Pekao S.A. merchant reference id=65165, id=1123;426.46;My trades
+120055884;Stocks/ETF purchase;09.11.2022 14:57:11;IBC5.DE;OPEN BUY 7 @ 5.0220;-35.15;My trades
+120045385;Stocks/ETF purchase;09.11.2022 14:37:19;SXR8.DE;OPEN BUY 1 @ 392.20;-392.2;My trades
+207157984;Dividend;29.12.2021 12:00:01;VHYD.UK;VHYD.UK USD 0.5074/ SHR;12.34;My trades
+566955344;Stocks/ETF sale;20.06.2024 09:05:00;ETFBWTECH.PL;CLOSE BUY 8 @ 132.02;1056.16;My trades
+483020694;Stocks/ETF purchase;04.01.2024 15:29:58;ETFBWTECH.PL;OPEN BUY 8 @ 128.26;-1026.08;My trades

tests/sources/test_xtb.py

@@ -31,7 +31,7 @@ def test_parse_xtb_xlsx(path_xtb_xlsx: Path) -> None:
 def test_parse_xtb_csv(path_xtb_csv: Path) -> None:
     xtb_source = XTB()
     txs = list(xtb_source.parse_source_file(path_xtb_csv))
-    assert len(txs) == 5
+    assert len(txs) == 7
 
     tx: Transaction = txs[0]
     assert tx.id == "130876160"
@@ -49,8 +49,8 @@ def test_parse_xtb_csv(path_xtb_csv: Path) -> None:
 
 
 def test_parse_comment() -> None:
-    assert XTB.parse_comment("OPEN BUY 1 @ 467.03") == ("BUY", 1, 467.03)
-    assert XTB.parse_comment("OPEN BUY 10 @ 30.680") == ("BUY", 10, 30.68)
+    assert XTB.parse_comment("OPEN BUY 1 @ 467.03") == (1, 467.03)
+    assert XTB.parse_comment("OPEN BUY 10 @ 30.680") == (10, 30.68)
 
 
 def test_infer_currency() -> None:

tests/test_views.py

@@ -3,7 +3,7 @@
 
 import pandas as pd
 import pytest
-from pandas.testing import assert_frame_equal
+from pandas.testing import assert_frame_equal, assert_series_equal
 
 from investd import views
 from investd.common import AssetType as Asset
@@ -31,44 +31,53 @@ def test_invested_ref_amount_col(
     )
 
 
-def _multi_cat_series(
-    values: Iterable[Any], first_level: list[Any], second_level: list[Any]
+def _series_2levels_categorical_index(
+    data: Iterable[Any], first_level: list[Any], second_level: list[Any]
 ) -> pd.Series:
     return pd.Series(
-        values,
+        data,
         index=pd.MultiIndex.from_product(
             [
                 pd.Categorical(first_level),
                 pd.Categorical(second_level),
-            ]
-        ),  # type: ignore
-    )
+            ]  # type: ignore
+        ),
+    ).dropna()
 
 
 @pytest.mark.parametrize(
     "col, expected",
     [
-        ("currency", pd.Series({Cur.USD: 190.0, Cur.EUR: 300.0})),
+        (
+            "currency",
+            pd.Series(
+                data=[190.0, 300.0],
+                index=pd.CategoricalIndex([Cur.USD, Cur.EUR]),  # type: ignore
+            ),
+        ),
         (
             "type",
-            _multi_cat_series(
-                [190.0, 0.0, 0.0, 300.0], [Cur.USD, Cur.EUR], [Asset.Stock, Asset.ETF]
+            _series_2levels_categorical_index(
+                data=[190.0, None, None, 300.0],
+                first_level=[Cur.USD, Cur.EUR],
+                second_level=[Asset.Stock, Asset.ETF],
             ),
         ),
         (
             "platform",
-            _multi_cat_series(
-                [190.0, 0.0, 0.0, 300.0], [Cur.USD, Cur.EUR], ["revolut_stocks", "xtb"]
+            _series_2levels_categorical_index(
+                data=[190.0, None, None, 300.0],
+                first_level=[Cur.USD, Cur.EUR],
+                second_level=["revolut_stocks", "xtb"],
             ),
         ),
     ],
 )
 def test_invested_amount_currency_amount_col(
     df_tx_minimal: pd.DataFrame, col: str, expected: pd.Series
 ) -> None:
-    assert views.invested_amount_original_cur_by_col(df_tx_minimal, col).equals(
-        expected
-    )
+    actual = views.invested_amount_original_cur_by_col(df_tx_minimal, col)
+    assert_series_equal(actual, expected, check_names=False)
 
 
 def test_amounts_by_currency(df_tx_minimal: pd.DataFrame) -> None:
@@ -78,23 +87,24 @@ def test_amounts_by_currency(df_tx_minimal: pd.DataFrame) -> None:
             "PLN": [849, 1500],
             "%": [36.1, 63.9],
         },
-        index=[Cur.USD, Cur.EUR],
+        index=pd.CategoricalIndex([Cur.USD, Cur.EUR]),  # type: ignore
     )
-    assert views.amounts_by_currency(df_tx_minimal).equals(expected)
+    actual = views.amounts_by_currency(df_tx_minimal)
+    assert_frame_equal(actual, expected, check_names=False)
 
 
 @pytest.mark.parametrize(
     "period, expected",
     [
         (
-            "Y",
+            "YE",
             pd.DataFrame(
                 {"PLN": [450, 1899], "Cumulated PLN": [450, 2349]},
                 index=pd.PeriodIndex(["2021", "2022"], freq="Y"),
             ),
         ),
         (
-            "M",
+            "ME",
             pd.DataFrame(
                 {"PLN": [450, 2175, -276], "Cumulated PLN": [450, 2625, 2349]},
                 index=pd.PeriodIndex(["2021-12", "2022-01", "2022-02"], freq="M"),