diff --git a/investd/__main__.py b/investd/__main__.py index f4a0736..7a7857c 100644 --- a/investd/__main__.py +++ b/investd/__main__.py @@ -1,13 +1,13 @@ +import datetime import logging import os -from datetime import date from pathlib import Path from typing import Optional import click -from investd import quotes, reports, sources -from investd.config import INVESTD_PERSIST, init_dirs +from investd import config, quotes, reports, sources +from investd.exceptions import InvestdException from investd.transaction import TX_FILENAME APP_NAME = "investd" @@ -32,13 +32,11 @@ @click.option( "--output", type=click.Path(dir_okay=False, writable=True), - default=INVESTD_PERSIST / TX_FILENAME, + default=config.INVESTD_PERSIST / TX_FILENAME, help="Output path", ) def ingest_sources_cmd(output: Path): - df_tx = sources.ingest_all() - log.info(f"Writing {output}") - df_tx.to_csv(output, index=False) + sources.ingest_to_path(output) @cli.command(name="report") @@ -62,9 +60,8 @@ def ingest_sources_cmd(output: Path): ) def report_cmd(report: str, ingest: bool, date: Optional[str]): if ingest: - df_tx = sources.ingest_all() - df_tx.to_csv(INVESTD_PERSIST / TX_FILENAME, index=False) - os.environ["REPORT_DATE"] = date + sources.ingest_to_path() + os.environ["REPORT_DATE"] = date or str(datetime.date.today()) path_output = reports.generate_report(report) log.info(f"Report created: {path_output}") print(path_output) @@ -81,8 +78,8 @@ def report_cmd(report: str, ingest: bool, date: Optional[str]): @click.option("--symbols", "-y", default=None, help="Symbols e.g. AAPL,CDR.PL") def quotes_cmd(start: Optional[str], end: Optional[str], symbols: Optional[str]): quotes.download_quotes_to_csv( - start_date=date.fromisoformat(start) if start else None, - end_date=date.fromisoformat(end) if end else None, + start_date=datetime.date.fromisoformat(start) if start else None, + end_date=datetime.date.fromisoformat(end) if end else None, symbols=symbols.split(",") if symbols else None, include_exchange_rates=True, ) @@ -90,7 +87,12 @@ def quotes_cmd(start: Optional[str], end: Optional[str], symbols: Optional[str]) def main() -> None: init_dirs() - cli() + try: + cli() + except InvestdException as exc: + log.error(exc.msg, exc_info=True) + click.echo(exc.msg) + exit(1) if __name__ == "__main__": diff --git a/investd/exceptions.py b/investd/exceptions.py new file mode 100644 index 0000000..be15d2d --- /dev/null +++ b/investd/exceptions.py @@ -0,0 +1,6 @@ +class InvestdException(Exception): + msg: str + + +class NoTransactions(InvestdException): + msg = "No transactions! Make sure you have non-empty files in the sources folder according to each source." diff --git a/investd/quotes.py b/investd/quotes.py index fcf9927..d00be83 100644 --- a/investd/quotes.py +++ b/investd/quotes.py @@ -1,3 +1,4 @@ +import logging from datetime import date from pathlib import Path from typing import Iterable, Optional @@ -5,10 +6,12 @@ import pandas as pd import yfinance +from investd import config from investd.common import Currency -from investd.config import INVESTD_PERSIST, INVESTD_REF_CURRENCY from investd.transaction import load_transactions +log = logging.getLogger(__name__) + QUOTES_FILENAME = "quotes.csv" SYMBOL_EXCH_ADJUST = {"FR": "PA", "UK": "L", "PL": "WA"} SYMBOL_NAME_ADJUST = {"DAXEX": "EXS1"} @@ -74,18 +77,22 @@ def download_quotes_to_csv( id_vars=["date"], value_vars=df.columns, var_name="symbol", value_name="price" ) df = df.sort_values(["date", "symbol"]) - - df.to_csv(output_path or INVESTD_PERSIST / QUOTES_FILENAME, index=False) + output_path = output_path or (config.INVESTD_PERSIST / QUOTES_FILENAME) + output_path.parent.mkdir(exist_ok=True, parents=True) + df.to_csv(output_path, index=False) def load_quotes() -> pd.DataFrame: - return pd.read_csv(INVESTD_PERSIST / QUOTES_FILENAME, parse_dates=["date"]) + if not (config.INVESTD_PERSIST / QUOTES_FILENAME).exists(): + log.warn("Quotes files does not exist.") + return pd.DataFrame(columns=["date", "symbol", "price"]) + return pd.read_csv((config.INVESTD_PERSIST / QUOTES_FILENAME), parse_dates=["date"]) def extract_exchange_rates_symbols( df_tx: pd.DataFrame, ref_currency: Optional[Currency] = None ) -> set[str]: - ref_currency = ref_currency or INVESTD_REF_CURRENCY + ref_currency = ref_currency or config.INVESTD_REF_CURRENCY return { f"{cur}{ref_currency}=X" for cur in df_tx["currency"].unique() diff --git a/investd/reports/__init__.py b/investd/reports/__init__.py index 0275a8b..54e02fe 100644 --- a/investd/reports/__init__.py +++ b/investd/reports/__init__.py @@ -5,11 +5,14 @@ from nbconvert.exporters import HTMLExporter, export from nbconvert.preprocessors import ExecutePreprocessor -from investd.config import INVESTD_REF_CURRENCY, INVESTD_REPORTS +from investd import config +from investd.exceptions import NoTransactions +from investd.transaction import load_transactions def generate_report(notebook_name: str) -> Path: path_notebook = Path(__file__).parent / f"{notebook_name}.py" + _assert_transactions() notebook = jupytext.read(path_notebook) preprocessor = ExecutePreprocessor(timeout=10, kernel_name="python3") nb_executed, _ = preprocessor.preprocess( @@ -32,8 +35,15 @@ def generate_report(notebook_name: str) -> Path: def _save_report(name: str, content: str) -> Path: today = datetime.today().strftime("%Y-%m-%d") - filename = f"{name}_{today}_{INVESTD_REF_CURRENCY}.html" - report_path = INVESTD_REPORTS / filename + filename = f"{name}_{today}_{config.INVESTD_REF_CURRENCY}.html" + report_path = config.INVESTD_REPORTS / filename + report_path.parent.mkdir(exist_ok=True, parents=True) with report_path.open("w") as out_file: out_file.write(content) return report_path + + +def _assert_transactions() -> None: + df_tx = load_transactions() + if df_tx.empty: + raise NoTransactions() diff --git a/investd/reports/overview.py b/investd/reports/overview.py index 8c13824..5357600 100644 --- a/investd/reports/overview.py +++ b/investd/reports/overview.py @@ -6,10 +6,9 @@ import numpy as np import pandas as pd import seaborn as sns -from IPython.display import Markdown, display +from IPython.display import display -from investd import views -from investd.config import INVESTD_REF_CURRENCY +from investd import config, views from investd.quotes import load_quotes from investd.transaction import load_transactions @@ -29,7 +28,7 @@ pd.DataFrame( { "Reporting date": [report_date], - "Reference currency": [INVESTD_REF_CURRENCY], + "Reference currency": [config.INVESTD_REF_CURRENCY], "Created date": [now.strftime("%Y-%m-%d")], }, index=[""], @@ -48,7 +47,7 @@ # %% ref_amount_cols = [ - col for col in df_portfolio.columns if str(INVESTD_REF_CURRENCY) in col + col for col in df_portfolio.columns if str(config.INVESTD_REF_CURRENCY) in col ] row_total = df_portfolio.loc[:, ref_amount_cols].sum(axis=0) row_total = pd.DataFrame(row_total, columns=["Total"]).transpose() @@ -70,7 +69,7 @@ def highlight_total_row(row: pd.Series) -> list[str]: df = views.invested_ref_amount_by_col(df_tx, "type") display(df) -fig = df.plot.pie(y=str(INVESTD_REF_CURRENCY)) +fig = df.plot.pie(y=str(config.INVESTD_REF_CURRENCY)) fig.get_legend().remove() # %% [markdown] @@ -80,7 +79,7 @@ def highlight_total_row(row: pd.Series) -> list[str]: df = views.amounts_by_currency(df_tx) display(df) -fig = df.plot.pie(y=str(INVESTD_REF_CURRENCY)) +fig = df.plot.pie(y=str(config.INVESTD_REF_CURRENCY)) fig.get_legend().remove() # %% [markdown] @@ -99,6 +98,6 @@ def highlight_total_row(row: pd.Series) -> list[str]: fig.autofmt_xdate() cumsum = df_tx["amount_ref_currency"].cumsum() -df_cum = pd.DataFrame({INVESTD_REF_CURRENCY: cumsum, "Date": df_tx["timestamp"]}) +df_cum = pd.DataFrame({config.INVESTD_REF_CURRENCY: cumsum, "Date": df_tx["timestamp"]}) -fig = sns.lineplot(x="Date", y=INVESTD_REF_CURRENCY, data=df_cum, ax=ax) +fig = sns.lineplot(x="Date", y=config.INVESTD_REF_CURRENCY, data=df_cum, ax=ax) diff --git a/investd/sources/__init__.py b/investd/sources/__init__.py index 20edfe6..48c33c0 100644 --- a/investd/sources/__init__.py +++ b/investd/sources/__init__.py @@ -1,13 +1,18 @@ +import logging from dataclasses import fields from itertools import chain -from typing import Type +from pathlib import Path +from typing import Optional, Type import pandas as pd +from investd import config from investd.sources import bonds, bossa, revolut_stocks, xtb from investd.sources.base import SourceBase from investd.transaction import Transaction +log = logging.getLogger(__name__) + sources: list[Type[SourceBase]] = [ xtb.XTB, revolut_stocks.RevolutStocks, @@ -23,3 +28,11 @@ def ingest_all() -> pd.DataFrame: ) df_tx.sort_values(by="timestamp", ascending=True, inplace=True) return df_tx + + +def ingest_to_path(path_output: Optional[Path] = None) -> None: + path_output = path_output or (config.INVESTD_PERSIST / "transactions.csv") + df_tx = ingest_all() + log.info(f"Writing {path_output}") + path_output.parent.mkdir(exist_ok=True, parents=True) + df_tx.to_csv(path_output, index=False) diff --git a/investd/sources/base.py b/investd/sources/base.py index a637f75..8ab447b 100644 --- a/investd/sources/base.py +++ b/investd/sources/base.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Generator, Iterable -from investd.config import INVESTD_SOURCES +from investd import config from investd.transaction import Transaction log = logging.getLogger(__name__) @@ -20,7 +20,7 @@ def parse_source_file(self, tx_path: Path) -> Iterable[Transaction]: pass def load_transactions(self) -> Generator[Transaction, None, None]: - for path in (INVESTD_SOURCES / self.source_name).glob("*"): + for path in (config.INVESTD_SOURCES / self.source_name).glob("*"): log.info(f"Loading {path}") for tx in self.parse_source_file(path): yield tx diff --git a/investd/transaction.py b/investd/transaction.py index 86b7c9a..4076366 100644 --- a/investd/transaction.py +++ b/investd/transaction.py @@ -6,8 +6,8 @@ import pandas as pd from pydantic.dataclasses import dataclass +from investd import config from investd.common import Action, AssetType, Currency -from investd.config import INVESTD_PERSIST TX_FILENAME = "transactions.csv" @@ -30,8 +30,12 @@ class Transaction: def load_transactions() -> pd.DataFrame: """Load transactions in persistence as a DataFrame.""" - path = INVESTD_PERSIST / TX_FILENAME - df_tx = pd.read_csv(path, converters=_enum_fields(Transaction)) + path = config.INVESTD_PERSIST / TX_FILENAME + df_tx = ( + pd.read_csv(path, converters=_enum_fields(Transaction)) + if path.exists() + else pd.DataFrame() + ) schema = _to_pandas_schema(Transaction) return df_tx.astype(schema) diff --git a/investd/views.py b/investd/views.py index 179321d..d161300 100644 --- a/investd/views.py +++ b/investd/views.py @@ -9,8 +9,8 @@ import pandas as pd +from investd import config from investd.common import Action -from investd.config import INVESTD_REF_CURRENCY def _add_signed_cols(df_tx: pd.DataFrame) -> pd.DataFrame: @@ -62,8 +62,9 @@ def invested_ref_amount_by_col(df_tx: pd.DataFrame, col: str) -> pd.Series: """ df_tx = _add_signed_cols(df_tx) grouped = df_tx.groupby(col)["amount_ref_currency_signed"].sum() - df = to_nice_df(grouped, columns=[str(INVESTD_REF_CURRENCY)]) - df = add_pct_col(df, based_on_col=str(INVESTD_REF_CURRENCY)) + ref_currency = config.INVESTD_REF_CURRENCY + df = to_nice_df(grouped, columns=[str(ref_currency)]) + df = add_pct_col(df, based_on_col=str(ref_currency)) return df @@ -76,10 +77,9 @@ def amounts_by_currency(df_tx: pd.DataFrame) -> pd.Series: df_cur = df_tx.groupby("currency")[ ["amount_signed", "amount_ref_currency_signed"] ].sum() - df_cur = to_nice_df( - df_cur, columns=["Original currency", str(INVESTD_REF_CURRENCY)] - ) - df_cur = add_pct_col(df_cur, str(INVESTD_REF_CURRENCY)) + ref_currency = config.INVESTD_REF_CURRENCY + df_cur = to_nice_df(df_cur, columns=["Original currency", str(ref_currency)]) + df_cur = add_pct_col(df_cur, str(ref_currency)) return df_cur @@ -106,9 +106,8 @@ def amount_over_time(df_tx: pd.DataFrame, period: str) -> pd.DataFrame: ) df_ot.index = df_ot.index.to_period() df_ot["cumsum"] = df_ot["amount_ref_currency_signed"].cumsum() - df_ot = to_nice_df( - df_ot, columns=[str(INVESTD_REF_CURRENCY), f"Cumulated {INVESTD_REF_CURRENCY}"] - ) + ref_currency = config.INVESTD_REF_CURRENCY + df_ot = to_nice_df(df_ot, columns=[str(ref_currency), f"Cumulated {ref_currency}"]) return df_ot @@ -134,11 +133,12 @@ def portfolio_value( df_portfolio["amount_at_date"] = ( df_portfolio["quote"] * df_portfolio["quantity_signed"] ) + ref_currency = config.INVESTD_REF_CURRENCY exchange_rate_to_ref_cur = ( df_portfolio["currency"] .map( - lambda cur: quotes.get(f"{cur}{INVESTD_REF_CURRENCY}=X") - if cur != INVESTD_REF_CURRENCY + lambda cur: quotes.get(f"{cur}{ref_currency}=X") + if cur != ref_currency else 1 ) .astype(float) @@ -149,9 +149,9 @@ def portfolio_value( df_portfolio = df_portfolio.rename( { "amount_at_date": "Amount at date", - "amount_ref_currency_at_date": f"Amount at date {INVESTD_REF_CURRENCY}", + "amount_ref_currency_at_date": f"Amount at date {ref_currency}", "amount_signed": "Invested amount", - "amount_ref_currency_signed": f"Invested amount {INVESTD_REF_CURRENCY}", + "amount_ref_currency_signed": f"Invested amount {ref_currency}", "quantity_signed": "Quantity", }, axis=1, diff --git a/logging.yaml b/logging.yaml index a55387f..a3e0789 100644 --- a/logging.yaml +++ b/logging.yaml @@ -1,5 +1,5 @@ version: 1 -disable_existing_loggers: false +disable_existing_loggers: true formatters: default: format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s' diff --git a/tests/conftest.py b/tests/conftest.py index b2589bd..f37fb58 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,8 +4,8 @@ import pandas as pd import pytest +from investd import config from investd.common import Action, AssetType, Currency -from investd.config import INVESTD_PERSIST, INVESTD_REPORTS, INVESTD_SOURCES from investd.quotes import QUOTES_FILENAME @@ -16,35 +16,35 @@ def path_resources() -> Path: @pytest.fixture def setup_reports() -> Generator[None, None, None]: - INVESTD_REPORTS.mkdir(exist_ok=True) + config.INVESTD_REPORTS.mkdir(exist_ok=True) yield - for path in INVESTD_REPORTS.glob("*.html"): + for path in config.INVESTD_REPORTS.glob("*.html"): path.unlink() @pytest.fixture def path_revolut_csv() -> Path: - return INVESTD_SOURCES / "revolut_stocks/revolut-stocks-statement.csv" + return config.INVESTD_SOURCES / "revolut_stocks/revolut-stocks-statement.csv" @pytest.fixture def path_xtb_xlsx() -> Path: - return INVESTD_SOURCES / "xtb/xtb-statement.xlsx" + return config.INVESTD_SOURCES / "xtb/xtb-statement.xlsx" @pytest.fixture def path_xtb_csv() -> Path: - return INVESTD_SOURCES / "xtb/xtb-statement.csv" + return config.INVESTD_SOURCES / "xtb/xtb-statement.csv" @pytest.fixture def path_bonds_xls() -> Path: - return INVESTD_SOURCES / "bonds/bonds-statement.xls" + return config.INVESTD_SOURCES / "bonds/bonds-statement.xls" @pytest.fixture def path_bossa_csv() -> Path: - return INVESTD_SOURCES / "bossa/bossa-statement.csv" + return config.INVESTD_SOURCES / "bossa/bossa-statement.csv" @pytest.fixture @@ -85,13 +85,13 @@ def df_quotes_minimal() -> pd.DataFrame: @pytest.fixture def df_quotes() -> pd.DataFrame: - return pd.read_csv(INVESTD_PERSIST / QUOTES_FILENAME, parse_dates=["date"]) + return pd.read_csv(config.INVESTD_PERSIST / QUOTES_FILENAME, parse_dates=["date"]) @pytest.fixture def yfinance_quotes() -> pd.DataFrame: df_quotes = pd.read_csv( - INVESTD_PERSIST / "yfinance_quotes.csv", header=[0, 1], index_col=0 + config.INVESTD_PERSIST / "yfinance_quotes.csv", header=[0, 1], index_col=0 ) df_quotes.index = df_quotes.index.map(pd.to_datetime) return df_quotes diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..b7592e2 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,22 @@ +from pathlib import Path + +import pytest +from click.testing import CliRunner +from pytest import MonkeyPatch + +import investd +from investd.__main__ import cli +from investd.exceptions import NoTransactions + + +@pytest.fixture +def empty_data_dir(monkeypatch: MonkeyPatch, tmp_path: Path) -> Path: + monkeypatch.setattr(investd.config, "INVESTD_DATA", tmp_path) + for conf in ["PERSIST", "SOURCES", "REPORTS"]: + monkeypatch.setattr(investd.config, f"INVESTD_{conf}", tmp_path / conf.lower()) + return tmp_path + + +def test_new_data_dir(empty_data_dir: Path) -> None: + result = CliRunner().invoke(cli, ["report"]) + assert isinstance(result.exception, NoTransactions) diff --git a/tests/test_reports.py b/tests/test_reports.py index 6dbe78d..472310d 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -1,8 +1,8 @@ -from investd.config import INVESTD_REPORTS +from investd import config from investd.reports import generate_report def test_generate_overview_report(setup_reports: None) -> None: - assert not list(INVESTD_REPORTS.glob("*.html")) + assert not list(config.INVESTD_REPORTS.glob("*.html")) report_path = generate_report("overview") - assert report_path in list(INVESTD_REPORTS.glob("*.html")) + assert report_path in list(config.INVESTD_REPORTS.glob("*.html"))