Mark down position (#1145)

- Support marking down problematic positions to zero
- Automatically include these positions on a blacklist
- Do not attempt to generate trades for blacklisted assets in alpha model

tests/cli/test_close_position.py

@@ -200,3 +200,61 @@ def test_close_all_simulate(
 
     state = State.read_json_file(state_file)
     assert len(state.portfolio.open_positions) == 1
+
+
+def test_mark_down_position(
+    environment: dict,
+    state_file: Path,
+):
+    """Perform close-position command without selling the asset
+
+    - End-to-end high level test for the command
+
+    - Create test EVM trading environment
+
+    - Initialise strategy command
+
+    - Perform buy only test trade command
+
+    - Perform close single command
+    """
+
+    # trade-executor init
+    cli = get_command(app)
+    with patch.dict(os.environ, environment, clear=True):
+        with pytest.raises(SystemExit) as e:
+            cli.main(args=["init"])
+        assert e.value.code == 0
+
+    # trade-executor perform-test-trade --buy-only
+    cli = get_command(app)
+    with patch.dict(os.environ, environment, clear=True):
+        with pytest.raises(SystemExit) as e:
+            cli.main(args=["perform-test-trade", "--buy-only"])
+        assert e.value.code == 0
+
+    state = State.read_json_file(state_file)
+    assert len(state.portfolio.open_positions) == 1
+
+    position = next(iter(state.portfolio.open_positions.values()))
+    environment["POSITION_ID"] = str(position.position_id)
+    environment["CLOSE_BY_SELL"] = "false"  # Do mark down
+    assert position.is_open()
+
+    # Run the command
+    cli = get_command(app)
+    with patch.dict(os.environ, environment, clear=True):
+        with pytest.raises(SystemExit) as e:
+            cli.main(args=["close-position"])
+        assert e.value.code == 0
+
+    state = State.read_json_file(state_file)
+    assert len(state.portfolio.open_positions) == 0
+
+    # Check marked down results look correct
+    position = state.portfolio.closed_positions[position.position_id]
+    assert position.is_marked_down()
+    assert position.is_closed()
+    assert not position.is_open()
+    assert not position.is_frozen()
+    assert not state.is_good_pair(position.pair)  # Blacklisted

tests/cli/test_console.py

@@ -6,10 +6,6 @@
 from tradeexecutor.cli.commands.app import app
 
 pytestmark = pytest.mark.skipif(os.environ.get("JSON_RPC_ETHEREUM") is None, reason="Set JSON_RPC_ETHEREUM environment variable torun this test")
-<<<<<<< Updated upstream
-=======
-
->>>>>>> Stashed changes
 
 
 @pytest.mark.skipif(os.environ.get("GITHUB_ACTIONS") == "true", reason="This test seems to block Github CI for some reason")

tests/mainnet_fork/test_frozen_position_and_blacklist.py

@@ -102,7 +102,7 @@ def anvil_bnb_chain_fork(large_busd_holder) -> str:
         yield launch.json_rpc_url
         # Wind down Ganache process after the test is complete
     finally:
-        launch.close(log_level=logging.INFO)
+        launch.close()
 
 
 @pytest.fixture
@@ -331,15 +331,16 @@ def runner(
 
 
 def test_buy_and_sell_blacklisted_asset(
-        strategy_path: Path,
-        web3: Web3,
-        hot_wallet: HotWallet,
-        pancakeswap_v2: UniswapV2Deployment,
-        universe_model: StaticUniverseModel,
-        state: State,
-        runner,
-        wbnb_busd_pair,
-        bit_busd_pair
+    logger: logging.Logger,
+    strategy_path: Path,
+    web3: Web3,
+    hot_wallet: HotWallet,
+    pancakeswap_v2: UniswapV2Deployment,
+    universe_model: StaticUniverseModel,
+    state: State,
+    runner,
+    wbnb_busd_pair,
+    bit_busd_pair
     ):
     """Try to buy/sell BIT token.
 

tradeexecutor/cli/close_position.py

@@ -45,6 +45,8 @@ def close_single_or_all_positions(
     interactive=True,
     position_id: int | None = None,
     unit_testing=False,
+    close_by_sell=True,
+    blacklist_marked_down=True,
 ):
     """Close single/all positions.
 
@@ -189,37 +191,60 @@ def close_single_or_all_positions(
         if confirmation != "y":
             raise CloseAllAborted()
 
-    for p in positions_to_close:
-        # Create trades to open the position
-        logger.info("Closing position %s", p)
+    if close_by_sell:
+        for p in positions_to_close:
+            # Create trades to open the position
+            logger.info("Closing position %s", p)
+
+            trades = position_manager.close_position(p)
+
+            assert len(trades) == 1
+            trade = trades[0]
+
+            # Compose the trades as approve() + swapTokenExact(),
+            # broadcast them to the blockchain network and
+            # wait for the confirmation
+            execution_model.execute_trades(
+                ts,
+                state,
+                trades,
+                routing_model,
+                routing_state,
+            )
 
-        trades = position_manager.close_position(p)
+            if not trade.is_success():
+                logger.error("Trade failed: %s", trade)
+                logger.error("Tx hash: %s", trade.blockchain_transactions[-1].tx_hash)
+                logger.error("Revert reason: %s", trade.blockchain_transactions[-1].revert_reason)
+                logger.error("Trade dump:\n%s", trade.get_debug_dump())
+                raise AssertionError("Trade to close position failed")
 
-        assert len(trades) == 1
-        trade = trades[0]
+            if p.notes is None:
+                p.notes = ""
 
-        # Compose the trades as approve() + swapTokenExact(),
-        # broadcast them to the blockchain network and
-        # wait for the confirmation
-        execution_model.execute_trades(
-            ts,
-            state,
-            trades,
-            routing_model,
-            routing_state,
-        )
+            p.add_notes_message(note)
+    else:
+        # TODO: Add accounting correction
+        # TODO: Add blacklist not to touch this position again
+        portfolio = state.portfolio
+        for p in positions_to_close:
+
+            if p.is_frozen():
+                del portfolio.open_positions[p.position_id]
+            elif p.is_open():
+                del portfolio.open_positions[p.position_id]
+            else:
+                raise NotImplementedError(f"Cannot mark down closed position: {p}")
+
+            portfolio.closed_positions[p.position_id] = p
 
-        if not trade.is_success():
-            logger.error("Trade failed: %s", trade)
-            logger.error("Tx hash: %s", trade.blockchain_transactions[-1].tx_hash)
-            logger.error("Revert reason: %s", trade.blockchain_transactions[-1].revert_reason)
-            logger.error("Trade dump:\n%s", trade.get_debug_dump())
-            raise AssertionError("Trade to close position failed")
+            p.mark_down()
 
-        if p.notes is None:
-            p.notes = ""
+            logger.info(f"Position was marked down and moved to closed positions: {p}")
 
-        p.add_notes_message(note)
+            # Also add to the blacklist
+            if blacklist_marked_down:
+                state.blacklist_asset(p.pair.base)
 
     gas_at_end = hot_wallet.get_native_currency_balance(web3)
     reserve_currency_at_end = state.portfolio.get_default_reserve_position().get_value()

tradeexecutor/cli/commands/close_position.py

@@ -59,7 +59,9 @@ def close_position(
     unit_testing: bool = shared_options.unit_testing,
     simulate: bool = shared_options.simulate,
 
-    position_id: Optional[int] = Option(None, envvar="POSITION_ID", help="Position id to close.")
+    position_id: Optional[int] = Option(None, envvar="POSITION_ID", help="Position id to close."),
+    close_by_sell: Optional[bool] = Option(True, envvar="CLOSE_BY_SELL", help="Attempt to close position by selling the underlying. If set to false, mark the position down to zero value."),
+    blacklist_marked_down: Optional[bool] = Option(True, envvar="BLACKLIST_MARKED_DOWN", help="Marked down trading pairs are automatically blacklisted for the future trades."),
 ):
     """Close a single positions.
 
@@ -192,6 +194,8 @@ def break_sync(x):
         valuation_model=valuation_model,
         execution_context=execution_context,
         position_id=position_id,
+        close_by_sell=close_by_sell,
+        blacklist_marked_down=blacklist_marked_down,
     )
 
     # Store the test trade data in the strategy history

tradeexecutor/state/position.py

@@ -13,6 +13,7 @@
 import numpy as np
 import pandas as pd
 from dataclasses_json import dataclass_json
+from jedi.inference.gradual.typing import TypedDict
 
 from tradeexecutor.state.balance_update import BalanceUpdate, BalanceUpdateCause
 from tradeexecutor.state.generic_position import GenericPosition, BalanceUpdateEventAlreadyAdded
@@ -40,6 +41,12 @@
 CLOSED_POSITION_DUST_EPSILON = 0.0001
 
 
+class PositionOtherData(TypedDict):
+    """Position data that is not relevant for most positions."""
+
+    marked_down_at: datetime.datetime
+
+
 @dataclass_json
 @dataclass(slots=True, frozen=True)
 class TriggerPriceUpdate:
@@ -282,6 +289,9 @@ class TradingPosition(GenericPosition):
     #: 
     liquidation_price: USDollarAmount | None = None
 
+    #: Misc bag of data, not often needed
+    other_data: PositionOtherData = field(default_factory=dict)
+
     def __repr__(self):
         if self.is_pending():
             return f"<Pending position #{self.position_id} {self.pair} ${self.get_value()}>"
@@ -387,6 +397,13 @@ def is_repaired(self) -> bool:
         """
         return any(t.is_repaired() for t in self.trades.values())
 
+    def is_marked_down(self) -> bool:
+        """Position value was forcefully set to zero.
+
+        See :py:meth:`mark_down`.
+        """
+        return self.other_data["marked_down_at"] is not None
+
     def has_automatic_close(self) -> bool:
         """This position has stop loss/take profit set."""
         return (self.stop_loss is not None) or (self.take_profit is not None)
@@ -2059,4 +2076,16 @@ def get_annualised_credit_interest(self) -> Percent:
         duration = self.get_duration()
         if duration is None:
             return 0.0
-        return (self.get_claimed_interest() / self.get_value_at_open()) * datetime.timedelta(days=365) / duration
+        return (self.get_claimed_interest() / self.get_value_at_open()) * datetime.timedelta(days=365) / duration
+
+    def mark_down(self):
+        """Manually set position value to zero.
+
+        - Must be done to unsellable assets like scam coins
+        """
+        self.other_data["marked_down_at"] = datetime.datetime.utcnow()
+        self.add_notes_message(f"Marked down to zero manually, last price was {self.last_token_price}, last value was: {self.get_value()}")
+        self.last_token_price = 0
+        self.last_pricing_at = datetime.datetime.utcnow()
+        self.closed_at = datetime.datetime.utcnow()
+        self.unfrozen_at = datetime.datetime.utcnow()

tradeexecutor/state/state.py

@@ -15,6 +15,7 @@
 import pandas as pd
 from dataclasses_json import dataclass_json
 from dataclasses_json.core import _ExtendedEncoder
+from qstrader.asset.asset import Asset
 
 from .other_data import OtherData
 from .sync import Sync
@@ -158,14 +159,18 @@ class State:
     #: Portfolio and position performance records over time.
     stats: Statistics = field(default_factory=Statistics)
 
-    #: Assets that the strategy is not allowed to touch,
-    #: or have failed to trade in the past, resulting to a frozen position.
-    #: Besides this internal black list, the executor can have other blacklists
-    #: based on the trading universe and these are not part of the state.
-    #: The main motivation of this list is to avoid assets that caused a freeze in the future.
-    #: Key is Ethereum address, lowercased.
+    #: Legacy: Do not use. Use function accessor to add and read.
+    #:
+    #: List of asset identifiers that are blacklisted.
+    #:
+    #:
+    #: See :py:meth:`blacklist_asset`
+    #:
     asset_blacklist: Set[str] = field(default_factory=set)
 
+    #: Maintain set of blacklisted asset identifiers
+    blacklisted_assets: Set[AssetIdentifier] = field(default_factory=set)
+
     #: Strategy visualisation and debug messages
     #: to show how the strategy is thinking.
     visualisation: Visualisation = field(default_factory=Visualisation)
@@ -197,7 +202,13 @@ def is_empty(self) -> bool:
     def is_good_pair(self, pair: TradingPairIdentifier) -> bool:
         """Check if the trading pair is blacklisted."""
         assert isinstance(pair, TradingPairIdentifier), f"Expected TradingPairIdentifier, got {type(pair)}: {pair}"
-        return (pair.base.get_identifier() not in self.asset_blacklist) and (pair.quote.get_identifier() not in self.asset_blacklist)
+
+        if pair.base.address in self.asset_blacklist:
+            # Legacy state compatiblity.
+            # Remove in the future.
+            return False
+
+        return (pair.base not in self.blacklisted_assets) and (pair.quote not in self.blacklisted_assets)
 
     def mark_ready(self, timestamp: datetime.datetime | pd.Timestamp):
         """Mark that the strategy has enough (backtest) data to decide the first trade.
@@ -561,22 +572,22 @@ def trade_short(
         )
 
     def supply_credit(
-            self,
-            strategy_cycle_at: datetime.datetime,
-            pair: TradingPairIdentifier,
-            trade_type: TradeType,
-            reserve_currency: AssetIdentifier,
-            collateral_asset_price: USDollarPrice,
-            collateral_quantity: Optional[Decimal] = None,
-            notes: Optional[str] = None,
-            pair_fee: Optional[float] = None,
-            lp_fees_estimated: Optional[USDollarAmount] = None,
-            planned_mid_price: Optional[USDollarPrice] = None,
-            price_structure: Optional[TradePricing] = None,
-            position: Optional[TradingPosition] = None,
-            slippage_tolerance: Optional[float] = None,
-            closing: Optional[bool] = False,
-            flags: Optional[Set[TradeFlag]] = None,
+        self,
+        strategy_cycle_at: datetime.datetime,
+        pair: TradingPairIdentifier,
+        trade_type: TradeType,
+        reserve_currency: AssetIdentifier,
+        collateral_asset_price: USDollarPrice,
+        collateral_quantity: Optional[Decimal] = None,
+        notes: Optional[str] = None,
+        pair_fee: Optional[float] = None,
+        lp_fees_estimated: Optional[USDollarAmount] = None,
+        planned_mid_price: Optional[USDollarPrice] = None,
+        price_structure: Optional[TradePricing] = None,
+        position: Optional[TradingPosition] = None,
+        slippage_tolerance: Optional[float] = None,
+        closing: Optional[bool] = False,
+        flags: Optional[Set[TradeFlag]] = None,
     ) -> Tuple[TradingPosition, TradeExecution, bool]:
         """Create or adjust credit supply position.
 
@@ -898,8 +909,13 @@ def revalue_positions(self, ts: datetime.datetime, valuation_method: Callable):
         raise RuntimeError(f"Removed. Use valuation.revalue_state()")
 
     def blacklist_asset(self, asset: AssetIdentifier):
-        """Add a asset to the blacklist."""
-        self.asset_blacklist.add(asset.get_identifier())
+        """Add a asset to the blacklist.
+
+        See :py:meth:`is_good_pair`.
+        """
+        logger.info("Blacklisted: %s", asset)
+        self.asset_blacklist.add(asset.get_identifier())  # Legacy compatibility
+        self.blacklisted_assets.add(asset)
 
     def perform_integrity_check(self):
         """Check that we are not reusing any trade or position ids and counters are correct.