Classifier update

.gitignore

@@ -126,4 +126,8 @@ venv.bak/
 
 /sandbox
 .env*
+env
 sandbox.ipynb
+Session.vim
+indexer/indexer/core/t_est.py
+some.txt

indexer/event_classifier.py

@@ -5,23 +5,21 @@
 import sys
 import time
 import traceback
-
-from typing import Optional
 from datetime import timedelta
+from typing import Optional
 
 from sqlalchemy import update, select, and_, or_
-from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import sessionmaker, contains_eager
 
 from indexer.core import redis
-from indexer.core.database import engine, Trace, Transaction, Message, Action, TraceEdge, SyncSessionMaker
+from indexer.core.database import engine, Trace, Transaction, Message, Action, SyncSessionMaker
 from indexer.core.settings import Settings
 from indexer.events import context
 from indexer.events.blocks.utils.address_selectors import extract_additional_addresses
 from indexer.events.blocks.utils.block_tree_serializer import block_to_action
 from indexer.events.blocks.utils.event_deserializer import deserialize_event
-from indexer.events.event_processing import process_event_async
+from indexer.events.event_processing import process_event_async, process_event_async_with_postprocessing
 from indexer.events.interface_repository import EmulatedTransactionsInterfaceRepository, gather_interfaces, \
     RedisInterfaceRepository
 
@@ -284,6 +282,9 @@ async def process_trace_batch_async(ids: list[str]):
                     # logger.error(f"query: {insert(Action).values(actions).on_conflict_do_nothing()}")
                     # session.execute(insert(Action).values(actions).on_conflict_do_nothing()) 
                     session.add_all(actions)
+                    for action in actions:
+                        session.add_all(action.get_action_accounts())
+
                     if state == 'ok':
                         ok_traces.append(trace_id)
                     else:
@@ -308,16 +309,20 @@ async def process_trace(trace: Trace) -> tuple[str, str, list[Action]]:
     if len(trace.transactions) == 1 and trace.transactions[0].descr == 'tick_tock':
         return trace.trace_id, 'ok', []
     try:
-        result = await process_event_async(trace)
+        result = await process_event_async_with_postprocessing(trace)
         actions = []
         state = 'ok'
-        for block in result.bfs_iter():
+        for block in result:
             if block.btype != 'root':
                 if block.btype == 'call_contract' and block.event_nodes[0].message.destination is None:
                     continue
+                if block.btype == 'empty':
+                    continue
+                if block.btype == 'call_contract' and block.event_nodes[0].message.source is None:
+                    continue
                 if block.broken:
                     state = 'broken'
-                action = block_to_action(block, trace.trace_id)
+                action = block_to_action(block, trace.trace_id, trace)
                 actions.append(action)
         return trace.trace_id, state, actions
     except Exception as e:

indexer/indexer/core/database.py

@@ -45,6 +45,7 @@ def get_engine(settings: Settings):
                                  pool_timeout=128,
                                  echo=False)
     return engine
+
 engine = get_engine(settings)
 SessionMaker = sessionmaker(bind=engine, class_=AsyncSession)
 
@@ -185,6 +186,15 @@ class TraceEdge(Base):
 
     trace: "Trace" = relationship("Trace", back_populates="edges", viewonly=True)
 
+class ActionAccount(Base):
+    __tablename__ = 'action_accounts'
+    action_id: str = Column(String, primary_key=True)
+    trace_id: str = Column(String, primary_key=True)
+    account: str = Column(String(70), primary_key=True)
+    trace_end_lt: int = Column(Numeric)
+    action_end_lt: int = Column(Numeric)
+    trace_end_utime: int = Column(Numeric)
+    action_end_utime: int = Column(Numeric)
 
 class Action(Base):
     __tablename__ = 'actions'
@@ -265,6 +275,40 @@ class Action(Base):
     ]))
     nft_mint_data = Column(CompositeType("nft_mint_details", [
         Column("nft_item_index", Numeric)]))
+    dex_deposit_liquidity_data = Column(CompositeType("dex_deposit_liquidity_details", [
+        Column("dex", String),
+        Column("amount1", Numeric),
+        Column("amount2", Numeric),
+        Column("asset1", String),
+        Column("asset2", String),
+        Column('user_jetton_wallet_1', String),
+        Column('user_jetton_wallet_2', String),
+        Column("lp_tokens_minted", Numeric),
+    ]))
+    dex_withdraw_liquidity_data = Column(CompositeType("dex_withdraw_liquidity_details", [
+        Column("dex", String),
+        Column("amount1", Numeric),
+        Column("amount2", Numeric),
+        Column('asset1_out', String),
+        Column('asset2_out', String),
+        Column('user_jetton_wallet_1', String),
+        Column('user_jetton_wallet_2', String),
+        Column('dex_jetton_wallet_1', String),
+        Column('dex_jetton_wallet_2', String),
+        Column("lp_tokens_burnt", Numeric),
+    ]))
+    staking_data = Column(CompositeType("staking_details", [
+        Column("provider", String),
+        Column("ts_nft", String),
+    ]))
+    trace_end_lt: int = Column(Numeric)
+    trace_end_utime: int = Column(Numeric)
+    trace_external_hash: str = Column(String)
+    mc_seqno_end: int = Column(Numeric)
+    trace_mc_seqno_end: int = Column(Numeric)
+    accounts: list[str]
+    value_extra_currencies: dict = Column(JSONB)
+
 
     def __repr__(self):
         full_repr = ""
@@ -274,6 +318,18 @@ def __repr__(self):
             full_repr += f"{key}={value}, "
         return full_repr
 
+    def get_action_accounts(self):
+        accounts = []
+        for account in self.accounts:
+            accounts.append(ActionAccount(action_id=self.action_id,
+                                          trace_id=self.trace_id,
+                                          account=account,
+                                          action_end_lt=self.end_lt,
+                                          trace_end_lt=self.trace_end_lt,
+                                          trace_end_utime=self.trace_end_utime,
+                                          action_end_utime=self.end_utime))
+        return accounts
+
 class Transaction(Base):
     __tablename__ = 'transactions'
     __table_args__ = (
@@ -409,6 +465,7 @@ class Message(Base):
     import_fee: int = Column(BigInteger)
     body_hash: str = Column(String(44))
     init_state_hash: Optional[str] = Column(String(44), nullable=True)
+    value_extra_currencies: dict = Column(JSONB, nullable=True)
 
     transaction = relationship("Transaction",
                                viewonly=True,

indexer/indexer/events/blocks/auction.py

@@ -35,18 +35,29 @@ async def build_block(self, block: Block, other_blocks: list[Block]) -> list[Blo
         bid_block = AuctionBid({})
 
         if 'NftAuction' in interfaces:
-            bid_block.data = {
+            nft_address = interfaces['NftAuction']['nft_addr']
+            nft_item = await context.interface_repository.get().get_nft_item(nft_address)
+            data = {
                 'amount': Amount(block.event_nodes[0].message.value),
                 'bidder': AccountId(block.event_nodes[0].message.source),
                 'auction': AccountId(block.event_nodes[0].message.destination),
-                'nft_address': AccountId(interfaces['NftAuction']['nft_addr']),
+                'nft_address': AccountId(nft_address),
+                'nft_item_index': None,
+                'nft_collection': None
             }
+            if nft_item:
+                data['nft_item_index'] = nft_item.index
+                data['nft_collection'] = AccountId(nft_item.collection_address)
+            bid_block.data = data
         elif 'NFTItem' in interfaces and _is_teleitem(interfaces['NFTItem']):
+            nft_data = interfaces['NFTItem']
             bid_block.data = {
                 'amount': Amount(block.event_nodes[0].message.value),
                 'bidder': AccountId(block.event_nodes[0].message.source),
                 'auction': AccountId(block.event_nodes[0].message.destination),
                 'nft_address': AccountId(block.event_nodes[0].message.destination),
+                'nft_collection': AccountId(nft_data['collection_address']) if nft_data['collection_address'] is not None else None,
+                'nft_item_index': nft_data['index'] if nft_data['index'] is not None else None
             }
         else:
             return []

indexer/indexer/events/blocks/basic_blocks.py

@@ -48,30 +48,34 @@ def __init__(self, node: EventNode):
             self.comment = None
 
         super().__init__('ton_transfer', [node], {
+            'extra_currencies': node.message.value_extra_currencies,
             'source': AccountId(node.message.source) if node.message.source is not None else None,
             'destination': AccountId(
                 node.message.destination) if node.message.destination is not None else None,
             'value': Amount(node.message.value),
             'comment': self.comment,
             'encrypted': self.encrypted,
         })
-        self.failed = node.failed
+        if node.failed:
+            if node.message is not None and node.message.bounce == True:
+                self.failed = True
+            elif node.get_tx() is not None and node.get_tx().end_status == 'uninit':
+                self.failed = False
+            else:
+                self.failed = True
         self.value = node.message.value
 
         _fill_flow_from_node(self.value_flow, node)
-
-
-class ContractDeploy(Block):
-    def __init__(self, node: EventNode):
-        super().__init__('contract_deploy', [], node.message.transaction.account)
-        self.failed = node.failed
-
+        tx = node.get_tx()
+        if tx is not None and tx.end_status == 'active' and tx.orig_status not in ('active', 'frozen'):
+            self.children_blocks.append(ContractDeploy(node))
 
 class CallContractBlock(Block):
     opcode: int
 
     def __init__(self, node: EventNode):
         super().__init__('call_contract', [node], {
+            'extra_currencies': node.message.value_extra_currencies,
             'opcode': node.get_opcode(),
             'source': AccountId(node.message.source) if node.message.source is not None else None,
             'destination': AccountId(
@@ -82,6 +86,9 @@ def __init__(self, node: EventNode):
         self.is_external = node.message.source is None
         self.opcode = node.get_opcode()
         _fill_flow_from_node(self.value_flow, node)
+        tx = node.get_tx()
+        if tx is not None and tx.end_status == 'active' and tx.orig_status not in ('active', 'frozen'):
+            self.children_blocks.append(ContractDeploy(node))
 
     def get_body(self) -> Slice:
         return Slice.one_from_boc(self.event_nodes[0].message.message_content.body)
@@ -91,3 +98,16 @@ def get_message(self) -> Message:
 
     def __repr__(self):
         return f"!{self.btype}:={hex(self.opcode)}"
+
+class ContractDeploy(Block):
+    def __init__(self, node: EventNode):
+        super().__init__('contract_deploy', [node], {
+            'opcode': node.get_opcode(),
+            'source': AccountId(node.message.source) if node.message.source is not None else None,
+            'destination': AccountId(
+                node.message.destination) if node.message.destination is not None else None,
+            'value': Amount(node.message.value),
+        })
+        self.failed = node.failed
+        self.is_external = node.message.source is None
+        self.opcode = node.get_opcode()

indexer/indexer/events/blocks/basic_matchers.py

@@ -15,14 +15,18 @@ def __init__(self, child_matcher: BlockMatcher | None = None,
                  parent_matcher: BlockMatcher | None = None,
                  optional=False,
                  children_matchers=None,
-                 include_excess=True):
+                 include_excess=True,
+                 include_bounces=True,
+                 pre_build_auto_append=False):
         self.child_matcher = child_matcher
         self.children_matchers = children_matchers
         self.parent_matcher = parent_matcher
         self.optional = optional
         self.include_excess = include_excess
+        self.include_bounces = include_bounces
+        self.pre_build_auto_append = pre_build_auto_append
 
-    def test_self(self, block: Block):
+    def test_self(self, block: Block) -> bool:
         return True
 
     async def try_build(self, block: Block) -> list[Block] | None:
@@ -36,10 +40,19 @@ async def try_build(self, block: Block) -> list[Block] | None:
         parent_matched = await self.process_parent_matcher(block, blocks, parent_matched)
         if self_matched and parent_matched and child_matched:
             try:
-                r = await self.build_block(block, blocks)
+                auto_append_opcodes = []
                 if self.include_excess:
+                    auto_append_opcodes.append(ExcessMessage.opcode)
+                if self.include_bounces:
+                    auto_append_opcodes.append(0xffffffff)
+                if len(auto_append_opcodes) > 0 and self.pre_build_auto_append:
+                    for next_block in block.next_blocks:
+                        if isinstance(next_block, CallContractBlock) and next_block.opcode in auto_append_opcodes:
+                            blocks.append(next_block)
+                r = await self.build_block(block, blocks)
+                if len(auto_append_opcodes) > 0 and not self.pre_build_auto_append:
                     for next_block in block.next_blocks:
-                        if isinstance(next_block, CallContractBlock) and next_block.opcode == ExcessMessage.opcode:
+                        if isinstance(next_block, CallContractBlock) and next_block.opcode in auto_append_opcodes:
                             r.append(next_block)
                 return r
             except Exception as e:
@@ -153,6 +166,13 @@ def __init__(self, block_type, child_matcher=None, parent_matcher=None, optional
     def test_self(self, block: Block):
         return block.btype == self.block_type
 
+class GenericMatcher(BlockMatcher):
+    def __init__(self, test_self_func, child_matcher=None, parent_matcher=None, optional=False):
+        super().__init__(child_matcher, parent_matcher, optional)
+        self.test_self_func = test_self_func
+
+    def test_self(self, block: Block):
+        return self.test_self_func(block)
 
 def child_sequence_matcher(matchers: list[BlockMatcher]) -> BlockMatcher | None:
     if len(matchers) == 0: