feat(saya): load data from Katana to run SNOS (#1535)

* chore: bump dependencies * fix: update cairo-lang * fix: add missing run_profiler * fix: fmt * update cairo-vm rev * wip * wip * fmt * fix: ensure semantics analyzer plugin is registered * chore(katana): make ethers non optional since U256 is used outside messaging * fix: run CAIRO_FIX_TESTS * fix(dojo-core): remove compiler warnings * git: add sozo generated bindings to gitignore * fix: fix class hash * fix: class hash generated code * fix(dojo-core): fix tests and compilation warnings * chore: integrate snos on cairo2.5 * chore: update Cargo.lock * wip: work on integration of katana types * wip: fetch strategy * fix * feat: add parsing of state diff for genesis block * wip * wip * fix: rework provider for saya and RPC implem * wip * wip: snos types * wip txs * wip: fix snos parsing need to complete inputs * chore: switch back to git snos * chore: bump blockifier to use Serde on TransactionExecutionInfo * feat: add saya RPC namespace * feat: add transaction execution info to in memory db * fix: remove unused code covered by other test * fix: add logic for small requests to get transactions executions info * fix: fetch transactions executions info instead of running blockifier * fix: fmt and clippy * docs: fix unused type * fix: add tests for new transactions execution retrieval * fix: add correct test to check no pending support * fix: restore unchanged file * fix: rename new provider trait to TransactionTraceProvider * refactor(katana): change stored compiled class (#1559) Both `blockifier` and `starknet_in_rust` use their own proprietary contract class types. So need to have a type where the types from those two crates can be derived from. * refactor(katana-primitives): move class related types to a separate module (#1560) * Katana executor traits (#1561) * feat(katana-executor): add `blockifier` executor implementations (#1562) implement the executor traits introduced in #1561 based on `blockifier` * Refactor Katana to using the new executor traits (#1563) integrate the new executor traits introduced in #1561 to katana * feat(katana-executor): add starknet_in_rust executor (#1564) * doc(katana-executor): add README * test(katana-executor): add test for transaction simulation (#1593) * test(katana-executor): refactor simulate tests (#1606) refactor tests * adjust new types for Katana blockchain and SNOS * fix: clippy * refactor(katana): change stored compiled class (#1559) Both `blockifier` and `starknet_in_rust` use their own proprietary contract class types. So need to have a type where the types from those two crates can be derived from. * refactor(katana-primitives): move class related types to a separate module (#1560) * Katana executor traits (#1561) * feat(katana-executor): add `blockifier` executor implementations (#1562) implement the executor traits introduced in #1561 based on `blockifier` * Refactor Katana to using the new executor traits (#1563) integrate the new executor traits introduced in #1561 to katana * feat(katana-executor): add starknet_in_rust executor (#1564) * doc(katana-executor): add README * test(katana-executor): add test for transaction simulation (#1593) * test(katana-executor): refactor simulate tests (#1606) refactor tests * feat: add new primitive for trace and blockifier conversions * feat: add TransactionExecutionInfo convert from blockifier * feat: add executor trait method to retrieve the execution info * fix: ensure tx execution info is passed in block producer * chore: restore dependency on blockifier as we use TxExecInfo primitive * refactor(katana): decouple `katana-primitives` from any executor impls (#1637) * fix: clippy / fmt * merge * fix: clippy --------- Co-authored-by: Kariy <[email protected]> Co-authored-by: Ammar Arif <[email protected]>
dojoengine · Mar 21, 2024 · e45509b · e45509b
1 parent 8e79558
commit e45509b
Show file tree

Hide file tree

Showing 66 changed files with 1,918 additions and 134 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -32,6 +32,7 @@ members = [
   "crates/katana/tasks",
   "crates/metrics",
   "crates/saya/core",
+  "crates/saya/provider",
   "crates/sozo/signers",
   "crates/torii/client",
   "crates/torii/server",
@@ -91,6 +92,7 @@ torii-server = { path = "crates/torii/server" }
 
 # saya
 saya-core = { path = "crates/saya/core" }
+saya-provider = { path = "crates/saya/provider" }
 
 # sozo
 sozo-signers = { path = "crates/sozo/signers" }

diff --git a/bin/katana/src/args.rs b/bin/katana/src/args.rs
@@ -218,7 +218,7 @@ impl KatanaArgs {
     }
 
     pub fn server_config(&self) -> ServerConfig {
-        let mut apis = vec![ApiKind::Starknet, ApiKind::Katana, ApiKind::Torii];
+        let mut apis = vec![ApiKind::Starknet, ApiKind::Katana, ApiKind::Torii, ApiKind::Saya];
         // only enable `katana` API in dev mode
         if self.dev {
             apis.push(ApiKind::Dev);

diff --git a/bin/saya/src/args/mod.rs b/bin/saya/src/args/mod.rs
@@ -49,7 +49,7 @@ pub struct SayaArgs {
 
 impl SayaArgs {
     pub fn init_logging(&self) -> Result<(), Box<dyn std::error::Error>> {
-        const DEFAULT_LOG_FILTER: &str = "info,saya_core=trace";
+        const DEFAULT_LOG_FILTER: &str = "info,saya_core=trace,blockchain=trace,provider=trace";
 
         let builder = fmt::Subscriber::builder().with_env_filter(
             EnvFilter::try_from_default_env().or(EnvFilter::try_new(DEFAULT_LOG_FILTER))?,

diff --git a/bin/saya/src/main.rs b/bin/saya/src/main.rs
@@ -16,7 +16,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let config = args.try_into()?;
     print_intro(&config);
 
-    let saya = Saya::new(config).await?;
+    let mut saya = Saya::new(config).await?;
     saya.start().await?;
 
     // Wait until Ctrl + C is pressed, then shutdown

diff --git a/crates/dojo-test-utils/src/sequencer.rs b/crates/dojo-test-utils/src/sequencer.rs
@@ -68,7 +68,13 @@ impl TestSequencer {
                 port: 0,
                 host: "127.0.0.1".into(),
                 max_connections: 100,
-                apis: vec![ApiKind::Starknet, ApiKind::Katana, ApiKind::Torii, ApiKind::Dev],
+                apis: vec![
+                    ApiKind::Starknet,
+                    ApiKind::Katana,
+                    ApiKind::Dev,
+                    ApiKind::Saya,
+                    ApiKind::Torii,
+                ],
             },
         )
         .await

diff --git a/crates/katana/core/src/backend/mod.rs b/crates/katana/core/src/backend/mod.rs
@@ -6,9 +6,7 @@ use katana_primitives::block::{
 };
 use katana_primitives::chain::ChainId;
 use katana_primitives::env::BlockEnv;
-use katana_primitives::receipt::Receipt;
 use katana_primitives::state::StateUpdatesWithDeclaredClasses;
-use katana_primitives::transaction::TxWithHash;
 use katana_primitives::version::CURRENT_STARKNET_VERSION;
 use katana_primitives::FieldElement;
 use katana_provider::providers::fork::ForkedProvider;
@@ -28,7 +26,7 @@ pub mod storage;
 use self::config::StarknetConfig;
 use self::storage::Blockchain;
 use crate::env::BlockContextGenerator;
-use crate::service::block_producer::{BlockProductionError, MinedBlockOutcome};
+use crate::service::block_producer::{BlockProductionError, MinedBlockOutcome, TxWithOutcome};
 use crate::utils::get_current_timestamp;
 
 pub struct Backend<EF: ExecutorFactory> {
@@ -120,10 +118,18 @@ impl<EF: ExecutorFactory> Backend<EF> {
     pub fn do_mine_block(
         &self,
         block_env: &BlockEnv,
-        tx_receipt_pairs: Vec<(TxWithHash, Receipt)>,
+        txs_outcomes: Vec<TxWithOutcome>,
         state_updates: StateUpdatesWithDeclaredClasses,
     ) -> Result<MinedBlockOutcome, BlockProductionError> {
-        let (txs, receipts): (Vec<TxWithHash>, Vec<Receipt>) = tx_receipt_pairs.into_iter().unzip();
+        let mut txs = vec![];
+        let mut receipts = vec![];
+        let mut execs = vec![];
+
+        for t in txs_outcomes {
+            txs.push(t.tx);
+            receipts.push(t.receipt);
+            execs.push(t.exec_info);
+        }
 
         let prev_hash = BlockHashProvider::latest_hash(self.blockchain.provider())?;
         let block_number = block_env.number;
@@ -150,6 +156,7 @@ impl<EF: ExecutorFactory> Backend<EF> {
             block,
             state_updates,
             receipts,
+            execs,
         )?;
 
         info!(target: "backend", "⛏️ Block {block_number} mined with {tx_count} transactions");

diff --git a/crates/katana/core/src/backend/storage.rs b/crates/katana/core/src/backend/storage.rs
@@ -12,7 +12,8 @@ use katana_provider::traits::env::BlockEnvProvider;
 use katana_provider::traits::state::{StateFactoryProvider, StateRootProvider, StateWriter};
 use katana_provider::traits::state_update::StateUpdateProvider;
 use katana_provider::traits::transaction::{
-    ReceiptProvider, TransactionProvider, TransactionStatusProvider, TransactionsProviderExt,
+    ReceiptProvider, TransactionProvider, TransactionStatusProvider, TransactionTraceProvider,
+    TransactionsProviderExt,
 };
 use katana_provider::BlockchainProvider;
 
@@ -21,6 +22,7 @@ pub trait Database:
     + BlockWriter
     + TransactionProvider
     + TransactionStatusProvider
+    + TransactionTraceProvider
     + TransactionsProviderExt
     + ReceiptProvider
     + StateUpdateProvider
@@ -40,6 +42,7 @@ impl<T> Database for T where
         + BlockWriter
         + TransactionProvider
         + TransactionStatusProvider
+        + TransactionTraceProvider
         + TransactionsProviderExt
         + ReceiptProvider
         + StateUpdateProvider
@@ -119,7 +122,13 @@ impl Blockchain {
         block: SealedBlockWithStatus,
         states: StateUpdatesWithDeclaredClasses,
     ) -> Result<Self> {
-        BlockWriter::insert_block_with_states_and_receipts(&provider, block, states, vec![])?;
+        BlockWriter::insert_block_with_states_and_receipts(
+            &provider,
+            block,
+            states,
+            vec![],
+            vec![],
+        )?;
         Ok(Self::new(provider))
     }
 }
@@ -245,6 +254,7 @@ mod tests {
                     dummy_block.clone(),
                     StateUpdatesWithDeclaredClasses::default(),
                     vec![Receipt::Invoke(InvokeTxReceipt::default())],
+                    vec![],
                 )
                 .unwrap();
 

diff --git a/crates/katana/core/src/service/block_producer.rs b/crates/katana/core/src/service/block_producer.rs
@@ -11,6 +11,7 @@ use futures::FutureExt;
 use katana_executor::{BlockExecutor, ExecutionOutput, ExecutorFactory};
 use katana_primitives::block::{BlockHashOrNumber, ExecutableBlock, PartialHeader};
 use katana_primitives::receipt::Receipt;
+use katana_primitives::trace::TxExecInfo;
 use katana_primitives::transaction::{ExecutableTxWithHash, TxWithHash};
 use katana_primitives::version::CURRENT_STARKNET_VERSION;
 use katana_provider::error::ProviderError;
@@ -43,17 +44,23 @@ pub struct MinedBlockOutcome {
     pub block_number: u64,
 }
 
+#[derive(Debug, Clone)]
+pub struct TxWithOutcome {
+    pub tx: TxWithHash,
+    pub receipt: Receipt,
+    pub exec_info: TxExecInfo,
+}
+
 type ServiceFuture<T> = Pin<Box<dyn Future<Output = BlockingTaskResult<T>> + Send + Sync>>;
 
 type BlockProductionResult = Result<MinedBlockOutcome, BlockProductionError>;
 type BlockProductionFuture = ServiceFuture<BlockProductionResult>;
 
-type TxExecutionResult = Result<TxWithHashAndReceiptPairs, BlockProductionError>;
+type TxExecutionResult = Result<Vec<TxWithOutcome>, BlockProductionError>;
 type TxExecutionFuture = ServiceFuture<TxExecutionResult>;
 
 type BlockProductionWithTxnsFuture =
-    ServiceFuture<Result<(MinedBlockOutcome, TxWithHashAndReceiptPairs), BlockProductionError>>;
-pub type TxWithHashAndReceiptPairs = Vec<(TxWithHash, Receipt)>;
+    ServiceFuture<Result<(MinedBlockOutcome, Vec<TxWithOutcome>), BlockProductionError>>;
 
 /// The type which responsible for block production.
 #[must_use = "BlockProducer does nothing unless polled"]
@@ -164,7 +171,7 @@ pub struct IntervalBlockProducer<EF: ExecutorFactory> {
     blocking_task_spawner: BlockingTaskPool,
     ongoing_execution: Option<TxExecutionFuture>,
     /// Listeners notified when a new executed tx is added.
-    tx_execution_listeners: RwLock<Vec<Sender<TxWithHashAndReceiptPairs>>>,
+    tx_execution_listeners: RwLock<Vec<Sender<Vec<TxWithOutcome>>>>,
 }
 
 impl<EF: ExecutorFactory> IntervalBlockProducer<EF> {
@@ -259,7 +266,9 @@ impl<EF: ExecutorFactory> IntervalBlockProducer<EF> {
 
         let transactions = transactions
             .into_iter()
-            .filter_map(|(tx, rct)| rct.map(|rct| (tx, rct)))
+            .filter_map(|(tx, rct, exec)| {
+                rct.map(|rct| TxWithOutcome { tx, receipt: rct, exec_info: exec })
+            })
             .collect::<Vec<_>>();
 
         let outcome = backend.do_mine_block(&block_env, transactions, states)?;
@@ -272,19 +281,20 @@ impl<EF: ExecutorFactory> IntervalBlockProducer<EF> {
     fn execute_transactions(
         executor: PendingExecutor,
         transactions: Vec<ExecutableTxWithHash>,
-    ) -> Result<TxWithHashAndReceiptPairs, BlockProductionError> {
-        let tx_receipt_pair = transactions
+    ) -> Result<Vec<TxWithOutcome>, BlockProductionError> {
+        let tx_outcome = transactions
             .into_iter()
             .map(|tx| {
                 let tx_ = TxWithHash::from(&tx);
                 let output = executor.write().execute(tx)?;
-
                 let receipt = output.receipt(tx_.as_ref());
-                Ok((tx_, receipt))
+                let exec_info = output.execution_info();
+
+                Ok(TxWithOutcome { tx: tx_, receipt, exec_info })
             })
-            .collect::<Result<TxWithHashAndReceiptPairs, BlockProductionError>>()?;
+            .collect::<Result<Vec<TxWithOutcome>, BlockProductionError>>()?;
 
-        Ok(tx_receipt_pair)
+        Ok(tx_outcome)
     }
 
     fn create_new_executor_for_next_block(&self) -> Result<PendingExecutor, BlockProductionError> {
@@ -301,15 +311,15 @@ impl<EF: ExecutorFactory> IntervalBlockProducer<EF> {
         Ok(PendingExecutor::new(executor))
     }
 
-    pub fn add_listener(&self) -> Receiver<TxWithHashAndReceiptPairs> {
+    pub fn add_listener(&self) -> Receiver<Vec<TxWithOutcome>> {
         const TX_LISTENER_BUFFER_SIZE: usize = 2048;
         let (tx, rx) = channel(TX_LISTENER_BUFFER_SIZE);
         self.tx_execution_listeners.write().push(tx);
         rx
     }
 
     /// notifies all listeners about the transaction
-    fn notify_listener(&self, txs: TxWithHashAndReceiptPairs) {
+    fn notify_listener(&self, txs: Vec<TxWithOutcome>) {
         let mut listener = self.tx_execution_listeners.write();
         // this is basically a retain but with mut reference
         for n in (0..listener.len()).rev() {
@@ -436,7 +446,7 @@ pub struct InstantBlockProducer<EF: ExecutorFactory> {
 
     blocking_task_pool: BlockingTaskPool,
     /// Listeners notified when a new executed tx is added.
-    tx_execution_listeners: RwLock<Vec<Sender<TxWithHashAndReceiptPairs>>>,
+    tx_execution_listeners: RwLock<Vec<Sender<Vec<TxWithOutcome>>>>,
 }
 
 impl<EF: ExecutorFactory> InstantBlockProducer<EF> {
@@ -462,7 +472,7 @@ impl<EF: ExecutorFactory> InstantBlockProducer<EF> {
     fn do_mine(
         backend: Arc<Backend<EF>>,
         transactions: Vec<ExecutableTxWithHash>,
-    ) -> Result<(MinedBlockOutcome, TxWithHashAndReceiptPairs), BlockProductionError> {
+    ) -> Result<(MinedBlockOutcome, Vec<TxWithOutcome>), BlockProductionError> {
         trace!(target: "miner", "creating new block");
 
         let provider = backend.blockchain.provider();
@@ -491,27 +501,29 @@ impl<EF: ExecutorFactory> InstantBlockProducer<EF> {
         executor.execute_block(block)?;
 
         let ExecutionOutput { states, transactions } = executor.take_execution_output().unwrap();
-        let tx_receipt_pairs = transactions
+        let txs_outcomes = transactions
             .into_iter()
-            .filter_map(|(tx, rct)| rct.map(|rct| (tx, rct)))
+            .filter_map(|(tx, rct, exec)| {
+                rct.map(|rct| TxWithOutcome { tx, receipt: rct, exec_info: exec })
+            })
             .collect::<Vec<_>>();
 
-        let outcome = backend.do_mine_block(&block_env, tx_receipt_pairs.clone(), states)?;
+        let outcome = backend.do_mine_block(&block_env, txs_outcomes.clone(), states)?;
 
         trace!(target: "miner", "created new block: {}", outcome.block_number);
 
-        Ok((outcome, tx_receipt_pairs))
+        Ok((outcome, txs_outcomes))
     }
 
-    pub fn add_listener(&self) -> Receiver<TxWithHashAndReceiptPairs> {
+    pub fn add_listener(&self) -> Receiver<Vec<TxWithOutcome>> {
         const TX_LISTENER_BUFFER_SIZE: usize = 2048;
         let (tx, rx) = channel(TX_LISTENER_BUFFER_SIZE);
         self.tx_execution_listeners.write().push(tx);
         rx
     }
 
     /// notifies all listeners about the transaction
-    fn notify_listener(&self, txs: TxWithHashAndReceiptPairs) {
+    fn notify_listener(&self, txs: Vec<TxWithOutcome>) {
         let mut listener = self.tx_execution_listeners.write();
         // this is basically a retain but with mut reference
         for n in (0..listener.len()).rev() {

diff --git a/crates/katana/executor/src/abstraction.rs b/crates/katana/executor/src/abstraction.rs
@@ -4,6 +4,7 @@ use katana_primitives::contract::{ContractAddress, Nonce, StorageKey, StorageVal
 use katana_primitives::env::{BlockEnv, CfgEnv};
 use katana_primitives::receipt::Receipt;
 use katana_primitives::state::StateUpdatesWithDeclaredClasses;
+use katana_primitives::trace::TxExecInfo;
 use katana_primitives::transaction::{ExecutableTxWithHash, Tx, TxWithHash};
 use katana_primitives::FieldElement;
 use katana_provider::traits::contract::ContractClassProvider;
@@ -107,7 +108,8 @@ pub struct ExecutionOutput {
     /// The state updates produced by the executions.
     pub states: StateUpdatesWithDeclaredClasses,
     /// The transactions that have been executed.
-    pub transactions: Vec<(TxWithHash, Option<Receipt>)>,
+    /// TODO: do we want a struct instead?
+    pub transactions: Vec<(TxWithHash, Option<Receipt>, TxExecInfo)>,
 }
 
 #[derive(Debug)]
@@ -152,7 +154,7 @@ pub trait BlockExecutor<'a>: TransactionExecutor + Send + Sync {
     fn state(&self) -> Box<dyn StateProvider + 'a>;
 
     /// Returns the transactions that have been executed.
-    fn transactions(&self) -> &[(TxWithHash, Option<Receipt>)];
+    fn transactions(&self) -> &[(TxWithHash, Option<Receipt>, TxExecInfo)];
 
     /// Returns the current block environment of the executor.
     fn block_env(&self) -> BlockEnv;
@@ -192,6 +194,9 @@ pub trait TransactionExecutionOutput {
 
     /// The error message if the transaction execution reverted, otherwise the value is `None`.
     fn revert_error(&self) -> Option<&str>;
+
+    /// Retrieves the execution info of the transaction.
+    fn execution_info(&self) -> TxExecInfo;
 }
 
 /// A wrapper around a boxed [StateProvider] for implementing the executor's own state reader