Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(katana): added chunk size limit to events method #2644

Merged
merged 25 commits into from
Nov 18, 2024
Merged
Show file tree
Hide file tree
Changes from 22 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
0c70748
added chunk size limit to events method
PoulavBhowmick03 Nov 6, 2024
32597b7
Merge branch 'dojoengine:main' into feat/katana-chunk_size_limit
PoulavBhowmick03 Nov 8, 2024
4abd447
Merge branch 'dojoengine:main' into feat/katana-chunk_size_limit
PoulavBhowmick03 Nov 12, 2024
1afd693
fixed changes
PoulavBhowmick03 Nov 12, 2024
86150ed
fixed changes
PoulavBhowmick03 Nov 12, 2024
c90845d
fixed changes for the StarknetApiConfig
PoulavBhowmick03 Nov 12, 2024
fd4c2e7
changed page_size to Some(...) instead of None and enhanced rpc_config
PoulavBhowmick03 Nov 12, 2024
ccb0c81
Update bin/katana/src/cli/node.rs
PoulavBhowmick03 Nov 13, 2024
4987d1c
Update crates/katana/rpc/rpc-types/src/error/starknet.rs
PoulavBhowmick03 Nov 13, 2024
3d170b8
Update bin/katana/src/cli/options.rs
PoulavBhowmick03 Nov 13, 2024
86134fc
Update crates/katana/node/src/config/rpc.rs
PoulavBhowmick03 Nov 13, 2024
07f2b79
Update crates/katana/rpc/rpc/src/starknet/mod.rs
PoulavBhowmick03 Nov 13, 2024
66bfe44
Update crates/katana/rpc/rpc/src/starknet/mod.rs
PoulavBhowmick03 Nov 13, 2024
606a544
Update crates/katana/rpc/rpc/src/starknet/mod.rs
PoulavBhowmick03 Nov 13, 2024
6d82fbd
fixed merge conflicts
PoulavBhowmick03 Nov 13, 2024
8772aa0
fixed failing tests
PoulavBhowmick03 Nov 13, 2024
a8288d3
Merge branch 'dojoengine:main' into feat/katana-chunk_size_limit
PoulavBhowmick03 Nov 14, 2024
acccb45
Update crates/katana/node/src/lib.rs
PoulavBhowmick03 Nov 14, 2024
e1103ea
Merge branch 'main' into feat/katana-chunk_size_limit
PoulavBhowmick03 Nov 14, 2024
099be6a
ran the commands for passing the tests
PoulavBhowmick03 Nov 14, 2024
d92ce69
ran the commands for passing the tests
PoulavBhowmick03 Nov 14, 2024
4761298
Merge branch 'dojoengine:main' into feat/katana-chunk_size_limit
PoulavBhowmick03 Nov 15, 2024
f2923de
fixed failing tests
PoulavBhowmick03 Nov 15, 2024
7752466
fix test and bump defualt page size
kariy Nov 16, 2024
56b8cb7
remove redundant default
kariy Nov 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions crates/dojo/test-utils/src/sequencer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ pub fn get_default_test_config(sequencing: SequencingConfig) -> Config {
addr: DEFAULT_RPC_ADDR,
max_connections: DEFAULT_RPC_MAX_CONNECTIONS,
apis: HashSet::from([ApiKind::Starknet, ApiKind::Dev, ApiKind::Saya, ApiKind::Torii]),
max_event_page_size: Some(100),
};

Config { sequencing, rpc, dev, chain, ..Default::default() }
Expand Down
1 change: 1 addition & 0 deletions crates/katana/cli/src/args.rs
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,7 @@ impl NodeArgs {
addr: self.server.http_addr,
max_connections: self.server.max_connections,
cors_origins: self.server.http_cors_origins.clone(),
max_event_page_size: Some(self.server.max_event_page_size),
}
}

Expand Down
12 changes: 12 additions & 0 deletions crates/katana/cli/src/options.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ use crate::utils::{parse_block_hash_or_number, parse_genesis, LogFormat};

const DEFAULT_DEV_SEED: &str = "0";
const DEFAULT_DEV_ACCOUNTS: u16 = 10;
const DEFAULT_RPC_PAGE_SIZE: u64 = 100;

#[cfg(feature = "server")]
#[derive(Debug, Args, Clone, Serialize, Deserialize, PartialEq)]
Expand Down Expand Up @@ -89,6 +90,12 @@ pub struct ServerOptions {
#[arg(default_value_t = DEFAULT_RPC_MAX_CONNECTIONS)]
#[serde(default = "default_max_connections")]
pub max_connections: u32,

/// Maximum page size for event queries.
#[arg(long = "rpc.max-event-page-size", value_name = "SIZE")]
#[arg(default_value_t = DEFAULT_RPC_PAGE_SIZE)]
#[serde(default = "default_page_size")]
pub max_event_page_size: u64,
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Codebase verification

Ohayo, sensei!

max_event_page_size is not enforced in starknet_getEvents

The starknet_getEvents method does not enforce the max_event_page_size limit, which may allow potential DoS attacks.

  • File: crates/katana/rpc/rpc/src/starknet/mod.rs
🔗 Analysis chain

Verify enforcement of max_event_page_size limit

Let's ensure this limit is properly enforced in the starknet_getEvents method.

🏁 Scripts executed

The following scripts were executed for the analysis:

Script:

#!/bin/bash
# Description: Check if max_event_page_size is enforced in starknet_getEvents

# Search for usage of max_event_page_size in the codebase
rg -l "max_event_page_size"

# Look for validation logic in starknet_getEvents
ast-grep --pattern 'fn events($$$) {
  $$$
}'

Length of output: 247

}

#[cfg(feature = "server")]
Expand All @@ -99,6 +106,7 @@ impl Default for ServerOptions {
http_port: DEFAULT_RPC_PORT,
max_connections: DEFAULT_RPC_MAX_CONNECTIONS,
http_cors_origins: None,
max_event_page_size: DEFAULT_RPC_PAGE_SIZE,
}
}
}
Expand Down Expand Up @@ -356,6 +364,10 @@ fn default_max_connections() -> u32 {
DEFAULT_RPC_MAX_CONNECTIONS
}

fn default_page_size() -> u64 {
DEFAULT_RPC_PAGE_SIZE
}

#[cfg(feature = "server")]
fn default_metrics_addr() -> IpAddr {
DEFAULT_METRICS_ADDR
Expand Down
4 changes: 3 additions & 1 deletion crates/katana/node/src/config/rpc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};
pub const DEFAULT_RPC_MAX_CONNECTIONS: u32 = 100;
pub const DEFAULT_RPC_ADDR: IpAddr = IpAddr::V4(Ipv4Addr::LOCALHOST);
pub const DEFAULT_RPC_PORT: u16 = 5050;

pub const DEFAULT_RPC_PAGE_SIZE: u64 = 100;
PoulavBhowmick03 marked this conversation as resolved.
Show resolved Hide resolved
/// List of APIs supported by Katana.
#[derive(
Debug, Copy, Clone, PartialEq, Eq, Hash, strum_macros::EnumString, strum_macros::Display,
Expand All @@ -24,6 +24,7 @@ pub struct RpcConfig {
pub port: u16,
pub max_connections: u32,
pub apis: HashSet<ApiKind>,
pub max_event_page_size: Option<u64>,
pub cors_origins: Option<Vec<String>>,
}

Expand All @@ -42,6 +43,7 @@ impl Default for RpcConfig {
port: DEFAULT_RPC_PORT,
max_connections: DEFAULT_RPC_MAX_CONNECTIONS,
apis: HashSet::from([ApiKind::Starknet]),
max_event_page_size: Some(DEFAULT_RPC_PAGE_SIZE),
}
}
}
13 changes: 11 additions & 2 deletions crates/katana/node/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ use katana_rpc::dev::DevApi;
use katana_rpc::metrics::RpcServerMetrics;
use katana_rpc::saya::SayaApi;
use katana_rpc::starknet::forking::ForkedClient;
use katana_rpc::starknet::StarknetApi;
use katana_rpc::starknet::{StarknetApi, StarknetApiConfig};
use katana_rpc::torii::ToriiApi;
use katana_rpc_api::dev::DevApiServer;
use katana_rpc_api::saya::SayaApiServer;
Expand Down Expand Up @@ -277,16 +277,25 @@ pub async fn spawn<EF: ExecutorFactory>(
methods.register_method("health", |_, _| Ok(serde_json::json!({ "health": true })))?;

if config.apis.contains(&ApiKind::Starknet) {
let starknet_api_config =
StarknetApiConfig { max_event_page_size: config.max_event_page_size };
let server = if let Some(client) = forked_client {
StarknetApi::new_forked(
backend.clone(),
pool.clone(),
block_producer.clone(),
validator,
client,
starknet_api_config,
)
} else {
StarknetApi::new(backend.clone(), pool.clone(), block_producer.clone(), validator)
StarknetApi::new(
backend.clone(),
pool.clone(),
block_producer.clone(),
validator,
starknet_api_config,
)
};

methods.merge(StarknetApiServer::into_rpc(server.clone()))?;
Expand Down
13 changes: 9 additions & 4 deletions crates/katana/rpc/rpc-types/src/error/starknet.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ pub enum StarknetApiError {
#[error("Class hash not found")]
ClassHashNotFound,
#[error("Requested page size is too big")]
PageSizeTooBig,
PageSizeTooBig { requested: u64, max_allowed: u64 },
#[error("There are no blocks")]
NoBlocks,
#[error("The supplied continuation token is invalid or unknown")]
Expand Down Expand Up @@ -96,7 +96,7 @@ impl StarknetApiError {
StarknetApiError::InvalidTxnIndex => 27,
StarknetApiError::ClassHashNotFound => 28,
StarknetApiError::TxnHashNotFound => 29,
StarknetApiError::PageSizeTooBig => 31,
StarknetApiError::PageSizeTooBig { .. } => 31,
StarknetApiError::NoBlocks => 32,
StarknetApiError::InvalidContinuationToken => 33,
StarknetApiError::TooManyKeysInFilter => 34,
Expand Down Expand Up @@ -136,6 +136,9 @@ impl StarknetApiError {
Some(Value::String(reason.to_string()))
}

StarknetApiError::PageSizeTooBig { requested: _, max_allowed: _ } => {
Some(serde_json::json!(self))
}
_ => None,
}
}
Expand Down Expand Up @@ -206,7 +209,9 @@ impl From<StarknetRsError> for StarknetApiError {
StarknetRsError::NoBlocks => Self::NoBlocks,
StarknetRsError::NonAccount => Self::NonAccount,
StarknetRsError::BlockNotFound => Self::BlockNotFound,
StarknetRsError::PageSizeTooBig => Self::PageSizeTooBig,
StarknetRsError::PageSizeTooBig => {
Self::PageSizeTooBig { requested: 0, max_allowed: 0 }
}
Comment on lines +209 to +211
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Consider preserving actual values in error conversion.

The current implementation uses default values (0, 0) when converting from StarknetRsError::PageSizeTooBig. This could be misleading for debugging purposes, sensei.

Consider modifying the StarknetRsError in the upstream library to include these fields, or retrieving the values from the context where the error is generated.

-            StarknetRsError::PageSizeTooBig => {
-                Self::PageSizeTooBig { requested: 0, max_allowed: 0 }
-            }
+            StarknetRsError::PageSizeTooBig => {
+                // TODO: Modify upstream StarknetRsError to include these fields
+                // For now, we could consider logging a warning about lost information
+                log::warn!("Converting PageSizeTooBig error with lost size information");
+                Self::PageSizeTooBig { requested: 0, max_allowed: 0 }
+            }

Committable suggestion skipped: line range outside the PR's diff.

StarknetRsError::DuplicateTx => Self::DuplicateTransaction,
StarknetRsError::ContractNotFound => Self::ContractNotFound,
StarknetRsError::CompilationFailed => Self::CompilationFailed,
Expand Down Expand Up @@ -278,7 +283,7 @@ mod tests {
#[case(StarknetApiError::TxnHashNotFound, 29, "Transaction hash not found")]
#[case(StarknetApiError::ClassAlreadyDeclared, 51, "Class already declared")]
#[case(StarknetApiError::InvalidContractClass, 50, "Invalid contract class")]
#[case(StarknetApiError::PageSizeTooBig, 31, "Requested page size is too big")]
#[case(StarknetApiError::PageSizeTooBig{requested: 1000, max_allowed: 500}, 31, "Requested page size is too big")]
#[case(StarknetApiError::FailedToReceiveTxn, 1, "Failed to write transaction")]
#[case(StarknetApiError::InvalidMessageSelector, 21, "Invalid message selector")]
#[case(StarknetApiError::NonAccount, 58, "Sender address in not an account contract")]
Expand Down
33 changes: 29 additions & 4 deletions crates/katana/rpc/rpc/src/starknet/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@ pub struct StarknetApi<EF: ExecutorFactory> {
inner: Arc<Inner<EF>>,
}

#[derive(Debug, Clone)]
pub struct StarknetApiConfig {
pub max_event_page_size: Option<u64>,
}

impl<EF: ExecutorFactory> Clone for StarknetApi<EF> {
fn clone(&self) -> Self {
Self { inner: Arc::clone(&self.inner) }
Expand All @@ -71,6 +76,7 @@ struct Inner<EF: ExecutorFactory> {
block_producer: BlockProducer<EF>,
blocking_task_pool: BlockingTaskPool,
forked_client: Option<ForkedClient>,
config: StarknetApiConfig,
}

impl<EF: ExecutorFactory> StarknetApi<EF> {
Expand All @@ -79,8 +85,9 @@ impl<EF: ExecutorFactory> StarknetApi<EF> {
pool: TxPool,
block_producer: BlockProducer<EF>,
validator: TxValidator,
config: StarknetApiConfig,
) -> Self {
Self::new_inner(backend, pool, block_producer, validator, None)
Self::new_inner(backend, pool, block_producer, validator, None, config)
}

pub fn new_forked(
Expand All @@ -89,8 +96,9 @@ impl<EF: ExecutorFactory> StarknetApi<EF> {
block_producer: BlockProducer<EF>,
validator: TxValidator,
forked_client: ForkedClient,
config: StarknetApiConfig,
) -> Self {
Self::new_inner(backend, pool, block_producer, validator, Some(forked_client))
Self::new_inner(backend, pool, block_producer, validator, Some(forked_client), config)
}

fn new_inner(
Expand All @@ -99,11 +107,19 @@ impl<EF: ExecutorFactory> StarknetApi<EF> {
block_producer: BlockProducer<EF>,
validator: TxValidator,
forked_client: Option<ForkedClient>,
config: StarknetApiConfig,
) -> Self {
let blocking_task_pool =
BlockingTaskPool::new().expect("failed to create blocking task pool");
let inner =
Inner { pool, backend, block_producer, blocking_task_pool, validator, forked_client };
let inner = Inner {
pool,
backend,
block_producer,
blocking_task_pool,
validator,
forked_client,
config,
};
Self { inner: Arc::new(inner) }
}

Expand Down Expand Up @@ -816,6 +832,15 @@ impl<EF: ExecutorFactory> StarknetApi<EF> {
let EventFilterWithPage { event_filter, result_page_request } = filter;
let ResultPageRequest { continuation_token, chunk_size } = result_page_request;

if let Some(max_size) = self.inner.config.max_event_page_size {
if chunk_size > max_size {
return Err(StarknetApiError::PageSizeTooBig {
requested: chunk_size,
max_allowed: max_size,
});
}
}

self.on_io_blocking_task(move |this| {
let from = match event_filter.from_block {
Some(id) => id,
Expand Down
Loading