From 39a0becbb7f1bdef256dd0ade5a0bb53d0e3ce6e Mon Sep 17 00:00:00 2001 From: Yan Chen <48968912+chenyan-dfinity@users.noreply.github.com> Date: Fri, 11 Aug 2023 10:03:31 -0700 Subject: [PATCH] offline mode send (#72) * output all offline calls to messages.json; add -i option to enter repl after script * checkpoint * works * fix * fix * add send function * fix --- Cargo.lock | 3 +- Cargo.toml | 3 +- README.md | 8 ++- src/exp.rs | 46 +++++++++++---- src/helper.rs | 6 ++ src/lib.rs | 14 ----- src/main.rs | 79 ++++++++++++++++---------- src/offline.rs | 151 ++++++++++++++++++++++++++++++++++++++++++++----- 8 files changed, 237 insertions(+), 73 deletions(-) delete mode 100644 src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 7ec446f..38847fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1217,7 +1217,7 @@ dependencies = [ [[package]] name = "ic-repl" -version = "0.4.1" +version = "0.4.2" dependencies = [ "ansi_term", "anyhow", @@ -1246,6 +1246,7 @@ dependencies = [ "rustyline", "rustyline-derive", "serde", + "serde_cbor", "serde_json", "sha2 0.10.7", "shellexpand", diff --git a/Cargo.toml b/Cargo.toml index 0804c69..7830402 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ic-repl" -version = "0.4.1" +version = "0.4.2" authors = ["DFINITY Team"] edition = "2021" default-run = "ic-repl" @@ -38,6 +38,7 @@ ring = "0.16" rpassword = "7.2" serde = "1.0" serde_json = "1.0" +serde_cbor = "0.11" hex = { version = "0.4", features = ["serde"] } sha2 = "0.10" crc32fast = "1.3" diff --git a/README.md b/README.md index b4afcbb..8cdcc9a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Canister REPL ``` -ic-repl [--replica [local|ic|url] | --offline [--format [ascii|png]]] --config [script file] +ic-repl [--replica [local|ic|url] | --offline [--format [json|ascii|png]]] --config [script file] ``` ## Commands @@ -47,7 +47,6 @@ You cannot define recursive functions, as there is no control flow in the langua We also provide some built-in functions: * `account(principal)`: convert principal to account id. * `neuron_account(principal, nonce)`: convert (principal, nonce) to account in the governance canister. -* `metadata(principal, path)`: fetch the HTTP endpoint of `canister//`. * `file(path)`: load external file as a blob value. * `gzip(blob)`: gzip a blob value. * `stringify(exp1, exp2, exp3, ...)`: convert all expressions to string and concat. Only supports primitive types. @@ -57,6 +56,11 @@ We also provide some built-in functions: * `concat(e1, e2)`: concatenate two vec/record/text together. * `add/sub/mul/div(e1, e2)`: addition/subtraction/multiplication/division of two integer/float numbers. If one of the arguments is float32/float64, the result is float64; otherwise, the result is integer. You can use type annotation to get the integer part of the float number. For example `div((mul(div(1, 3.0), 1000) : nat), 100.0)` returns `3.33`. +The following functions are only available in non-offline mode: +* `metadata(principal, path)`: fetch the HTTP endpoint of `canister//`. +* `send(blob)`: send signed JSON messages generated from offline mode. The function can take a single message or an array of messages. Most likely use is `send(file("messages.json"))`. The return result is the return results of all calls. Alternatively, you can use `ic-repl -s messages.json -r ic`. + + ## Object methods For `vec`, `record` or `text` value, we provide some built-in methods for value transformation: diff --git a/src/exp.rs b/src/exp.rs index 952d78e..fcd5a63 100644 --- a/src/exp.rs +++ b/src/exp.rs @@ -12,7 +12,6 @@ use candid::{ utils::check_unique, Principal, TypeEnv, }; -use ic_agent::Agent; use std::collections::BTreeMap; #[derive(Debug, Clone)] @@ -127,7 +126,7 @@ impl Exp { } _ => return Err(anyhow!("neuron_account expects (principal, nonce)")), }, - "metadata" => match args.as_slice() { + "metadata" if helper.offline.is_none() => match args.as_slice() { [IDLValue::Principal(id), IDLValue::Text(path)] => { let res = fetch_metadata(&helper.agent, *id, path)?; IDLValue::Vec(res.into_iter().map(IDLValue::Nat8).collect()) @@ -164,6 +163,23 @@ impl Exp { } _ => return Err(anyhow!("gzip expects blob")), }, + "send" if helper.offline.is_none() => match args.as_slice() { + [IDLValue::Vec(blob)] => { + use crate::offline::{send, send_messages}; + let blob: Vec = blob.iter().filter_map(|v| match v { + IDLValue::Nat8(n) => Some(*n), + _ => None, + }).collect(); + let json = std::str::from_utf8(&blob)?; + let res = match json.trim_start().chars().next() { + Some('{') => send(helper, &serde_json::from_str(json)?)?, + Some('[') => send_messages(helper, &serde_json::from_str(json)?)?, + _ => return Err(anyhow!("not a valid json message")), + }; + args_to_value(res) + } + _ => return Err(anyhow!("send expects a json blob")), + }, "wasm_profiling" => match args.as_slice() { [IDLValue::Text(file)] | [IDLValue::Text(file), IDLValue::Vec(_)] => { let path = resolve_path(&helper.base_path, file); @@ -360,7 +376,7 @@ impl Exp { 0 }; let res = call( - &helper.agent, + helper, &info.canister_id, &method.method, &bytes, @@ -482,7 +498,7 @@ pub struct MethodInfo { pub profiling: Option>, } impl Method { - fn get_info(&self, helper: &MyHelper) -> Result { + pub fn get_info(&self, helper: &MyHelper) -> Result { let canister_id = str_to_principal(&self.canister, helper)?; let agent = &helper.agent; let mut map = helper.canister_map.borrow_mut(); @@ -528,7 +544,7 @@ impl Method { #[tokio::main] async fn call( - agent: &Agent, + helper: &MyHelper, canister_id: &Principal, method: &str, args: &[u8], @@ -536,6 +552,7 @@ async fn call( offline: &Option, ) -> anyhow::Result { use crate::offline::*; + let agent = &helper.agent; let effective_id = get_effective_canister_id(*canister_id, method, args)?; let is_query = opt_func .as_ref() @@ -547,12 +564,17 @@ async fn call( .with_arg(args) .with_effective_canister_id(effective_id); if let Some(offline) = offline { + let mut msgs = helper.messages.borrow_mut(); let signed = builder.sign()?; - let message = Ingress { - call_type: "query".to_owned(), - request_id: None, - content: hex::encode(signed.signed_query), + let message = IngressWithStatus { + ingress: Ingress { + call_type: "query".to_owned(), + request_id: None, + content: hex::encode(signed.signed_query), + }, + request_status: None, }; + msgs.push(message.clone()); output_message(serde_json::to_string(&message)?, offline)?; return Ok(IDLArgs::new(&[])); } else { @@ -564,6 +586,7 @@ async fn call( .with_arg(args) .with_effective_canister_id(effective_id); if let Some(offline) = offline { + let mut msgs = helper.messages.borrow_mut(); let signed = builder.sign()?; let status = agent.sign_request_status(effective_id, signed.request_id)?; let message = IngressWithStatus { @@ -572,12 +595,13 @@ async fn call( request_id: Some(hex::encode(signed.request_id.as_slice())), content: hex::encode(signed.signed_update), }, - request_status: RequestStatus { + request_status: Some(RequestStatus { canister_id: status.effective_canister_id, request_id: hex::encode(status.request_id.as_slice()), content: hex::encode(status.signed_request_status), - }, + }), }; + msgs.push(message.clone()); output_message(serde_json::to_string(&message)?, offline)?; return Ok(IDLArgs::new(&[])); } else { diff --git a/src/helper.rs b/src/helper.rs index 41a3230..687c689 100644 --- a/src/helper.rs +++ b/src/helper.rs @@ -87,6 +87,7 @@ pub struct MyHelper { pub func_env: FuncEnv, pub base_path: std::path::PathBuf, pub history: Vec, + pub messages: RefCell>, } impl MyHelper { @@ -108,6 +109,7 @@ impl MyHelper { agent: self.agent.clone(), agent_url: self.agent_url.clone(), offline: self.offline.clone(), + messages: self.messages.clone(), } } pub fn new(agent: Agent, agent_url: String, offline: Option) -> Self { @@ -125,6 +127,7 @@ impl MyHelper { func_env: FuncEnv::default(), base_path: std::env::current_dir().unwrap(), history: Vec::new(), + messages: Vec::new().into(), agent, agent_url, offline, @@ -186,6 +189,9 @@ impl MyHelper { }; Ok(()) } + pub fn dump_ingress(&self) -> anyhow::Result<()> { + crate::offline::dump_ingress(&self.messages.borrow()) + } } #[derive(Debug, PartialEq, Clone)] diff --git a/src/lib.rs b/src/lib.rs deleted file mode 100644 index 72139c0..0000000 --- a/src/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -mod account_identifier; -mod command; -mod error; -mod exp; -mod grammar; -mod helper; -mod offline; -mod profiling; -mod selector; -mod token; -mod utils; - -pub use command::Command; -pub use helper::MyHelper; diff --git a/src/main.rs b/src/main.rs index b723d71..ab17c71 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,6 @@ use ansi_term::Color; use clap::Parser; +use ic_agent::agent::http_transport::ReqwestHttpReplicaV2Transport as V2Transport; use ic_agent::Agent; use rustyline::error::ReadlineError; use rustyline::CompletionType; @@ -38,8 +39,8 @@ fn repl(opts: Opts) -> anyhow::Result<()> { .url .unwrap_or_else(|| "https://qhmh2-niaaa-aaaab-qadta-cai.raw.icp0.io/?msg=".to_string()); Some(match opts.format.as_deref() { - Some("json") => OfflineOutput::Json, - None | Some("ascii") => OfflineOutput::Ascii(send_url), + None | Some("json") => OfflineOutput::Json, + Some("ascii") => OfflineOutput::Ascii(send_url), Some("png") => OfflineOutput::Png(send_url), Some("png_no_url") => OfflineOutput::PngNoUrl, Some("ascii_no_url") => OfflineOutput::AsciiNoUrl, @@ -55,9 +56,7 @@ fn repl(opts: Opts) -> anyhow::Result<()> { }; println!("Ping {url}..."); let agent = Agent::builder() - .with_transport( - ic_agent::agent::http_transport::ReqwestHttpReplicaV2Transport::create(url)?, - ) + .with_transport(V2Transport::create(url)?) .build()?; println!("Canister REPL"); @@ -66,6 +65,13 @@ fn repl(opts: Opts) -> anyhow::Result<()> { .completion_type(CompletionType::List) .build(); let h = MyHelper::new(agent, url.to_string(), offline); + if let Some(file) = opts.send { + use crate::offline::{send_messages, Messages}; + let json = std::fs::read_to_string(file)?; + let msgs = serde_json::from_str::(&json)?; + send_messages(&h, &msgs)?; + return Ok(()); + } let mut rl = rustyline::Editor::with_config(config)?; rl.set_helper(Some(h)); if rl.load_history("./.history").is_err() { @@ -75,36 +81,45 @@ fn repl(opts: Opts) -> anyhow::Result<()> { let config = std::fs::read_to_string(file)?; rl.helper_mut().unwrap().config = candid::parser::configs::Configs::from_dhall(&config)?; } + + let enter_repl = opts.script.is_none() || opts.interactive; if let Some(file) = opts.script { let cmd = Command::Load(file); let helper = rl.helper_mut().unwrap(); - return cmd.run(helper); + cmd.run(helper)?; } - - let mut count = 1; - loop { - let identity = &rl.helper().unwrap().current_identity; - let p = format!("{identity}@{replica} {count}> "); - rl.helper_mut().unwrap().colored_prompt = format!("{}", Color::Green.bold().paint(&p)); - let input = rl.readline(&p); - match input { - Ok(line) => { - rl.add_history_entry(&line)?; - unwrap(pretty_parse::("stdin", &line), |cmd| { - let helper = rl.helper_mut().unwrap(); - helper.history.push(line.clone()); - unwrap(cmd.run(helper), |_| {}); - }); - } - Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => break, - Err(err) => { - eprintln!("Error: {err:?}"); - break; + if enter_repl { + let mut count = 1; + loop { + let identity = &rl.helper().unwrap().current_identity; + let p = format!("{identity}@{replica} {count}> "); + rl.helper_mut().unwrap().colored_prompt = format!("{}", Color::Green.bold().paint(&p)); + let input = rl.readline(&p); + match input { + Ok(line) => { + rl.add_history_entry(&line)?; + unwrap(pretty_parse::("stdin", &line), |cmd| { + let helper = rl.helper_mut().unwrap(); + helper.history.push(line.clone()); + unwrap(cmd.run(helper), |_| {}); + }); + } + Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => break, + Err(err) => { + eprintln!("Error: {err:?}"); + break; + } } + count += 1; + } + rl.save_history("./.history")?; + } + if opts.offline { + let helper = rl.helper().unwrap(); + if !helper.messages.borrow().is_empty() { + helper.dump_ingress()?; } - count += 1; } - rl.save_history("./.history")?; Ok(()) } @@ -115,7 +130,7 @@ struct Opts { /// Specifies replica URL, possible values: local, ic, URL replica: Option, #[clap(short, long, conflicts_with("replica"))] - /// Offline mode to be run in air-gap machines + /// Offline mode to be run in air-gap machines. All signed messages will be stored in messages.json offline: bool, #[clap(short, long, requires("offline"), value_parser = ["ascii", "json", "png", "ascii_no_url", "png_no_url"])] /// Offline output format @@ -128,6 +143,12 @@ struct Opts { config: Option, /// ic-repl script file script: Option, + #[clap(short, long, requires("script"))] + /// Enter repl once the script is finished + interactive: bool, + #[clap(short, long, conflicts_with("script"), conflicts_with("offline"))] + /// Send signed messages + send: Option, } fn main() -> anyhow::Result<()> { diff --git a/src/offline.rs b/src/offline.rs index 4deadab..8a72190 100644 --- a/src/offline.rs +++ b/src/offline.rs @@ -1,25 +1,64 @@ -use crate::helper::OfflineOutput; +use crate::helper::{MyHelper, OfflineOutput}; +use crate::utils::args_to_value; +use anyhow::{anyhow, Context, Result}; use candid::Principal; +use candid::{types::Function, IDLArgs, TypeEnv}; +use ic_agent::{agent::RequestStatusResponse, Agent}; +use serde::{Deserialize, Serialize}; -#[derive(serde::Serialize)] +#[derive(Serialize, Deserialize, Clone)] pub struct Ingress { pub call_type: String, pub request_id: Option, pub content: String, } -#[derive(serde::Serialize)] +#[derive(Serialize, Deserialize, Clone)] pub struct RequestStatus { pub canister_id: Principal, pub request_id: String, pub content: String, } -#[derive(serde::Serialize)] +#[derive(Serialize, Deserialize, Clone)] pub struct IngressWithStatus { pub ingress: Ingress, - pub request_status: RequestStatus, + pub request_status: Option, } +#[derive(Serialize, Deserialize, Clone)] +pub struct Messages(Vec); + static mut PNG_COUNTER: u32 = 0; -pub fn output_message(json: String, format: &OfflineOutput) -> anyhow::Result<()> { + +impl Ingress { + pub fn parse(&self) -> Result<(Principal, Principal, String, Vec)> { + use serde_cbor::Value; + let cbor: Value = serde_cbor::from_slice(&hex::decode(&self.content)?) + .context("Invalid cbor data in the content of the message.")?; + if let Value::Map(m) = cbor { + let cbor_content = m + .get(&Value::Text("content".to_string())) + .ok_or_else(|| anyhow!("Invalid cbor content"))?; + if let Value::Map(m) = cbor_content { + if let ( + Some(Value::Bytes(sender)), + Some(Value::Bytes(canister_id)), + Some(Value::Text(method_name)), + Some(Value::Bytes(arg)), + ) = ( + m.get(&Value::Text("sender".to_string())), + m.get(&Value::Text("canister_id".to_string())), + m.get(&Value::Text("method_name".to_string())), + m.get(&Value::Text("arg".to_string())), + ) { + let sender = Principal::try_from(sender)?; + let canister_id = Principal::try_from(canister_id)?; + return Ok((sender, canister_id, method_name.to_string(), arg.to_vec())); + } + } + } + Err(anyhow!("Invalid cbor content")) + } +} +pub fn output_message(json: String, format: &OfflineOutput) -> Result<()> { match format { OfflineOutput::Json => println!("{json}"), _ => { @@ -50,7 +89,6 @@ pub fn output_message(json: String, format: &OfflineOutput) -> anyhow::Result<() OfflineOutput::Ascii(_) | OfflineOutput::AsciiNoUrl => { let img = code.render::().build(); println!("{img}"); - pause()?; } OfflineOutput::Png(_) | OfflineOutput::PngNoUrl => { let img = code.render::>().build(); @@ -67,13 +105,96 @@ pub fn output_message(json: String, format: &OfflineOutput) -> anyhow::Result<() }; Ok(()) } - -fn pause() -> anyhow::Result<()> { - use std::io::{Read, Write}; - let mut stdin = std::io::stdin(); - let mut stdout = std::io::stdout(); - eprint!("Press [enter] to continue..."); - stdout.flush()?; - let _ = stdin.read(&mut [0u8])?; +pub fn dump_ingress(msgs: &[IngressWithStatus]) -> Result<()> { + use std::fs::File; + use std::io::Write; + let msgs = Messages(msgs.to_vec()); + let json = serde_json::to_string(&msgs)?; + let mut file = File::create("messages.json")?; + file.write_all(json.as_bytes())?; Ok(()) } + +pub fn send_messages(helper: &MyHelper, msgs: &Messages) -> Result { + let len = msgs.0.len(); + let mut res = Vec::with_capacity(len); + println!("Sending {} messages to {}", len, helper.agent_url); + for (i, msg) in msgs.0.iter().enumerate() { + print!("[{}/{}] ", i + 1, len); + let args = send(helper, msg)?; + res.push(args_to_value(args)) + } + Ok(IDLArgs::new(&res)) +} +pub fn send(helper: &MyHelper, msg: &IngressWithStatus) -> Result { + let message = &msg.ingress; + let (sender, canister_id, method_name, bytes) = message.parse()?; + let meth = crate::exp::Method { + canister: canister_id.to_string(), + method: method_name.clone(), + }; + let opt_func = meth.get_info(helper)?.signature; + let args = if let Some((env, func)) = &opt_func { + IDLArgs::from_bytes_with_types(&bytes, env, &func.args)? + } else { + IDLArgs::from_bytes(&bytes)? + }; + println!("Sending {} call as {}:", message.call_type, sender); + println!(" call \"{}\".{}{};", canister_id, method_name, args); + println!("Do you want to send this message? [y/N]"); + let mut input = String::new(); + std::io::stdin().read_line(&mut input)?; + if !["y", "yes"].contains(&input.to_lowercase().trim()) { + return Err(anyhow!("Send abort")); + } + send_internal(&helper.agent, canister_id, msg, &opt_func) +} +#[tokio::main] +async fn send_internal( + agent: &Agent, + canister_id: Principal, + message: &IngressWithStatus, + opt_func: &Option<(TypeEnv, Function)>, +) -> Result { + let content = hex::decode(&message.ingress.content)?; + let response = match message.ingress.call_type.as_str() { + "query" => agent.query_signed(canister_id, content).await?, + "update" => { + let request_id = agent.update_signed(canister_id, content).await?; + println!("Request ID: 0x{}", String::from(request_id)); + let status = message + .request_status + .as_ref() + .ok_or_else(|| anyhow!("Cannot get request status for update call"))?; + if !(status.canister_id == canister_id && status.request_id == String::from(request_id)) + { + return Err(anyhow!("request_id does match, cannot request status")); + } + let status = hex::decode(&status.content)?; + let ic_agent::agent::Replied::CallReplied(blob) = async { + loop { + match agent + .request_status_signed(&request_id, canister_id, status.clone()) + .await? + { + RequestStatusResponse::Replied { reply } => return Ok(reply), + RequestStatusResponse::Rejected(response) => return Err(anyhow!(response)), + RequestStatusResponse::Done => return Err(anyhow!("No response")), + _ => println!("The request is being processed..."), + }; + std::thread::sleep(std::time::Duration::from_millis(500)); + } + } + .await?; + blob + } + _ => unreachable!(), + }; + let res = if let Some((env, func)) = &opt_func { + IDLArgs::from_bytes_with_types(&response, env, &func.rets)? + } else { + IDLArgs::from_bytes(&response)? + }; + println!("{}", res); + Ok(res) +}