From 892f11ef6a2ad4e74dcb31fbeb680ccb29b74cdc Mon Sep 17 00:00:00 2001 From: xander Date: Mon, 15 Jan 2024 08:44:02 +0100 Subject: [PATCH] update core logic of notary, verifier and prover --- tlsn/examples/simple/README.md | 100 ++++------ tlsn/examples/simple/keys/notary.key | 5 + tlsn/examples/simple/keys/notary.pub | 4 + tlsn/examples/simple/simple_notary.rs | 58 ++++++ tlsn/examples/simple/simple_prover.rs | 253 ++++++++++++------------ tlsn/examples/simple/simple_verifier.rs | 37 ++-- 6 files changed, 247 insertions(+), 210 deletions(-) create mode 100644 tlsn/examples/simple/keys/notary.key create mode 100644 tlsn/examples/simple/keys/notary.pub create mode 100644 tlsn/examples/simple/simple_notary.rs diff --git a/tlsn/examples/simple/README.md b/tlsn/examples/simple/README.md index 9c5d1bf454..3ca5c13495 100644 --- a/tlsn/examples/simple/README.md +++ b/tlsn/examples/simple/README.md @@ -1,90 +1,70 @@ -## Simple Example: Notarize Public Data from example.com (Rust) +# Simple Notarization -This example demonstrates the simplest possible use case for TLSNotary: -1. Notarize: Fetch and create a proof of its content. -2. Verify the proof. +This guide will take you through the steps of: +- starting a `Notary` server +- running a `Prover` to notarize some web data +- running a `Verifier` to verify the notarized data -Next, we will redact the content and verify it again: -1. Redact the `USER_AGENT` and titles. -2. Verify the redacted proof. +Note that the TLSNotary protocol assumes that the `Notary` is trusted by the `Verifier`. To minimize the trust, the `Verifier` itself can act as a `Notary`. -### 1. Notarize +## Preliminaries -Run a simple prover: +### Install rust +If you don't have `rust` installed yet, install it with [rustup](https://rustup.rs/): ```shell -cargo run --release --example simple_prover +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ``` -If the notarization was successful, you should see this output in the console: +## Guide +Clone this repository first -```log -Starting an MPC TLS connection with the server -Got a response from the server -Notarization completed successfully! -The proof has been written to `simple_proof.json` +```shell +git clone https://github.com/tlsnotary/tlsn ``` -⚠️ In this simple example the `Notary` server is automatically started in the background. Note that this is for demonstration purposes only. In a real work example, the notary should be run by a neutral party or the verifier of the proofs. Consult the [Notary Server Docs](https://docs.tlsnotary.org/developers/notary_server.html) for more details on how to run a notary server. - -### 2. Verify the Proof - -When you open `simple_proof.json` in an editor, you will see a JSON file with lots of non-human-readable byte arrays. You can decode this file by running: +### Start a simple Notary server: ```shell -cargo run --release --example simple_verifier +cd tlsn/tlsn/examples/simple +cargo run --release --example simple_notary ``` -This will output the TLS-transaction in clear text: +The `Notary` server will now be running in the background waiting for connections from a `Prover`. You can switch to another console to run the `Prover`. + +P/S: The notary server used in this example is less functional compared to its [advanced version](../../../notary-server). This simple version is easier to integrate with from prover perspective, whereas the advanced version provides additional features like TLS connection with prover, WebSocket endpoint, API endpoints for further customisation etc. -```log -Successfully verified that the bytes below came from a session with Dns("example.com") at 2023-11-03 08:48:20 UTC. -Note that the bytes which the Prover chose not to disclose are shown as X. +### Run a simple Prover: -Bytes sent: -... +```shell +RUST_LOG=DEBUG,yamux=INFO cargo run --release --example simple_prover ``` -### 3. Redact Information +The notarization session usually takes a few moments and the resulting proof will be written to the "proof.json" file. The proof can then be passed on to the `Verifier` for verification. -Open `simple/src/examples/simple_prover.rs` and locate the line with: +The `simple_prover` notarizes and redacts the `USER_AGENT` HTTP header from the proof for the `Verifier`. You can change the code in `tlsn/tlsn/examples/simple/simple_prover.rs` to meet your needs: -```rust -let redact = false; -``` +- change which server the `Prover` connects to +- add or remove HTTP request headers +- redact other strings in the request or the response -and change it to: +⚠️ Please note that by default the `Notary` server expects that the cumulative size of the request and the server response is not more than 16KB. -```rust -let redact = true; -``` -Next, if you run the `simple_prover` and `simple_verifier` again, you'll notice redacted `X`'s in the output: +### Run a simple Verifier: ```shell -cargo run --release --example simple_prover cargo run --release --example simple_verifier ``` -```log - - - - XXXXXXXXXXXXXX -... -``` - -You can also use to inspect your proofs. Simply drag and drop `simple_proof.json` from your proof file explorer into the drop zone. Redacted bytes are marked with red Xs characters. - -### (Optional) Extra Experiments - -Feel free to try these extra challenges: - -- [ ] Modify the `server_name` (or any other data) in `simple_proof.json` and verify that the proof is no longer valid. -- [ ] Modify the `build_proof_with_redactions` function in `simple_prover.rs` to redact more or different data. - -### Next steps - -Try out the [Discord example](../Discord/README.md) and notarize a Discord conversations. - +This will verify the proof from the `simple_prover` (`proof.json`) and output the result to the console. +Note how the parts which the prover chose not to disclose will be shown as "X": +```plaintext +GET / HTTP/1.1 +host: example.com +accept: */* +accept-encoding: identity +connection: close +user-agent: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +``` diff --git a/tlsn/examples/simple/keys/notary.key b/tlsn/examples/simple/keys/notary.key new file mode 100644 index 0000000000..a88cf51f80 --- /dev/null +++ b/tlsn/examples/simple/keys/notary.key @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgEvBc/VMWn3E4PGfe +ETc/ekdTRmRwNN9J6eKDPxJ98ZmhRANCAAQG/foUjhkWzMlrQNAUnfBYJe9UsWtx +HMwbmRpN4cahLMO7pwWrHe4RZikUajoLQQ5SB/6YSBuS0utehy/nIfMq +-----END PRIVATE KEY----- diff --git a/tlsn/examples/simple/keys/notary.pub b/tlsn/examples/simple/keys/notary.pub new file mode 100644 index 0000000000..fa63c8d282 --- /dev/null +++ b/tlsn/examples/simple/keys/notary.pub @@ -0,0 +1,4 @@ +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEBv36FI4ZFszJa0DQFJ3wWCXvVLFr +cRzMG5kaTeHGoSzDu6cFqx3uEWYpFGo6C0EOUgf+mEgbktLrXocv5yHzKg== +-----END PUBLIC KEY----- diff --git a/tlsn/examples/simple/simple_notary.rs b/tlsn/examples/simple/simple_notary.rs new file mode 100644 index 0000000000..c73afc4cc9 --- /dev/null +++ b/tlsn/examples/simple/simple_notary.rs @@ -0,0 +1,58 @@ +/// This is a simple implementation of the notary server with minimal functionalities (without TLS, does not support WebSocket and configuration etc.) +/// For a more functional notary server implementation, please use the notary server in `../../notary-server` +use p256::pkcs8::DecodePrivateKey; +use std::env; + +use tokio::net::TcpListener; +use tokio_util::compat::TokioAsyncReadCompatExt; + +use tlsn_verifier::tls::{Verifier, VerifierConfig}; +const SESSION_ID: &str = "example"; + +const NOTARY_SIGNING_KEY_PATH: &str = "./keys/notary.key"; + +#[tokio::main] +async fn main() { + tracing_subscriber::fmt::init(); + + // Allow passing an address to listen on as the first argument of this + // program, but otherwise we'll just set up our TCP listener on + // 127.0.0.1:8080 for connections. + let addr = env::args() + .nth(1) + .unwrap_or_else(|| "127.0.0.1:8080".to_string()); + + // Next up we create a TCP listener which will listen for incoming + // connections. This TCP listener is bound to the address we determined + // above and must be associated with an event loop. + let listener = TcpListener::bind(&addr).await.unwrap(); + + println!("Listening on: {}", addr); + + // Load the notary signing key + let signing_key = + p256::ecdsa::SigningKey::read_pkcs8_pem_file(NOTARY_SIGNING_KEY_PATH).unwrap(); + + loop { + // Asynchronously wait for an inbound socket. + let (socket, socket_addr) = listener.accept().await.unwrap(); + + println!("Accepted connection from: {}", socket_addr); + + { + let signing_key = signing_key.clone(); + + // Spawn notarization task to be run concurrently + tokio::spawn(async move { + // Setup default config. Normally a different ID would be generated + // for each notarization. + let config = VerifierConfig::builder().id(SESSION_ID).build().unwrap(); + + Verifier::new(config) + .notarize::<_, p256::ecdsa::Signature>(socket.compat(), &signing_key) + .await + .unwrap(); + }); + } + } +} diff --git a/tlsn/examples/simple/simple_prover.rs b/tlsn/examples/simple/simple_prover.rs index d9dabcadb5..be4fd24444 100644 --- a/tlsn/examples/simple/simple_prover.rs +++ b/tlsn/examples/simple/simple_prover.rs @@ -3,39 +3,54 @@ /// /// The example uses the notary server implemented in ./simple_notary.rs use futures::AsyncWriteExt; -use hyper::{Body, Request, StatusCode}; +use hyper::{Body, Method, Request, StatusCode}; +use serde::Deserialize; use std::ops::Range; use tlsn_core::proof::TlsProof; -use tokio::io::{AsyncWriteExt as _, DuplexStream}; +use tokio::io::AsyncWriteExt as _; use tokio_util::compat::{FuturesAsyncReadCompatExt, TokioAsyncReadCompatExt}; -use tlsn_prover::tls::{state::Notarize, Prover, ProverConfig}; +use std::env; +use std::str::FromStr; +use tlsn_prover::tls::{Prover, ProverConfig}; // Setting of the application server -const SERVER_DOMAIN: &str = "example.com"; -const USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"; -use p256::pkcs8::DecodePrivateKey; -use std::str; +const USER_AGENT: &str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"; -use tlsn_verifier::tls::{Verifier, VerifierConfig}; +// Setting of the notary server — make sure these are the same with those in ./simple_notary.rs +const NOTARY_HOST: &str = "127.0.0.1"; +const NOTARY_PORT: u16 = 8080; +const SESSION_ID: &str = "example"; #[tokio::main] async fn main() { + // Initialize logging tracing_subscriber::fmt::init(); - let (prover_socket, notary_socket) = tokio::io::duplex(1 << 16); + // recieve logging here from the cli about which url to call + let args: Vec = env::args().collect(); - // Start a local simple notary service - start_notary_thread(prover_socket).await; + // validate that at least one more parameter is provided + assert!(args.len() >= 2, "Please provide request structure"); + let request_json = args[1].clone(); + + // read the proxy request + let req_proxy: ProxyRequest = serde_json::from_str(&request_json[..]).unwrap(); // A Prover configuration let config = ProverConfig::builder() - .id("example") - .server_dns(SERVER_DOMAIN) + .id(SESSION_ID) + .server_dns(req_proxy.host.clone()) .build() .unwrap(); + // Connect to the Notary + let notary_socket = tokio::net::TcpStream::connect((NOTARY_HOST, NOTARY_PORT)) + .await + .unwrap(); + println!("Connected to the Notary"); + // Create a Prover and set it up with the Notary // This will set up the MPC backend prior to connecting to the server. let prover = Prover::new(config) @@ -44,7 +59,7 @@ async fn main() { .unwrap(); // Connect to the Server via TCP. This is the TLS client socket. - let client_socket = tokio::net::TcpStream::connect((SERVER_DOMAIN, 443)) + let client_socket = tokio::net::TcpStream::connect((req_proxy.host.clone(), 443)) .await .unwrap(); @@ -66,26 +81,17 @@ async fn main() { let connection_task = tokio::spawn(connection.without_shutdown()); // Build a simple HTTP request with common headers - let request = Request::builder() - .uri("/") - .header("Host", SERVER_DOMAIN) - .header("Accept", "*/*") - // Using "identity" instructs the Server not to use compression for its HTTP response. - // TLSNotary tooling does not support compression. - .header("Accept-Encoding", "identity") - .header("Connection", "close") - .header("User-Agent", USER_AGENT) - .body(Body::empty()) - .unwrap(); + let request: Request = build_request(req_proxy); println!("Starting an MPC TLS connection with the server"); - // Send the request to the Server and get a response via the MPC TLS connection + // Pass our request builder object to our client. let response = request_sender.send_request(request).await.unwrap(); - println!("Got a response from the server"); + println!("Got a response from the server: {:?}", response); - assert!(response.status() == StatusCode::OK); + assert!(response.status() == StatusCode::OK || response.status() == StatusCode::CREATED); + println!("{:?}", response); // Close the connection to the server let mut client_socket = connection_task.await.unwrap().unwrap().io.into_inner(); @@ -95,85 +101,8 @@ async fn main() { let prover = prover_task.await.unwrap().unwrap(); // Prepare for notarization. - let prover = prover.start_notarize(); - - // Build proof (with or without redactions) - let redact = false; - let proof = if !redact { - build_proof_without_redactions(prover).await - } else { - build_proof_with_redactions(prover).await - }; - - // Write the proof to a file - let mut file = tokio::fs::File::create("simple_proof.json").await.unwrap(); - file.write_all(serde_json::to_string_pretty(&proof).unwrap().as_bytes()) - .await - .unwrap(); - - println!("Notarization completed successfully!"); - println!("The proof has been written to `simple_proof.json`"); -} + let mut prover = prover.start_notarize(); -/// Find the ranges of the public and private parts of a sequence. -/// -/// Returns a tuple of `(public, private)` ranges. -fn find_ranges(seq: &[u8], private_seq: &[&[u8]]) -> (Vec>, Vec>) { - let mut private_ranges = Vec::new(); - for s in private_seq { - for (idx, w) in seq.windows(s.len()).enumerate() { - if w == *s { - private_ranges.push(idx..(idx + w.len())); - } - } - } - - let mut sorted_ranges = private_ranges.clone(); - sorted_ranges.sort_by_key(|r| r.start); - - let mut public_ranges = Vec::new(); - let mut last_end = 0; - for r in sorted_ranges { - if r.start > last_end { - public_ranges.push(last_end..r.start); - } - last_end = r.end; - } - - if last_end < seq.len() { - public_ranges.push(last_end..seq.len()); - } - - (public_ranges, private_ranges) -} - -async fn build_proof_without_redactions(mut prover: Prover) -> TlsProof { - let sent_len = prover.sent_transcript().data().len(); - let recv_len = prover.recv_transcript().data().len(); - - let builder = prover.commitment_builder(); - let sent_commitment = builder.commit_sent(0..sent_len).unwrap(); - let recv_commitment = builder.commit_recv(0..recv_len).unwrap(); - - // Finalize, returning the notarized session - let notarized_session = prover.finalize().await.unwrap(); - - // Create a proof for all committed data in this session - let mut proof_builder = notarized_session.data().build_substrings_proof(); - - // Reveal all the public ranges - proof_builder.reveal(sent_commitment).unwrap(); - proof_builder.reveal(recv_commitment).unwrap(); - - let substrings_proof = proof_builder.build().unwrap(); - - TlsProof { - session: notarized_session.session_proof(), - substrings: substrings_proof, - } -} - -async fn build_proof_with_redactions(mut prover: Prover) -> TlsProof { // Identify the ranges in the outbound data which contain data which we want to disclose let (sent_public_ranges, _) = find_ranges( prover.sent_transcript().data(), @@ -188,7 +117,7 @@ async fn build_proof_with_redactions(mut prover: Prover) -> TlsProof { prover.recv_transcript().data(), &[ // Redact the value of the title. It will NOT be disclosed. - "Example Domain".as_bytes(), + // "Example Domain".as_bytes(), ], ); @@ -221,31 +150,97 @@ async fn build_proof_with_redactions(mut prover: Prover) -> TlsProof { let substrings_proof = proof_builder.build().unwrap(); - TlsProof { + let proof = TlsProof { session: notarized_session.session_proof(), substrings: substrings_proof, + }; + + // Write the proof to a file + let mut file = tokio::fs::File::create("proof.json").await.unwrap(); + file.write_all(serde_json::to_string_pretty(&proof).unwrap().as_bytes()) + .await + .unwrap(); + + println!("Notarization completed successfully!"); + println!("The proof has been written to proof.json"); +} + +/// Find the ranges of the public and private parts of a sequence. +/// +/// Returns a tuple of `(public, private)` ranges. +fn find_ranges(seq: &[u8], private_seq: &[&[u8]]) -> (Vec>, Vec>) { + let mut private_ranges = Vec::new(); + for s in private_seq { + for (idx, w) in seq.windows(s.len()).enumerate() { + if w == *s { + private_ranges.push(idx..(idx + w.len())); + } + } + } + + let mut sorted_ranges = private_ranges.clone(); + sorted_ranges.sort_by_key(|r| r.start); + + let mut public_ranges = Vec::new(); + let mut last_end = 0; + for r in sorted_ranges { + if r.start > last_end { + public_ranges.push(last_end..r.start); + } + last_end = r.end; + } + + if last_end < seq.len() { + public_ranges.push(last_end..seq.len()); } + + (public_ranges, private_ranges) } -async fn start_notary_thread(socket: DuplexStream) { - tokio::spawn(async { - // Load the notary signing key - let signing_key_str = str::from_utf8(include_bytes!( - "../../../notary-server/fixture/notary/notary.key" - )) - .unwrap(); - let signing_key = p256::ecdsa::SigningKey::from_pkcs8_pem(signing_key_str).unwrap(); - - // Spawn notarization task to be run concurrently - tokio::spawn(async move { - // Setup default config. Normally a different ID would be generated - // for each notarization. - let config = VerifierConfig::builder().id("example").build().unwrap(); - - Verifier::new(config) - .notarize::<_, p256::ecdsa::Signature>(socket.compat(), &signing_key) - .await - .unwrap(); - }); - }); +#[derive(Debug, Deserialize, Clone)] +struct ProxyRequest { + url: String, + method: String, + host: String, + headers: Vec
, + body: Option, +} + +#[derive(Debug, Deserialize, Clone)] +struct Header { + key: String, + value: String, +} + +fn build_request(proxy_request: ProxyRequest) -> Request { + let request_method = Method::from_str(&proxy_request.method[..]).unwrap(); + let request_body = proxy_request + .clone() + .body + .map(|_| Body::from(proxy_request.body.clone().unwrap())) + .unwrap_or_else(Body::empty); + + println!( + "Building proxy request {:?} for MPC-TLS connection", + proxy_request + ); + + let mut builder = Request::builder() + .uri(proxy_request.url) + .method(request_method) + // add basic headers + .header("Host", proxy_request.host) + .header("Accept", "*/*") + .header("Cache-Control", "no-cache") + .header("Connection", "close") + .header("User-Agent", USER_AGENT) + // Using "identity" instructs the Server not to use compression for its HTTP response. + // TLSNotary tooling does not support compression. + .header("Accept-Encoding", "identity"); + + for item in proxy_request.headers.iter() { + builder = builder.header(item.key.clone(), item.value.clone()); + } + + builder.body(request_body).unwrap() } diff --git a/tlsn/examples/simple/simple_verifier.rs b/tlsn/examples/simple/simple_verifier.rs index c3d65f7aa0..de77b0bf1c 100644 --- a/tlsn/examples/simple/simple_verifier.rs +++ b/tlsn/examples/simple/simple_verifier.rs @@ -1,4 +1,4 @@ -use std::{str, time::Duration}; +use std::time::Duration; use elliptic_curve::pkcs8::DecodePublicKey; @@ -8,7 +8,7 @@ use tlsn_core::proof::{SessionProof, TlsProof}; /// it and prints the verified data to the console. fn main() { // Deserialize the proof - let proof = std::fs::read_to_string("simple_proof.json").unwrap(); + let proof = std::fs::read_to_string("proof.json").unwrap(); let proof: TlsProof = serde_json::from_str(proof.as_str()).unwrap(); let TlsProof { @@ -49,28 +49,23 @@ fn main() { sent.set_redacted(b'X'); recv.set_redacted(b'X'); - println!("-------------------------------------------------------------------"); - println!( - "Successfully verified that the bytes below came from a session with {:?} at {}.", - session_info.server_name, time - ); - println!("Note that the bytes which the Prover chose not to disclose are shown as X."); - println!(); - println!("Bytes sent:"); - println!(); + // println!( + // "Successfully verified that the bytes below came from a session with {:?} at {}.", + // session_info.server_name, time + // ); + // println!("Bytes sent:"); + // println!(); print!("{}", String::from_utf8(sent.data().to_vec()).unwrap()); - println!(); - println!("Bytes received:"); - println!(); - println!("{}", String::from_utf8(recv.data().to_vec()).unwrap()); - println!("-------------------------------------------------------------------"); + println!("seperator"); + // println!("Bytes received:"); + // println!(); + print!("{}", String::from_utf8(recv.data().to_vec()).unwrap()); + // println!("-------------------------------------------------------------------"); } /// Returns a Notary pubkey trusted by this Verifier fn notary_pubkey() -> p256::PublicKey { - let pem_file = str::from_utf8(include_bytes!( - "../../../notary-server/fixture/notary/notary.pub" - )) - .unwrap(); - p256::PublicKey::from_public_key_pem(pem_file).unwrap() + let pem_file_path = "./keys/notary.pub"; + + p256::PublicKey::read_public_key_pem_file(pem_file_path).unwrap() }