diff --git a/docs/docs/migration_notes.md b/docs/docs/migration_notes.md index 84912b63530..2a52292eb8d 100644 --- a/docs/docs/migration_notes.md +++ b/docs/docs/migration_notes.md @@ -6,6 +6,27 @@ keywords: [sandbox, aztec, notes, migration, updating, upgrading] Aztec is in full-speed development. Literally every version breaks compatibility with the previous ones. This page attempts to target errors and difficulties you might encounter when upgrading, and how to resolve them. +### TBD + +### [Aztec.nr] Introduction of `WithHash` +`WithHash` is a struct that allows for efficient reading of value `T` from public storage in private. +This is achieved by storing the value with its hash, then obtaining the values via an oracle and verifying them against the hash. +This results in in a fewer tree inclusion proofs for values `T` that are packed into more than a single field. + +`WithHash` is leveraged by state variables like `PublicImmutable`. +This is a breaking change because now we require values stored in `PublicImmutable` and `SharedMutable` to implement the `Eq` trait. + +To implement the `Eq` trait you can use the `#[derive(Eq)]` macro: + +```diff ++ use std::meta::derive; + ++ #[derive(Eq)] +pub struct YourType { + ... +} +``` + ## 0.73.0 ### [Token, FPC] Moving fee-related complexity from the Token to the FPC diff --git a/noir-projects/aztec-nr/aztec/src/state_vars/public_immutable.nr b/noir-projects/aztec-nr/aztec/src/state_vars/public_immutable.nr index ec8c4a2f959..012a0592d2c 100644 --- a/noir-projects/aztec-nr/aztec/src/state_vars/public_immutable.nr +++ b/noir-projects/aztec-nr/aztec/src/state_vars/public_immutable.nr @@ -1,12 +1,19 @@ use crate::{ context::{PrivateContext, PublicContext, UnconstrainedContext}, - history::public_storage::PublicStorageHistoricalRead, state_vars::storage::Storage, + utils::with_hash::WithHash, }; use dep::protocol_types::{constants::INITIALIZATION_SLOT_SEPARATOR, traits::Packable}; /// Stores an immutable value in public state which can be read from public, private and unconstrained execution /// contexts. +/// +/// Leverages `WithHash` to enable efficient private reads of public storage. `WithHash` wrapper allows for +/// efficient reads by verifying large values through a single hash check and then proving inclusion only of the hash +/// in the public storage. This reduces the number of required tree inclusion proofs from O(M) to O(1). +/// +/// This is valuable when T packs to multiple fields, as it maintains "almost constant" verification overhead +/// regardless of the original data size. // docs:start:public_immutable_struct pub struct PublicImmutable { context: Context, @@ -14,9 +21,11 @@ pub struct PublicImmutable { } // docs:end:public_immutable_struct -impl Storage for PublicImmutable +/// `WithHash` stores both the packed value (using N fields) and its hash (1 field), requiring N = M + 1 total +/// fields. +impl Storage for PublicImmutable where - T: Packable, + WithHash: Packable, { fn get_storage_slot(self) -> Field { self.storage_slot @@ -38,7 +47,7 @@ impl PublicImmutable { impl PublicImmutable where - T: Packable, + T: Packable + Eq, { // docs:start:public_immutable_struct_write pub fn initialize(self, value: T) { @@ -49,41 +58,36 @@ where // We populate the initialization slot with a non-zero value to indicate that the struct is initialized self.context.storage_write(initialization_slot, 0xdead); - self.context.storage_write(self.storage_slot, value); + self.context.storage_write(self.storage_slot, WithHash::new(value)); } // docs:end:public_immutable_struct_write // Note that we don't access the context, but we do call oracles that are only available in public // docs:start:public_immutable_struct_read pub fn read(self) -> T { - self.context.storage_read(self.storage_slot) + WithHash::public_storage_read(*self.context, self.storage_slot) } // docs:end:public_immutable_struct_read } impl PublicImmutable where - T: Packable, + T: Packable + Eq, { pub unconstrained fn read(self) -> T { - self.context.storage_read(self.storage_slot) + WithHash::unconstrained_public_storage_read(self.context, self.storage_slot) } } impl PublicImmutable where - T: Packable, + T: Packable + Eq, { pub fn read(self) -> T { - let header = self.context.get_block_header(); - let mut fields = [0; T_PACKED_LEN]; - - for i in 0..fields.len() { - fields[i] = header.public_storage_historical_read( - self.storage_slot + i as Field, - (*self.context).this_address(), - ); - } - T::unpack(fields) + WithHash::historical_public_storage_read( + self.context.get_block_header(), + self.context.this_address(), + self.storage_slot, + ) } } diff --git a/noir-projects/aztec-nr/aztec/src/utils/mod.nr b/noir-projects/aztec-nr/aztec/src/utils/mod.nr index 9cdf9c0125d..8117b646990 100644 --- a/noir-projects/aztec-nr/aztec/src/utils/mod.nr +++ b/noir-projects/aztec-nr/aztec/src/utils/mod.nr @@ -5,3 +5,4 @@ pub mod field; pub mod point; pub mod to_bytes; pub mod secrets; +pub mod with_hash; diff --git a/noir-projects/aztec-nr/aztec/src/utils/with_hash.nr b/noir-projects/aztec-nr/aztec/src/utils/with_hash.nr new file mode 100644 index 00000000000..c9c9bf95c4f --- /dev/null +++ b/noir-projects/aztec-nr/aztec/src/utils/with_hash.nr @@ -0,0 +1,238 @@ +use crate::{ + context::{PublicContext, UnconstrainedContext}, + history::public_storage::PublicStorageHistoricalRead, + oracle, +}; +use dep::protocol_types::{ + address::AztecAddress, block_header::BlockHeader, hash::poseidon2_hash, traits::Packable, +}; + +/// A struct that allows for efficient reading of value `T` from public storage in private. +/// +/// The efficient reads are achieved by verifying large values through a single hash check +/// and then proving inclusion only of the hash in public storage. This reduces the number +/// of required tree inclusion proofs from `N` to 1. +/// +/// # Type Parameters +/// - `T`: The underlying type being wrapped, must implement `Packable` +/// - `N`: The number of field elements required to pack values of type `T` +pub struct WithHash { + value: T, + packed: [Field; N], + hash: Field, +} + +impl WithHash +where + T: Packable + Eq, +{ + pub fn new(value: T) -> Self { + let packed = value.pack(); + Self { value, packed, hash: poseidon2_hash(packed) } + } + + pub fn get_value(self) -> T { + self.value + } + + pub fn get_hash(self) -> Field { + self.hash + } + + pub fn public_storage_read(context: PublicContext, storage_slot: Field) -> T { + context.storage_read(storage_slot) + } + + pub unconstrained fn unconstrained_public_storage_read( + context: UnconstrainedContext, + storage_slot: Field, + ) -> T { + context.storage_read(storage_slot) + } + + pub fn historical_public_storage_read( + header: BlockHeader, + address: AztecAddress, + storage_slot: Field, + ) -> T { + let historical_block_number = header.global_variables.block_number as u32; + + // We could simply produce historical inclusion proofs for each field in `packed`, but that would require one + // full sibling path per storage slot (since due to kernel siloing the storage is not contiguous). Instead, we + // get an oracle to provide us the values, and instead we prove inclusion of their hash, which is both a much + // smaller proof (a single slot), and also independent of the size of T (except in that we need to pack and hash T). + let hint = WithHash::new( + /// Safety: We verify that a hash of the hint/packed data matches the stored hash. + unsafe { + oracle::storage::storage_read(address, storage_slot, historical_block_number) + }, + ); + + let hash = header.public_storage_historical_read(storage_slot + N as Field, address); + + if hash != 0 { + assert_eq(hash, hint.get_hash(), "Hint values do not match hash"); + } else { + // The hash slot can only hold a zero if it is uninitialized. Therefore, the hints must then be zero + // (i.e. the default value for public storage) as well. + assert_eq( + hint.get_value(), + T::unpack(std::mem::zeroed()), + "Non-zero hint for zero hash", + ); + }; + + hint.get_value() + } +} + +impl Packable for WithHash +where + T: Packable, +{ + fn pack(self) -> [Field; N + 1] { + let mut result: [Field; N + 1] = std::mem::zeroed(); + for i in 0..N { + result[i] = self.packed[i]; + } + result[N] = self.hash; + + result + } + + fn unpack(packed: [Field; N + 1]) -> Self { + let mut value_packed: [Field; N] = std::mem::zeroed(); + for i in 0..N { + value_packed[i] = packed[i]; + } + let hash = packed[N]; + + Self { value: T::unpack(value_packed), packed: value_packed, hash } + } +} + +mod test { + use crate::{ + oracle::random::random, + test::{ + helpers::{cheatcodes, test_environment::TestEnvironment}, + mocks::mock_struct::MockStruct, + }, + utils::with_hash::WithHash, + }; + use dep::protocol_types::hash::poseidon2_hash; + use dep::std::{mem, test::OracleMock}; + + global storage_slot: Field = 47; + + #[test] + unconstrained fn create_and_recover() { + let value = MockStruct { a: 5, b: 3 }; + let value_with_hash = WithHash::new(value); + let recovered = WithHash::unpack(value_with_hash.pack()); + + assert_eq(recovered.value, value); + assert_eq(recovered.packed, value.pack()); + assert_eq(recovered.hash, poseidon2_hash(value.pack())); + } + + #[test] + unconstrained fn read_uninitialized_value() { + let mut env = TestEnvironment::new(); + + let block_header = env.private().historical_header; + let address = env.contract_address(); + + let result = WithHash::::historical_public_storage_read( + block_header, + address, + storage_slot, + ); + + // We should get zeroed value + let expected: MockStruct = mem::zeroed(); + assert_eq(result, expected); + } + + #[test] + unconstrained fn read_initialized_value() { + let mut env = TestEnvironment::new(); + + let value = MockStruct { a: 5, b: 3 }; + let value_with_hash = WithHash::new(value); + + // We write the value with hash to storage + cheatcodes::direct_storage_write( + env.contract_address(), + storage_slot, + value_with_hash.pack(), + ); + + // We advance block by 1 because env.private() currently returns context at latest_block - 1 + env.advance_block_by(1); + + let result = WithHash::::historical_public_storage_read( + env.private().historical_header, + env.contract_address(), + storage_slot, + ); + + assert_eq(result, value); + } + + #[test(should_fail_with = "Non-zero hint for zero hash")] + unconstrained fn test_bad_hint_uninitialized_value() { + let mut env = TestEnvironment::new(); + + env.advance_block_to(6); + + let value_packed = MockStruct { a: 1, b: 1 }.pack(); + + let block_header = env.private().historical_header; + let address = env.contract_address(); + + // Mock the oracle to return a non-zero hint/packed value + let _ = OracleMock::mock("storageRead") + .with_params(( + address.to_field(), storage_slot, block_header.global_variables.block_number as u32, + value_packed.len(), + )) + .returns(value_packed) + .times(1); + + // This should revert because the hint value is non-zero and the hash is zero (default value of storage) + let _ = WithHash::::historical_public_storage_read( + block_header, + address, + storage_slot, + ); + } + + #[test(should_fail_with = "Hint values do not match hash")] + unconstrained fn test_bad_hint_initialized_value() { + let mut env = TestEnvironment::new(); + + let value_packed = MockStruct { a: 5, b: 3 }.pack(); + + // We write the value to storage + cheatcodes::direct_storage_write(env.contract_address(), storage_slot, value_packed); + + // Now we write incorrect hash to the hash storage slot + let incorrect_hash = random(); + let hash_storage_slot = storage_slot + (value_packed.len() as Field); + cheatcodes::direct_storage_write( + env.contract_address(), + hash_storage_slot, + [incorrect_hash], + ); + + // We advance block by 1 because env.private() currently returns context at latest_block - 1 + env.advance_block_by(1); + + let _ = WithHash::::historical_public_storage_read( + env.private().historical_header, + env.contract_address(), + storage_slot, + ); + } +} diff --git a/noir-projects/aztec-nr/compressed-string/src/field_compressed_string.nr b/noir-projects/aztec-nr/compressed-string/src/field_compressed_string.nr index 54d21c79a30..6f922cdaa20 100644 --- a/noir-projects/aztec-nr/compressed-string/src/field_compressed_string.nr +++ b/noir-projects/aztec-nr/compressed-string/src/field_compressed_string.nr @@ -6,7 +6,7 @@ use std::meta::derive; // A Fixedsize Compressed String. // Essentially a special version of Compressed String for practical use. -#[derive(Deserialize, Packable, Serialize)] +#[derive(Deserialize, Eq, Packable, Serialize)] pub struct FieldCompressedString { value: Field, } diff --git a/noir-projects/noir-contracts/contracts/amm_contract/src/config.nr b/noir-projects/noir-contracts/contracts/amm_contract/src/config.nr index c18136f4412..fe43bb564c3 100644 --- a/noir-projects/noir-contracts/contracts/amm_contract/src/config.nr +++ b/noir-projects/noir-contracts/contracts/amm_contract/src/config.nr @@ -4,7 +4,7 @@ use std::meta::derive; /// We store the tokens of the pool in a struct such that to load it from SharedImmutable asserts only a single /// merkle proof. /// (Once we actually do the optimization. WIP in https://github.com/AztecProtocol/aztec-packages/pull/8022). -#[derive(Deserialize, Packable, Serialize)] +#[derive(Deserialize, Eq, Packable, Serialize)] pub struct Config { pub token0: AztecAddress, pub token1: AztecAddress, diff --git a/noir-projects/noir-contracts/contracts/app_subscription_contract/src/main.nr b/noir-projects/noir-contracts/contracts/app_subscription_contract/src/main.nr index 454a0a52b94..3639423f2bd 100644 --- a/noir-projects/noir-contracts/contracts/app_subscription_contract/src/main.nr +++ b/noir-projects/noir-contracts/contracts/app_subscription_contract/src/main.nr @@ -18,6 +18,7 @@ pub contract AppSubscription { use router::utils::privately_check_block_number; use token::Token; + // TODO: This can be optimized by storing the values in Config struct in 1 PublicImmutable (less merkle proofs). #[storage] struct Storage { target_address: PublicImmutable, diff --git a/noir-projects/noir-contracts/contracts/claim_contract/src/main.nr b/noir-projects/noir-contracts/contracts/claim_contract/src/main.nr index faa721a531d..c9219815e8a 100644 --- a/noir-projects/noir-contracts/contracts/claim_contract/src/main.nr +++ b/noir-projects/noir-contracts/contracts/claim_contract/src/main.nr @@ -11,6 +11,7 @@ pub contract Claim { use dep::uint_note::uint_note::UintNote; use token::Token; + // TODO: This can be optimized by storing the addresses in Config struct in 1 PublicImmutable (less merkle proofs). #[storage] struct Storage { // Address of a contract based on whose notes we distribute the rewards diff --git a/noir-projects/noir-contracts/contracts/crowdfunding_contract/src/main.nr b/noir-projects/noir-contracts/contracts/crowdfunding_contract/src/main.nr index 4131944789c..bf564557f61 100644 --- a/noir-projects/noir-contracts/contracts/crowdfunding_contract/src/main.nr +++ b/noir-projects/noir-contracts/contracts/crowdfunding_contract/src/main.nr @@ -31,6 +31,7 @@ pub contract Crowdfunding { amount: U128, } + // TODO: This can be optimized by storing the values in Config struct in 1 PublicImmutable (less merkle proofs). // docs:start:storage #[storage] struct Storage { diff --git a/noir-projects/noir-contracts/contracts/docs_example_contract/src/types/leader.nr b/noir-projects/noir-contracts/contracts/docs_example_contract/src/types/leader.nr index b2b352b1986..9e35488bebc 100644 --- a/noir-projects/noir-contracts/contracts/docs_example_contract/src/types/leader.nr +++ b/noir-projects/noir-contracts/contracts/docs_example_contract/src/types/leader.nr @@ -2,7 +2,7 @@ use dep::aztec::protocol_types::{address::AztecAddress, traits::{Deserialize, Pa use std::meta::derive; // Shows how to create a custom struct in Public -#[derive(Deserialize, Packable, Serialize)] +#[derive(Deserialize, Eq, Packable, Serialize)] pub struct Leader { account: AztecAddress, points: u8, diff --git a/noir-projects/noir-contracts/contracts/fpc_contract/src/config.nr b/noir-projects/noir-contracts/contracts/fpc_contract/src/config.nr index cb07cc225f5..fbf1b50eecd 100644 --- a/noir-projects/noir-contracts/contracts/fpc_contract/src/config.nr +++ b/noir-projects/noir-contracts/contracts/fpc_contract/src/config.nr @@ -1,7 +1,7 @@ use dep::aztec::protocol_types::{address::AztecAddress, traits::{Deserialize, Packable, Serialize}}; use std::meta::derive; -#[derive(Deserialize, Packable, Serialize)] +#[derive(Deserialize, Eq, Packable, Serialize)] pub struct Config { pub accepted_asset: AztecAddress, // Asset the FPC accepts (denoted as AA below) pub admin: AztecAddress, // Address to which AA is sent during the private fee payment flow diff --git a/noir-projects/noir-contracts/contracts/fpc_contract/src/main.nr b/noir-projects/noir-contracts/contracts/fpc_contract/src/main.nr index c36130592c9..1bba78878d6 100644 --- a/noir-projects/noir-contracts/contracts/fpc_contract/src/main.nr +++ b/noir-projects/noir-contracts/contracts/fpc_contract/src/main.nr @@ -76,7 +76,6 @@ pub contract FPC { // docs:start:fee_entrypoint_private #[private] fn fee_entrypoint_private(max_fee: U128, nonce: Field) { - // TODO(PR #8022): Once PublicImmutable performs only 1 merkle proof here, we'll save ~4k gates let accepted_asset = storage.config.read().accepted_asset; let user = context.msg_sender(); @@ -150,7 +149,6 @@ pub contract FPC { /// 4. The protocol deducts the actual fee denominated in fee juice from the FPC's balance. #[private] fn fee_entrypoint_public(max_fee: U128, nonce: Field) { - // TODO(PR #8022): Once PublicImmutable performs only 1 merkle proof here, we'll save ~4k gates let config = storage.config.read(); // We pull the max fee from the user's balance of the accepted asset to this contract. @@ -198,7 +196,6 @@ pub contract FPC { /// this function. #[public] fn pull_funds(to: AztecAddress) { - // TODO(PR #8022): Once PublicImmutable performs only 1 merkle proof here, we'll save ~4k gates let config = storage.config.read(); assert(context.msg_sender() == config.admin, "Only admin can pull funds"); diff --git a/noir-projects/noir-contracts/contracts/test_contract/src/main.nr b/noir-projects/noir-contracts/contracts/test_contract/src/main.nr index 1d9c33febf5..a8461de2020 100644 --- a/noir-projects/noir-contracts/contracts/test_contract/src/main.nr +++ b/noir-projects/noir-contracts/contracts/test_contract/src/main.nr @@ -14,7 +14,7 @@ pub contract Test { }; use dep::aztec::prelude::{ AztecAddress, EthAddress, FunctionSelector, Map, NoteGetterOptions, NoteViewerOptions, - PrivateImmutable, PrivateSet, + PrivateImmutable, PrivateSet, PublicImmutable, }; use dep::aztec::protocol_types::{ @@ -77,6 +77,7 @@ pub contract Test { example_constant: PrivateImmutable, example_set: PrivateSet, example_struct: PrivateImmutable, + example_struct_in_public_immutable: PublicImmutable, example_struct_in_map: Map, Context>, another_example_struct: PrivateImmutable, } diff --git a/noir-projects/noir-contracts/contracts/test_contract/src/test.nr b/noir-projects/noir-contracts/contracts/test_contract/src/test.nr index 346ec5a0fc0..2318510695f 100644 --- a/noir-projects/noir-contracts/contracts/test_contract/src/test.nr +++ b/noir-projects/noir-contracts/contracts/test_contract/src/test.nr @@ -25,6 +25,7 @@ unconstrained fn test_storage_slot_allocation() { // example_constant: PrivateImmutable, // example_set: PrivateSet, // example_struct: PrivateImmutable, + // example_struct_in_public_immutable: PublicImmutable, // example_struct_in_map: Map, Context>, // another_example_struct: PrivateImmutable, // } @@ -36,22 +37,27 @@ unconstrained fn test_storage_slot_allocation() { // The first slot is always 1. let mut expected_slot = 1; assert_eq(Test::storage_layout().example_constant.slot, expected_slot); - // Even though example_constant holds TestNote, which packs to a length larger than 1, notes always reserve a // single slot. expected_slot += 1; - assert_eq(Test::storage_layout().example_set.slot, expected_slot); + assert_eq(Test::storage_layout().example_set.slot, expected_slot); // example_set also held a note, so it should have only allocated a single slot. expected_slot += 1; - assert_eq(Test::storage_layout().example_struct.slot, expected_slot); + assert_eq(Test::storage_layout().example_struct.slot, expected_slot); // example_struct allocates 5 slots because it is not a note and it's packed length is 5. expected_slot += 5; - assert_eq(Test::storage_layout().example_struct_in_map.slot, expected_slot); - // example_struct_in_map should allocate a single note because it's a map, regardless of whatever it holds. The Map + assert_eq(Test::storage_layout().example_struct_in_public_immutable.slot, expected_slot); + // example_struct_in_public_immutable should allocate 6 slots because ExampleStruct occupies 5 slots + // and `PublicImmutable` wraps the struct in a `WithHash` that adds an additional slot. + expected_slot += 6; + + assert_eq(Test::storage_layout().example_struct_in_map.slot, expected_slot); + // example_struct_in_map should allocate a single slot because it's a map, regardless of whatever it holds. The Map // type is going to deal with its own dynamic allocation based on keys expected_slot += 1; + assert_eq(Test::storage_layout().another_example_struct.slot, expected_slot); } diff --git a/noir-projects/noir-contracts/contracts/token_bridge_contract/src/main.nr b/noir-projects/noir-contracts/contracts/token_bridge_contract/src/main.nr index 0f080d3e3e1..b7a4a5919f6 100644 --- a/noir-projects/noir-contracts/contracts/token_bridge_contract/src/main.nr +++ b/noir-projects/noir-contracts/contracts/token_bridge_contract/src/main.nr @@ -24,6 +24,7 @@ pub contract TokenBridge { }; // docs:end:token_bridge_imports + // TODO: This can be optimized by storing the values in Config struct in 1 PublicImmutable (less merkle proofs). // docs:start:token_bridge_storage_and_constructor // Storage structure, containing all storage, and specifying what slots they use. #[storage] diff --git a/yarn-project/foundation/src/fields/fields.ts b/yarn-project/foundation/src/fields/fields.ts index fbecce8f06a..594a9bf981d 100644 --- a/yarn-project/foundation/src/fields/fields.ts +++ b/yarn-project/foundation/src/fields/fields.ts @@ -261,7 +261,7 @@ export class Fr extends BaseField { return fromHexString(buf, Fr); } - throw new Error('Tried to create a Fr from an invalid string'); + throw new Error(`Tried to create a Fr from an invalid string: ${buf}`); } /** @@ -412,7 +412,7 @@ export class Fq extends BaseField { return fromHexString(buf, Fq); } - throw new Error('Tried to create a Fq from an invalid string'); + throw new Error(`Tried to create a Fq from an invalid string: ${buf}`); } /** diff --git a/yarn-project/types/src/abi/contract_artifact.ts b/yarn-project/types/src/abi/contract_artifact.ts index 2182451baf4..50410b0a233 100644 --- a/yarn-project/types/src/abi/contract_artifact.ts +++ b/yarn-project/types/src/abi/contract_artifact.ts @@ -220,7 +220,7 @@ function getStorageLayout(input: NoirCompiledContract) { const name = field.name; const slot = field.value.fields[0].value as IntegerValue; acc[name] = { - slot: slot.value, + slot: `0x${slot.value}`, }; return acc; }, {});