From 8e787f04610f036bc95f90bc43f94df71eb35d6a Mon Sep 17 00:00:00 2001 From: JayT106 Date: Wed, 21 Sep 2022 04:18:05 -0400 Subject: [PATCH 1/4] crypto/merkle: optimize merkle tree hashing (#6513) (#9446) * crypto/merkle: optimize merkle tree hashing (#6513) Upstream https://github.com/lazyledger/lazyledger-core/pull/351 to optimize merkle tree hashing ``` benchmark old ns/op new ns/op delta BenchmarkHashAlternatives/recursive-8 22914 21949 -4.21% BenchmarkHashAlternatives/iterative-8 21634 21939 +1.41% benchmark old allocs new allocs delta BenchmarkHashAlternatives/recursive-8 398 200 -49.75% BenchmarkHashAlternatives/iterative-8 399 301 -24.56% benchmark old bytes new bytes delta BenchmarkHashAlternatives/recursive-8 19088 6496 -65.97% BenchmarkHashAlternatives/iterative-8 21776 13984 -35.78% ``` cc @odeke-em @cuonglm * update pending log Co-authored-by: Marko --- crypto/merkle/hash.go | 18 ++++++++++++++++++ crypto/merkle/proof.go | 3 --- crypto/merkle/proof_key_path_test.go | 1 + crypto/merkle/proof_test.go | 2 +- crypto/merkle/tree.go | 18 ++++++++++++------ 5 files changed, 32 insertions(+), 10 deletions(-) diff --git a/crypto/merkle/hash.go b/crypto/merkle/hash.go index e5f731855f2..fc3c6a29361 100644 --- a/crypto/merkle/hash.go +++ b/crypto/merkle/hash.go @@ -1,6 +1,8 @@ package merkle import ( + "hash" + "github.com/cometbft/cometbft/crypto/tmhash" ) @@ -20,7 +22,23 @@ func leafHash(leaf []byte) []byte { return tmhash.Sum(append(leafPrefix, leaf...)) } +// returns tmhash(0x00 || leaf) +func leafHashOpt(s hash.Hash, leaf []byte) []byte { + s.Reset() + s.Write(leafPrefix) + s.Write(leaf) + return s.Sum(nil) +} + // returns tmhash(0x01 || left || right) func innerHash(left []byte, right []byte) []byte { return tmhash.Sum(append(innerPrefix, append(left, right...)...)) } + +func innerHashOpt(s hash.Hash, left []byte, right []byte) []byte { + s.Reset() + s.Write(innerPrefix) + s.Write(left) + s.Write(right) + return s.Sum(nil) +} diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index 85b2db1e91a..2f083499d08 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -50,9 +50,6 @@ func ProofsFromByteSlices(items [][]byte) (rootHash []byte, proofs []*Proof) { // Verify that the Proof proves the root hash. // Check sp.Index/sp.Total manually if needed func (sp *Proof) Verify(rootHash []byte, leaf []byte) error { - if rootHash == nil { - return fmt.Errorf("invalid root hash: cannot be nil") - } if sp.Total < 0 { return errors.New("proof total must be positive") } diff --git a/crypto/merkle/proof_key_path_test.go b/crypto/merkle/proof_key_path_test.go index 25a61af9291..0d6d3354d33 100644 --- a/crypto/merkle/proof_key_path_test.go +++ b/crypto/merkle/proof_key_path_test.go @@ -36,6 +36,7 @@ func TestKeyPath(t *testing.T) { res, err := KeyPathToKeys(path.String()) require.Nil(t, err) + require.Equal(t, len(keys), len(res)) for i, key := range keys { require.Equal(t, key, res[i]) diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index 45d565e40c8..f307380aad6 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -173,12 +173,12 @@ func TestProofValidateBasic(t *testing.T) { } } func TestVoteProtobuf(t *testing.T) { - _, proofs := ProofsFromByteSlices([][]byte{ []byte("apple"), []byte("watermelon"), []byte("kiwi"), }) + testCases := []struct { testName string v1 *Proof diff --git a/crypto/merkle/tree.go b/crypto/merkle/tree.go index 089c2f82ee0..896b67c5952 100644 --- a/crypto/merkle/tree.go +++ b/crypto/merkle/tree.go @@ -1,22 +1,28 @@ package merkle import ( + "crypto/sha256" + "hash" "math/bits" ) // HashFromByteSlices computes a Merkle tree where the leaves are the byte slice, // in the provided order. It follows RFC-6962. func HashFromByteSlices(items [][]byte) []byte { + return hashFromByteSlices(sha256.New(), items) +} + +func hashFromByteSlices(sha hash.Hash, items [][]byte) []byte { switch len(items) { case 0: return emptyHash() case 1: - return leafHash(items[0]) + return leafHashOpt(sha, items[0]) default: k := getSplitPoint(int64(len(items))) - left := HashFromByteSlices(items[:k]) - right := HashFromByteSlices(items[k:]) - return innerHash(left, right) + left := hashFromByteSlices(sha, items[:k]) + right := hashFromByteSlices(sha, items[k:]) + return innerHashOpt(sha, left, right) } } @@ -61,7 +67,7 @@ func HashFromByteSlices(items [][]byte) []byte { // implementation for so little benefit. func HashFromByteSlicesIterative(input [][]byte) []byte { items := make([][]byte, len(input)) - + sha := sha256.New() for i, leaf := range input { items[i] = leafHash(leaf) } @@ -78,7 +84,7 @@ func HashFromByteSlicesIterative(input [][]byte) []byte { wp := 0 // write position for rp < size { if rp+1 < size { - items[wp] = innerHash(items[rp], items[rp+1]) + items[wp] = innerHashOpt(sha, items[rp], items[rp+1]) rp += 2 } else { items[wp] = items[rp] From 9e015e05f1d5daa5d67773ebc30b017076591d86 Mon Sep 17 00:00:00 2001 From: Adam Tucker Date: Mon, 22 Apr 2024 21:43:28 -0600 Subject: [PATCH 2/4] add changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9b01dfa8f2..4a79f9973bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * [#21](https://github.com/osmosis-labs/cometbft/pull/21) Move websocket logs to Debug * [#22](https://github.com/osmosis-labs/cometbft/pull/22) Fix txSearch pagination performance issues +* [#25](https://github.com/osmosis-labs/cometbft/pull/25) Optimize merkle tree hashing ## v0.37.4-v24-osmo-2 From 4b602f88299310f1777a44e0a0582d1a53dbb7e2 Mon Sep 17 00:00:00 2001 From: Adam Tucker Date: Mon, 22 Apr 2024 22:13:09 -0600 Subject: [PATCH 3/4] add back rootHash nil check --- crypto/merkle/proof.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index 2f083499d08..85b2db1e91a 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -50,6 +50,9 @@ func ProofsFromByteSlices(items [][]byte) (rootHash []byte, proofs []*Proof) { // Verify that the Proof proves the root hash. // Check sp.Index/sp.Total manually if needed func (sp *Proof) Verify(rootHash []byte, leaf []byte) error { + if rootHash == nil { + return fmt.Errorf("invalid root hash: cannot be nil") + } if sp.Total < 0 { return errors.New("proof total must be positive") } From 6b1d05f96223c067b530b681e7ac9c00fcdff904 Mon Sep 17 00:00:00 2001 From: Adam Tucker Date: Tue, 23 Apr 2024 20:17:19 -0600 Subject: [PATCH 4/4] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a79f9973bf..ea3d53f86c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +## v0.37.4-v24-osmo-3 + * [#21](https://github.com/osmosis-labs/cometbft/pull/21) Move websocket logs to Debug * [#22](https://github.com/osmosis-labs/cometbft/pull/22) Fix txSearch pagination performance issues * [#25](https://github.com/osmosis-labs/cometbft/pull/25) Optimize merkle tree hashing