[release/4.x] Cherry pick: Replace Secret Sharing implementation (#5655…

…) (#5675)

.daily_canary

@@ -1,4 +1,4 @@
    -^-     ___           ___
   (- -)   (= =)   |     Y   &  +--?
  (  V  ) /  .  \  |    +---=---'
-/--x-m- /--n-n---xXx--/--yY------
+/--x-m- /--n-n---xXx--/--yY------>>>+++<<<

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [4.0.9]
+
+[4.0.9]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.9
+
+- Secret sharing used for ledger recovery now relies on a much simpler implementation that requires no external dependencies. Note that while the code still accepts shares generated by the old code for now, it only generates shares with the new implementation. As a result, a DR attempt that would downgrade the code to a version that pre-dates this change, after having previously picked it up, would not succeed if a reshare had already taken place (#5655).
+
 ## [4.0.8]
 
 [4.0.8]: https://github.com/microsoft/CCF/releases/tag/ccf-4.0.8

CMakeLists.txt

@@ -912,6 +912,13 @@ if(BUILD_TESTS)
     target_include_directories(crypto_test PRIVATE ${CCFCRYPTO_INC})
     target_link_libraries(crypto_test PRIVATE ccfcrypto.host)
 
+    add_unit_test(
+      sharing_test
+      ${CMAKE_CURRENT_SOURCE_DIR}/src/crypto/test/secret_sharing.cpp
+    )
+    target_include_directories(sharing_test PRIVATE ${CCFCRYPTO_INC})
+    target_link_libraries(sharing_test PRIVATE ccfcrypto.host)
+
     add_unit_test(
       key_exchange_test
       ${CMAKE_CURRENT_SOURCE_DIR}/src/crypto/test/key_exchange.cpp

cmake/crypto.cmake

@@ -24,6 +24,7 @@ set(CCFCRYPTO_SRC
     ${CCF_DIR}/src/crypto/openssl/rsa_key_pair.cpp
     ${CCF_DIR}/src/crypto/openssl/verifier.cpp
     ${CCF_DIR}/src/crypto/openssl/cose_verifier.cpp
+    ${CCF_DIR}/src/crypto/sharing.cpp
 )
 
 if(COMPILE_TARGET STREQUAL "sgx")

doc/governance/adding_member.rst

@@ -3,8 +3,6 @@ Adding New Members
 
 It is possible for existing members to add new members to the consortium after a CCF network has been started.
 
-.. note:: The maximum number of allowed active recovery members (i.e. those with a recovery share) at any given time is 255.
-
 Generating Member Keys and Certificates
 ---------------------------------------
 

doc/operations/recovery.rst

@@ -7,6 +7,8 @@ The disaster recovery procedure is costly (e.g. the service identity certificate
 
 .. tip:: See :ccf_repo:`tests/infra/health_watcher.py` for an example of how a network can be monitored to detect a disaster recovery scenario.
 
+.. note:: From 4.0.9/5.0.0-dev2 onwards secret sharing used for ledger recovery now relies on a much simpler implementation that requires no external dependencies. Note that while the code still accepts shares generated by the old code for now, it only generates shares with the new implementation. As a result, a DR attempt that would downgrade the code to a version that pre-dates this change, after having previously picked it up, would not succeed if a reshare had already taken place.
+
 Overview
 --------
 

include/ccf/crypto/hkdf.h

@@ -4,6 +4,7 @@
 
 #include "ccf/crypto/md_type.h"
 
+#include <span>
 #include <vector>
 
 namespace crypto
@@ -12,7 +13,7 @@ namespace crypto
   std::vector<uint8_t> hkdf(
     MDType md_type,
     size_t length,
-    const std::vector<uint8_t>& ikm,
-    const std::vector<uint8_t>& salt = {},
-    const std::vector<uint8_t>& info = {});
+    const std::span<const uint8_t>& ikm,
+    const std::span<const uint8_t>& salt = {},
+    const std::span<const uint8_t>& info = {});
 }

include/ccf/crypto/sha256.h

@@ -10,12 +10,16 @@ namespace crypto
 {
   /** Compute the SHA256 hash of @p data
    * @param data The data to compute the hash of
+   *
+   * @return hashed value
    */
-  HashBytes sha256(const std::vector<uint8_t>& data);
+  HashBytes sha256(const std::span<uint8_t const>& data);
 
   /** Compute the SHA256 hash of @p data
    * @param data The data to compute the hash of
    * @param len Length of the data
+   *
+   * @return hashed value
    */
   HashBytes sha256(const uint8_t* data, size_t len);
 }

src/crypto/hash.cpp

@@ -21,6 +21,14 @@ namespace crypto
     return r;
   }
 
+  std::vector<uint8_t> sha256(const std::span<uint8_t const>& data)
+  {
+    size_t hash_size = EVP_MD_size(OpenSSL::get_md_type(MDType::SHA256));
+    std::vector<uint8_t> r(hash_size);
+    openssl_sha256(data, r.data());
+    return r;
+  }
+
   std::vector<uint8_t> sha256(const uint8_t* data, size_t len)
   {
     std::span<const uint8_t> buf(data, len);
@@ -43,9 +51,9 @@ namespace crypto
   std::vector<uint8_t> hkdf(
     MDType md_type,
     size_t length,
-    const std::vector<uint8_t>& ikm,
-    const std::vector<uint8_t>& salt,
-    const std::vector<uint8_t>& info)
+    const std::span<const uint8_t>& ikm,
+    const std::span<const uint8_t>& salt,
+    const std::span<const uint8_t>& info)
   {
     return OpenSSL::hkdf(md_type, length, ikm, salt, info);
   }

src/crypto/openssl/hash.cpp

@@ -14,9 +14,9 @@ namespace crypto
     std::vector<uint8_t> hkdf(
       MDType md_type,
       size_t length,
-      const std::vector<uint8_t>& ikm,
-      const std::vector<uint8_t>& salt,
-      const std::vector<uint8_t>& info)
+      const std::span<const uint8_t>& ikm,
+      const std::span<const uint8_t>& salt,
+      const std::span<const uint8_t>& info)
     {
       auto md = get_md_type(md_type);
       EVP_PKEY_CTX* pctx;

src/crypto/openssl/hash.h

@@ -7,6 +7,7 @@
 
 #include <openssl/evp.h>
 #include <openssl/kdf.h>
+#include <span>
 
 #define FMT_HEADER_ONLY
 #include <fmt/format.h>
@@ -38,9 +39,9 @@ namespace crypto
     std::vector<uint8_t> hkdf(
       MDType md_type,
       size_t length,
-      const std::vector<uint8_t>& ikm,
-      const std::vector<uint8_t>& salt = {},
-      const std::vector<uint8_t>& info = {});
+      const std::span<const uint8_t>& ikm,
+      const std::span<const uint8_t>& salt = {},
+      const std::span<const uint8_t>& info = {});
   }
 
   // Hash Provider (OpenSSL)

src/crypto/sharing.cpp

@@ -0,0 +1,184 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#include "sharing.h"
+
+#include "ccf/crypto/entropy.h"
+
+#include <stdexcept>
+
+namespace crypto
+{
+  /* PRIME FIELD
+
+     For simplicity, we use a finite field F[prime] where all operations
+     are defined in plain uint64_t arithmetic, and we reduce after every
+     operation. This is not meant to be efficient. Compared to e.g. GF(2^n), the
+     main drawback is that we need to hash the "raw secret" to obtain a
+     uniformly-distributed secret.
+  */
+
+  using element = uint64_t;
+  constexpr element prime = (1ul << 31) - 1ul; // a notorious Mersenne prime
+
+  static element reduce(uint64_t x)
+  {
+    return (x % prime);
+  }
+
+  static element mul(element x, element y)
+  {
+    return ((x * y) % prime);
+  }
+
+  static element add(element x, element y)
+  {
+    return ((x + y) % prime);
+  }
+
+  static element sub(element x, element y)
+  {
+    return ((prime + x - y)) % prime;
+  }
+
+  // naive algorithm, used only to compute coefficients, not for use on secrets!
+  static element exp(element x, size_t n)
+  {
+    element y = 1;
+    while (n > 0)
+    {
+      if (n & 1)
+        y = mul(y, x);
+      x = mul(x, x);
+      n >>= 1;
+    }
+    return y;
+  }
+
+  static element inv(element x)
+  {
+    if (x == 0)
+    {
+      throw std::invalid_argument("division by zero");
+    }
+    return exp(x, prime - 2);
+  }
+
+  // This function is specific to prime=2^31-1.
+  // We assume the lower 31 bits are uniformly distributed,
+  // and retry if they are all set to get uniformity in F[prime].
+
+  static element sample(const crypto::EntropyPtr& entropy)
+  {
+    uint64_t res = prime;
+    while (res == prime)
+    {
+      res = entropy->random64() & prime;
+    }
+    return res;
+  }
+
+  /* POLYNOMIAL SHARING AND INTERPOLATION */
+
+  static void sample_polynomial(
+    element p[], size_t degree, const crypto::EntropyPtr& entropy)
+  {
+    for (size_t i = 0; i <= degree; i++)
+    {
+      p[i] = sample(entropy);
+    }
+  }
+
+  static element eval(element p[], size_t degree, element x)
+  {
+    element y = 0, x_i = 1;
+    for (size_t i = 0; i <= degree; i++)
+    {
+      // x_i == x^i
+      y = add(y, mul(p[i], x_i));
+      x_i = mul(x, x_i);
+    }
+    return y;
+  }
+
+  void sample_secret_and_shares(
+    Share& raw_secret, const std::span<Share>& shares, size_t threshold)
+  {
+    if (shares.size() < 1)
+    {
+      throw std::invalid_argument("insufficient number of shares");
+    }
+
+    if (threshold < 1 || threshold > shares.size())
+    {
+      throw std::invalid_argument("invalid threshold");
+    }
+
+    size_t degree = threshold - 1;
+
+    raw_secret.x = 0;
+    for (size_t s = 0; s < shares.size(); s++)
+    {
+      shares[s].x = s + 1;
+    }
+
+    auto entropy = crypto::create_entropy();
+
+    for (size_t limb = 0; limb < LIMBS; limb++)
+    {
+      element p[degree + 1]; /*SECRET*/
+      sample_polynomial(p, degree, entropy);
+      raw_secret.y[limb] = p[0];
+      for (size_t s = 0; s < shares.size(); s++)
+      {
+        shares[s].y[limb] = eval(p, degree, shares[s].x);
+      }
+    }
+  }
+
+  void recover_unauthenticated_secret(
+    Share& raw_secret, const std::span<Share const>& shares, size_t threshold)
+  {
+    if (shares.size() < threshold)
+    {
+      throw std::invalid_argument("insufficient input shares");
+    }
+    // We systematically reduce the input shares instead of checking they are
+    // well-formed.
+
+    size_t degree = threshold - 1;
+
+    // Precomputes Lagrange coefficients for interpolating p(0). No secrets
+    // involved.
+    element lagrange[degree + 1];
+    for (size_t i = 0; i <= degree; i++)
+    {
+      element numerator = 1, denominator = 1;
+      for (size_t j = 0; j <= degree; j++)
+      {
+        if (i != j)
+        {
+          numerator = mul(numerator, reduce(shares[j].x));
+          denominator =
+            mul(denominator, sub(reduce(shares[j].x), reduce(shares[i].x)));
+        }
+      }
+      if (denominator == 0)
+      {
+        throw std::invalid_argument("duplicate input share");
+      }
+      lagrange[i] = mul(numerator, inv(denominator));
+    }
+
+    // Interpolate every limb of the secret. Constant-time on y values.
+    raw_secret.x = 0;
+    for (size_t limb = 0; limb < LIMBS; limb++)
+    {
+      element y = 0;
+      for (size_t i = 0; i <= degree; i++)
+      {
+        y = add(y, mul(lagrange[i], reduce(shares[i].y[limb])));
+      }
+      raw_secret.y[limb] = y;
+    }
+  }
+}