feat(polka-storage-provider-server): implement config file (#674)

eigerco · Jan 14, 2025 · 498840d · 498840d
1 parent 31533cd
commit 498840d
Show file tree

Hide file tree

Showing 8 changed files with 265 additions and 143 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -110,6 +110,7 @@ thiserror = { version = "2.0.3", default-features = false }
 tokio = "1.37.0"
 tokio-stream = "0.1.15"
 tokio-util = "0.7.11"
+toml = "0.8.19"
 tower = "0.4.13"
 tower-http = "0.5.2"
 tracing = "0.1.40"

diff --git a/docs/src/storage-provider-cli/server.md b/docs/src/storage-provider-cli/server.md
@@ -4,27 +4,6 @@ This chapter covers the available CLI options for the Polka Storage Provider ser
 
 <!-- Sadly, tables will not cut it here, since the text is just too big for the table. -->
 
-#### `--upload-listen-address`
-
-The storage server's endpoint address — i.e. where the client will upload their files to.
-
-It takes in an IP address along with a port in the format: `<ip>:<port>`.
-Defaults to `127.0.0.1:8001`.
-
-#### `--rpc-listen-address`
-
-The RPC server endpoint's address — i.e. where you will submit your deals to.
-
-It takes in an IP address along with a port in the format: `<ip>:<port>`.
-Defaults to `127.0.0.1:8000`.
-
-#### `--node-url`
-
-The target parachain node's address — i.e. the parachain node the storage provider will submit deals to, etc.
-
-It takes in an URL, it supports both HTTP and WebSockets and their secure variants.
-Defaults to `ws://127.0.0.1:42069`.
-
 ### `--sr25519-key`
 
 Sr25519 keypair, encoded as hex, BIP-39 or a dev phrase like `//Alice`.
@@ -49,6 +28,27 @@ See [`sp_core::crypto::Pair::from_string_with_seed`](https://docs.rs/sp-core/lat
 
 If this `--ed25519-key` is not used, either [`--ecdsa-key`](#--ecdsa-key) or [`--sr25519-key`](#--sr25519-key) MUST be used.
 
+### `--upload-listen-address`
+
+The storage server's endpoint address — i.e. where the client will upload their files to.
+
+It takes in an IP address along with a port in the format: `<ip>:<port>`.
+Defaults to `127.0.0.1:8001`.
+
+### `--rpc-listen-address`
+
+The RPC server endpoint's address — i.e. where you will submit your deals to.
+
+It takes in an IP address along with a port in the format: `<ip>:<port>`.
+Defaults to `127.0.0.1:8000`.
+
+### `--node-url`
+
+The target parachain node's address — i.e. the parachain node the storage provider will submit deals to, etc.
+
+It takes in an URL, it supports both HTTP and WebSockets and their secure variants.
+Defaults to `ws://127.0.0.1:42069`.
+
 ### `--database-directory`
 
 The RocksDB storage directory, where deal information will be kept.
@@ -72,3 +72,37 @@ The kind of replication proof. Currently, only `StackedDRGWindow2KiBV1P1` is sup
 ### `--post-proof`
 
 The kind of storage proof. Currently, only `StackedDRGWindow2KiBV1P1` is supported to which it defaults.
+
+### `--porep-parameters`
+
+The path to the PoRep proving parameters. They are shared across all of the nodes in the network, as the chain stores corresponding Verifying Key parameters.
+
+### `--post-parameters`
+
+The path to the PoSt proving parameters. They are shared across all of the nodes in the network, as the chain stores corresponding Verifying Key parameters.
+
+### `--config`
+
+Takes in a path to a configuration file, it supports both JSON and TOML (files _must_ have the right extension).
+The supported configuration parameters are:
+
+| Name                    | Default                        |
+| ----------------------- | ------------------------------ |
+| `upload-listen-address` | `127.0.0.1:8000`               |
+| `rpc-listen-address`    | `127.0.0.1:8001`               |
+| `node-url`              | `ws://127.0.0.1:42069`         |
+| `database-directory`    | `/tmp/<random>/deals_database` |
+| `storage-directory`     | `/tmp/<random>/deals_storage`  |
+| `seal-proof`            | 2KiB                           |
+| `post-proof`            | 2KiB                           |
+| `porep_parameters`      | NA                             |
+| `post_parameters`       | NA                             |
+
+#### Bare bones configuration
+
+```json
+{
+    "porep_parameters": "/home/storage_provider/porep.params",
+    "post_parameters": "/home/storage_provider/post.params",
+}
+```
diff --git a/examples/start_sp.sh b/examples/start_sp.sh
@@ -24,4 +24,12 @@ RUST_LOG=debug target/release/storagext-cli --sr25519-key "//Alice" proofs set-p
 RUST_LOG=debug target/release/storagext-cli --sr25519-key "//Bob" proofs set-post-verifying-key @2KiB.post.vk.scale &
 wait
 
-RUST_LOG=debug target/release/polka-storage-provider-server --sr25519-key "$PROVIDER" --seal-proof "2KiB" --post-proof "2KiB" --porep-parameters 2KiB.porep.params --post-parameters 2KiB.post.params
+echo '{
+    "seal_proof": "2KiB",
+    "post_proof": "2KiB",
+    "porep_parameters": "2KiB.porep.params",
+    "post_parameters": "2KiB.post.params"
+}' > /tmp/storage_provider.config.json
+RUST_LOG=debug target/release/polka-storage-provider-server \
+    --sr25519-key "$PROVIDER" \
+    --config /tmp/storage_provider.config.json
diff --git a/primitives/src/proofs.rs b/primitives/src/proofs.rs
@@ -57,7 +57,9 @@ pub enum RegisteredSealProof {
 
 impl RegisteredSealProof {
     pub fn sector_size(&self) -> SectorSize {
-        SectorSize::_2KiB
+        match self {
+            RegisteredSealProof::StackedDRG2KiBV1P1 => SectorSize::_2KiB,
+        }
     }
 
     /// Produces the windowed PoSt-specific RegisteredProof corresponding
@@ -79,6 +81,14 @@ impl RegisteredSealProof {
             RegisteredSealProof::StackedDRG2KiBV1P1 => 192,
         }
     }
+
+    /// Returns [`StackedDRG2KiBV1P1`](RegisteredSealProof::StackedDRG2KiBV1P1).
+    // NOTE(@jmg-duarte,14/01/2025): wanted to avoid setting a default to use in serde
+    // this is the alternative
+    #[allow(non_snake_case)]
+    pub const fn _2KiB() -> Self {
+        Self::StackedDRG2KiBV1P1
+    }
 }
 
 /// Proof of Spacetime type, indicating version and sector size of the proof.
@@ -122,6 +132,14 @@ impl RegisteredPoStProof {
             RegisteredPoStProof::StackedDRGWindow2KiBV1P1 => 2,
         }
     }
+
+    /// Returns [`StackedDRGWindow2KiBV1P1`](RegisteredPoStProof::StackedDRGWindow2KiBV1P1).
+    // NOTE(@jmg-duarte,14/01/2025): wanted to avoid setting a default to use in serde
+    // this is the alternative
+    #[allow(non_snake_case)]
+    pub const fn _2KiB() -> Self {
+        Self::StackedDRGWindow2KiBV1P1
+    }
 }
 
 // serde_json requires std, hence, to test the serialization, we need:

diff --git a/storage-provider/server/Cargo.toml b/storage-provider/server/Cargo.toml
@@ -38,12 +38,13 @@ tempfile = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }
 tokio-util = { workspace = true, features = ["rt"] }
+toml = { workspace = true }
 tower = { workspace = true }
 tower-http = { workspace = true, features = ["trace"] }
 tracing = { workspace = true }
 tracing-appender = { workspace = true }
 tracing-subscriber = { workspace = true, features = ["env-filter"] }
-url = { workspace = true }
+url = { workspace = true, features = ["serde"] }
 uuid = { workspace = true, features = ["v4"] }
 
 [lints]

diff --git a/storage-provider/server/src/config.rs b/storage-provider/server/src/config.rs
@@ -0,0 +1,101 @@
+use std::{
+    net::{IpAddr, Ipv4Addr, SocketAddr},
+    num::NonZero,
+    path::PathBuf,
+};
+
+use clap::Args;
+use primitives::proofs::{RegisteredPoStProof, RegisteredSealProof};
+use serde::{Deserialize, Serialize};
+use url::Url;
+
+use crate::DEFAULT_NODE_ADDRESS;
+
+/// Default address to bind the RPC server to.
+const fn default_rpc_listen_address() -> SocketAddr {
+    SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8000)
+}
+
+/// Default address to bind the RPC server to.
+const fn default_upload_listen_address() -> SocketAddr {
+    SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 8001)
+}
+
+/// Default number of parallel prove commits.
+const fn default_parallel_prove_commits() -> NonZero<usize> {
+    // SAFETY: 2 != 0
+    unsafe { NonZero::new_unchecked(2) }
+}
+
+fn default_node_address() -> Url {
+    Url::parse(DEFAULT_NODE_ADDRESS).expect("DEFAULT_NODE_ADDRESS must be a valid Url")
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, Args)]
+#[group(multiple = true, conflicts_with = "config")]
+#[serde(deny_unknown_fields)]
+pub struct ConfigurationArgs {
+    /// The server's listen address.
+    #[serde(default = "default_upload_listen_address")]
+    #[arg(long, default_value_t = default_upload_listen_address())]
+    pub(crate) upload_listen_address: SocketAddr,
+
+    /// The server's listen address.
+    #[serde(default = "default_rpc_listen_address")]
+    #[arg(long, default_value_t = default_rpc_listen_address())]
+    pub(crate) rpc_listen_address: SocketAddr,
+
+    /// The target parachain node's address.
+    #[serde(default = "default_node_address")]
+    #[arg(long, default_value_t = default_node_address())]
+    pub(crate) node_url: Url,
+
+    /// RocksDB storage directory.
+    /// Defaults to a temporary random directory, like `/tmp/<random>/deals_database`.
+    #[arg(long)]
+    pub(crate) database_directory: Option<PathBuf>,
+
+    /// Piece storage directory.
+    /// Defaults to a temporary random directory, like `/tmp/<random>/...`.
+    #[arg(long)]
+    pub(crate) storage_directory: Option<PathBuf>,
+
+    /// The number of prove commits to be run in parallel.
+    /// MUST BE > 0 or the pipeline will not progress.
+    ///
+    /// Creating a replica is memory-heavy process.
+    /// E.g. With 2KiB sector sizes and 16GiB of RAM, it goes OOM at 4 parallel.
+    #[serde(default = "default_parallel_prove_commits")]
+    #[arg(long, default_value_t = default_parallel_prove_commits())]
+    pub(crate) parallel_prove_commits: NonZero<usize>,
+
+    // NOTE: the following parameters are marked as "not required" so the CLI doesn't require them
+    // when --config is used, otherwise, they're very much required
+    /// Proof of Replication proof type.
+    #[serde(default = "RegisteredSealProof::_2KiB")]
+    #[arg(long, required = false)]
+    pub(crate) seal_proof: RegisteredSealProof,
+
+    /// Proof of Spacetime proof type.
+    #[serde(default = "RegisteredPoStProof::_2KiB")]
+    #[arg(long, required = false)]
+    pub(crate) post_proof: RegisteredPoStProof,
+
+    /// Proving Parameters for PoRep proof, corresponding to given `seal_proof` sector size.
+    /// They are shared across all of the nodes in the network, as the chain stores corresponding Verifying Key parameters.
+    ///
+    /// Testing/temporary parameters can be generated via `polka-storage-provider-client proofs porep-params` command.
+    /// Note that when you generate keys, for local testnet,
+    /// **they need to be set** via an extrinsic pallet-proofs::set_porep_verifyingkey.
+    #[arg(long, required = false)]
+    pub(crate) porep_parameters: PathBuf,
+
+    /// Proving Parameters for PoSt proof, corresponding to given `post_proof` sector size.
+    /// They are shared across all of the nodes in the network, as the chain stores corresponding Verifying Key parameters.
+    ///
+    /// Testing/temporary parameters can be generated via `polka-storage-provider-client proofs post-params` command.
+    /// Note that when you generate keys, for local testnet,
+    /// **they need to be set** via an extrinsic pallet-proofs::set_post_verifyingkey.
+    #[arg(long, required = false)]
+    pub(crate) post_parameters: PathBuf,
+}