refactor: actors builder

Cargo.toml

@@ -36,6 +36,8 @@ members = [
   "fendermint/tracing",
   "fendermint/vm/*",
   "fendermint/actors",
+  "fendermint/actors-custom-car",
+  "fendermint/actors-builtin-car",
   "fendermint/actors/api",
   "fendermint/actors/chainmetadata",
   "fendermint/actors/activity-tracker",
@@ -69,6 +71,7 @@ blake2b_simd = "1.0"
 bloom = "0.3"
 bytes = "1.4"
 clap = { version = "4.1", features = ["derive", "env", "string"] }
+color-eyre = "0.5.11"
 byteorder = "1.5.0"
 config = "0.13"
 dirs = "5.0"

fendermint/actors-builtin-car/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "actors-builtin-fetch"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+license-file.workspace = true
+
+[build-dependencies]
+reqwest = { version = "0.12", features = ["rustls-tls", "stream"] }
+tokio = { version = "1", features = ["full"] }
+fs-err = { workspace = true }
+sha2 = { workspace = true }
+tempfile = { workspace = true }
+color-eyre = { workspace = true }
+futures-util = { workspace = true }
+bytes = { workspace = true }

fendermint/actors-builtin-car/build.rs

@@ -0,0 +1,155 @@
+// Copyright 2022-2024 Protocol Labs
+// SPDX-License-Identifier: Apache-2.0, MIT
+//! Download filecoin's builtin actors car file
+
+use bytes::buf::Buf;
+use color_eyre::eyre::{self, bail, Result, WrapErr};
+use fs_err as fs;
+use futures_util::stream::StreamExt;
+use sha2::Digest;
+use std::io::{Read, Write};
+use std::path::Path;
+use tempfile::NamedTempFile;
+
+const BUILTIN_ACTORS_TAG: &str = "v15.0.0";
+
+const FORCE_RERUN: &str = "IPC_BUILTIN_ACTORS_FORCE_FETCH";
+
+const VERSION_OVERRIDE: &str = "IPC_BUILTIN_ACTORS_VERSION_OVERRIDE";
+
+/// Handle `Interrupt` errors, call a closure for each read kb piece
+fn read_file_piecewise<R: Read, F: FnMut(&[u8]) -> Result<()>>(
+    mut reader: R,
+    mut f: F,
+) -> Result<()> {
+    let mut buf = [0u8; 1024];
+    loop {
+        match reader.read(&mut buf[..]) {
+            Ok(0) => {
+                return Ok(());
+            }
+            Ok(n) => {
+                f(&buf[..n])?;
+            }
+            Err(e) => match e.kind() {
+                std::io::ErrorKind::Interrupted => continue,
+                _ => return Err(e.into()),
+            },
+        }
+    }
+}
+
+/// Calculate the digest of a file
+fn file_digest(reader: impl Read) -> Result<Checksum> {
+    let mut sha = sha2::Sha256::default();
+    read_file_piecewise(reader, |bytes| {
+        sha.write_all(bytes)?;
+        Ok(())
+    })?;
+    let digest = sha.finalize();
+    Checksum::try_from(digest.as_slice())
+}
+
+/// Convert a slice to an array
+///
+/// The slice must have the exact length of the array size `N`.
+fn slice_to_array<const N: usize>(bytes: &[u8]) -> Result<[u8; N]> {
+    if bytes.len() != N {
+        bail!("Length mismatch, execpted {}, but got {}", N, bytes.len());
+    }
+    let mut buf = [0u8; N];
+    buf.copy_from_slice(&bytes[..N]);
+    Ok(buf)
+}
+
+/// Digest wrapping type
+#[derive(Debug, PartialEq, Eq, Hash)]
+struct Checksum([u8; 32]);
+
+impl TryFrom<&[u8]> for Checksum {
+    type Error = eyre::Error;
+    fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
+        Ok(Checksum(slice_to_array::<32>(value)?))
+    }
+}
+
+fn tempfile(out_dir: &Path) -> Result<NamedTempFile> {
+    let t = NamedTempFile::new_in(out_dir)?;
+    Ok(t)
+}
+
+/// Download the file piecewise to a temporary file and calculate the digest
+async fn download_builtin_actors_bundle(
+    tag: impl AsRef<str>,
+    out_dir: &Path,
+) -> Result<(NamedTempFile, Checksum)> {
+    let tag = tag.as_ref();
+    let mut tmp = tempfile(out_dir)?;
+
+    let url = format!("https://github.com/filecoin-project/builtin-actors/releases/download/{tag}/builtin-actors-mainnet.car");
+    let url = reqwest::Url::parse(&url)?;
+
+    let response = reqwest::get(url).await?;
+    let mut stream = response.bytes_stream();
+    // concurrently compute the sha2 & write to temp file
+    let mut sha = sha2::Sha256::default();
+    while let Some(piece) = stream.next().await {
+        let piece = piece?;
+        let piece = piece.slice(..);
+        let mut reader = piece.clone().reader();
+        std::io::copy(&mut reader, &mut sha)?;
+
+        let mut reader = piece.clone().reader();
+        std::io::copy(&mut reader, &mut tmp.as_file_mut())?;
+    }
+    let digest = sha.finalize();
+    let digest = Checksum::try_from(digest.as_slice())?;
+    Ok((tmp, digest))
+}
+
+#[tokio::main]
+async fn main() -> color_eyre::eyre::Result<()> {
+    let out_dir = std::env::var("OUT_DIR").wrap_err("Missing OUT_DIR env")?;
+    let out_dir = std::path::PathBuf::from(out_dir);
+
+    let builtin_car_path = out_dir.join("builtin_actors.car");
+
+    println!("cargo:rerun_if_changed=build.rs");
+    println!("cargo:rerun_if_env_changed={}", FORCE_RERUN);
+    println!("cargo:rerun_if_env_changed={}", VERSION_OVERRIDE);
+
+    println!("cargo:rerun_if_changed={}", builtin_car_path.display());
+
+    let tag = std::env::var(VERSION_OVERRIDE).unwrap_or_else(|_e| BUILTIN_ACTORS_TAG.to_owned());
+
+    let (tmp, tmp_digest) = download_builtin_actors_bundle(tag, &out_dir).await?;
+
+    match fs::File::open(&builtin_car_path) {
+        Ok(f) => {
+            // compare digests, if mismatch, replace existing with the downloaded file
+            let actual = file_digest(f)?;
+            if tmp_digest != actual {
+                fs::remove_file(&builtin_car_path)?;
+                fs::rename(tmp.path(), &builtin_car_path)?;
+            } else {
+                println!("Nothing to do, identical file is already present");
+            }
+        }
+        Err(e) => {
+            if let std::io::ErrorKind::NotFound = e.kind() {
+                // the file is not found, so a simple rename is good enough
+                fs::rename(tmp.path(), &builtin_car_path)?;
+            } else {
+                // other errors always must lead to an error
+                bail!(e)
+            }
+        }
+    }
+
+    println!(
+        "Builtin actors file is ready for inclusion: {}",
+        builtin_car_path.display()
+    );
+
+    Ok(())
+}

fendermint/actors-builtin-car/src/lib.rs

@@ -0,0 +1,5 @@
+// Copyright 2022-2024 Protocol Labs
+// SPDX-License-Identifier: Apache-2.0, MIT
+/// Included builtin actors bundle
+pub const BUILTIN_ACTORS_CAR: &[u8] =
+    include_bytes!(concat!(env!("OUT_DIR"), "/", "builtin_actors.car"));

fendermint/actors-custom-car/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "fendermint_actors"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[dependencies]
+cid = { workspace = true }
+anyhow = { workspace = true }
+fvm_ipld_blockstore = { workspace = true }
+fvm_ipld_encoding = { workspace = true }
+# only included for their static names (!)
+fendermint_actor_activity_tracker = { path = "../actors/activity-tracker" }
+fendermint_actor_chainmetadata = { path = "../actors/chainmetadata" }
+fendermint_actor_gas_market_eip1559 = { path = "../actors/gas_market/eip1559" }
+fendermint_actor_eam = { path = "../actors/eam" }
+
+[build-dependencies]
+color-eyre = { workspace = true }
+fs-err = { workspace = true }
+fil_actors_runtime = { workspace = true, features = ["test_utils"] }
+fil_actor_bundler = "6.1.0"
+num-traits = { workspace = true }
+toml = "0.8.19"
+cargo_metadata = "0.19"
+ignore = "0.4"
+tracing = "0.1"

fendermint/actors-custom-car/README.md

@@ -0,0 +1,3 @@
+# Actors Car
+
+The `.car` file, containing all custom / extra actors.