From 9280cff927054dfc239b675a9ecbcb3f1b61c28f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Wed, 26 Feb 2025 14:58:30 +0100 Subject: [PATCH 1/8] refactor actors bulding --- Cargo.lock | 69 +++- Cargo.toml | 1 + fendermint/actors-car/Cargo.toml | 28 ++ fendermint/actors-car/README.md | 3 + fendermint/actors-car/build.rs | 350 ++++++++++++++++++ fendermint/actors-car/src/lib.rs | 5 + .../{actors => actors-car}/src/manifest.rs | 0 fendermint/actors/Cargo.toml | 26 +- fendermint/actors/build.rs | 152 -------- fendermint/actors/src/lib.rs | 6 +- fendermint/vm/interpreter/Cargo.toml | 2 +- 11 files changed, 464 insertions(+), 178 deletions(-) create mode 100644 fendermint/actors-car/Cargo.toml create mode 100644 fendermint/actors-car/README.md create mode 100644 fendermint/actors-car/build.rs create mode 100644 fendermint/actors-car/src/lib.rs rename fendermint/{actors => actors-car}/src/manifest.rs (100%) delete mode 100644 fendermint/actors/build.rs diff --git a/Cargo.lock b/Cargo.lock index a21237968..d81751c25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,6 +12,16 @@ dependencies = [ "regex", ] +[[package]] +name = "actors-umbrella" +version = "0.1.0" +dependencies = [ + "fendermint_actor_activity_tracker", + "fendermint_actor_chainmetadata", + "fendermint_actor_eam", + "fendermint_actor_gas_market_eip1559", +] + [[package]] name = "addr2line" version = "0.21.0" @@ -1171,6 +1181,16 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "bstr" +version = "1.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -1251,6 +1271,20 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "cargo_metadata" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8769706aad5d996120af43197bf46ef6ad0fda35216b4505f926a365a232d924" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror 2.0.3", +] + [[package]] name = "castaway" version = "0.2.3" @@ -2661,7 +2695,7 @@ checksum = "82d80cc6ad30b14a48ab786523af33b37f28a8623fc06afd55324816ef18fb1f" dependencies = [ "arrayvec 0.7.6", "bytes", - "cargo_metadata", + "cargo_metadata 0.18.1", "chrono", "const-hex", "elliptic-curve 0.13.8", @@ -3048,7 +3082,9 @@ name = "fendermint_actors" version = "0.1.0" dependencies = [ "anyhow", + "cargo_metadata 0.19.1", "cid", + "color-eyre 0.5.11", "fendermint_actor_activity_tracker", "fendermint_actor_chainmetadata", "fendermint_actor_eam", @@ -3058,8 +3094,10 @@ dependencies = [ "fs-err", "fvm_ipld_blockstore", "fvm_ipld_encoding", + "ignore", "num-traits", "toml 0.8.19", + "tracing", ] [[package]] @@ -4536,6 +4574,19 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "globset" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f1ce686646e7f1e19bf7d5533fe443a45dbfb990e00629110797578b42fb19" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", +] + [[package]] name = "gloo-timers" version = "0.2.6" @@ -5241,6 +5292,22 @@ dependencies = [ "xmltree", ] +[[package]] +name = "ignore" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata 0.4.9", + "same-file", + "walkdir", + "winapi-util", +] + [[package]] name = "im" version = "15.1.0" diff --git a/Cargo.toml b/Cargo.toml index 1a4ee2498..50c838285 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ members = [ "fendermint/tracing", "fendermint/vm/*", "fendermint/actors", + "fendermint/actors-car", "fendermint/actors/api", "fendermint/actors/chainmetadata", "fendermint/actors/activity-tracker", diff --git a/fendermint/actors-car/Cargo.toml b/fendermint/actors-car/Cargo.toml new file mode 100644 index 000000000..f6cb240cb --- /dev/null +++ b/fendermint/actors-car/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "fendermint_actors" +version = "0.1.0" +authors.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +cid = { workspace = true } +anyhow = { workspace = true } +fvm_ipld_blockstore = { workspace = true } +fvm_ipld_encoding = { workspace = true } +# only included for their static names (!) +fendermint_actor_activity_tracker = { path = "../actors/activity-tracker" } +fendermint_actor_chainmetadata = { path = "../actors/chainmetadata" } +fendermint_actor_gas_market_eip1559 = { path = "../actors/gas_market/eip1559" } +fendermint_actor_eam = { path = "../actors/eam" } + +[build-dependencies] +color-eyre = { workspace = true } +fs-err = { workspace = true } +fil_actors_runtime = { workspace = true, features = ["test_utils"] } +fil_actor_bundler = "6.1.0" +num-traits = { workspace = true } +toml = "0.8.19" +cargo_metadata = "0.19" +ignore = "0.4" +tracing = "0.1" diff --git a/fendermint/actors-car/README.md b/fendermint/actors-car/README.md new file mode 100644 index 000000000..84afef2fc --- /dev/null +++ b/fendermint/actors-car/README.md @@ -0,0 +1,3 @@ +# Actors Car + +The `.car` file, containing all custom / extra actors. diff --git a/fendermint/actors-car/build.rs b/fendermint/actors-car/build.rs new file mode 100644 index 000000000..f92819747 --- /dev/null +++ b/fendermint/actors-car/build.rs @@ -0,0 +1,350 @@ +// Copyright 2022-2024 Protocol Labs +// Copyright Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0, MIT + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use cargo_metadata::{DependencyKind, MetadataCommand}; +use fil_actor_bundler::Bundler; +use fs_err as fs; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::thread; +use toml::Value; +use color_eyre::eyre::{bail, eyre, OptionExt, Result}; +use tracing::info; +use std::collections::HashSet; +use std::ops::Deref; + +fn parse_dependencies_of_umbrella_crate(manifest_path: &Path) -> Result> { + let manifest = dbg!(fs::read_to_string(manifest_path))?; + let document = dbg!(manifest.parse::()?); + + let dependencies = document + .get("target") + .ok_or_eyre("could not find target table")? + .get(r#"cfg(target_arch = "wasm32")"#) + .ok_or_eyre("could not find target_arch table")? + .get("dependencies") + .ok_or_eyre("could not find dependencies table")? + .as_table() + .ok_or_eyre("could not find dependencies")?; + + let mut ret = Vec::with_capacity(dependencies.len()); + for (name, details) in dependencies.iter() { + let Some(path) = details + .get("path") + .and_then(Value::as_str) else { continue }; + ret.push((name.clone(), std::path::PathBuf::from(path))); + } + + Ok(ret) +} + +pub fn rerun_if_changed(path: &Path) { + println!("cargo:rerun-if-changed={}", path.display()); +} + +/// Custom wrapper for a [`cargo_metadata::Package`] to store it in +/// a `HashSet`. +#[derive(Debug)] +struct DeduplicatePackage<'a> { + package: &'a cargo_metadata::Package, + identifier: String, +} +impl<'a> From<&'a cargo_metadata::Package> for DeduplicatePackage<'a> { + fn from(package: &'a cargo_metadata::Package) -> Self { + Self { + package, + identifier: format!("{}{}{:?}", package.name, package.version, package.source), + } + } +} + +impl<'a> std::hash::Hash for DeduplicatePackage<'a> { + fn hash(&self, state: &mut H) { + self.identifier.hash(state); + } +} + +impl<'a> PartialEq for DeduplicatePackage<'a> { + fn eq(&self, other: &Self) -> bool { + self.identifier == other.identifier + } +} + +impl<'a> Eq for DeduplicatePackage<'a> {} + +impl<'a> Deref for DeduplicatePackage<'a> { + type Target = cargo_metadata::Package; + + fn deref(&self) -> &Self::Target { + self.package + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Actor { + package_name: String, + crate_path: PathBuf, + wasm_blob_path: PathBuf, +} + +/// Bundle all indidivual wasm blobs together +fn bundle_wasm_blobs_into_car(actor_wasm_blobs: &[Actor], dst: &Path) -> Result { + let mut bundler = Bundler::new(dst); + for (Actor { package_name: pkg, wasm_blob_path: path, ..}, id) in actor_wasm_blobs.iter().zip(1u32..) { + + // This actor version doesn't force synthetic CIDs; it uses genuine + // content-addressed CIDs. + let forced_cid = None; + + let actor_name = pkg + .strip_prefix("fendermint_actor_") + .ok_or_eyre( + format!("expected fendermint_actor_ prefix in actor package name; got: {pkg}") + )? + .to_owned(); + + let cid = bundler + .add_from_file(id, actor_name, forced_cid, &path) + .map_err(|err| eyre!( + "failed to add file {:?} to bundle for actor {}: {}", + path, id, err + ))?; + info!( + "added {} ({}) to bundle with CID {}", + pkg, id, cid + ); + } + + bundler.finish().map_err(|e| eyre!("Failed to finish bundle builder: {e}"))?; + + Ok(dst.to_path_buf()) +} + +fn create_metadata_command(path: impl Into) -> MetadataCommand { + let mut metadata_command = MetadataCommand::new(); + metadata_command.manifest_path(path); + // metadata_command.other_options(vec!["--offline".to_owned()]); + metadata_command +} + +/// Find the `Cargo.lock` relative to the `OUT_DIR` environment variable. +/// +/// If the `Cargo.lock` cannot be found, we emit a warning and return `None`. +fn find_cargo_lock(out_dir: &Path) -> Option { + fn find_impl(mut path: PathBuf) -> Option { + loop { + if path.join("Cargo.lock").exists() { + return Some(path.join("Cargo.lock")) + } + + if !path.pop() { + return None + } + } + } + + if let Some(path) = find_impl(out_dir.to_path_buf()) { + return Some(path) + } + + None +} + +/// Track files and paths related to the given package to rerun `build.rs` on any relevant change. +fn package_rerun_if_changed(package: &DeduplicatePackage) { + let mut manifest_path = package.manifest_path.clone(); + if manifest_path.ends_with("Cargo.toml") { + manifest_path.pop(); + } + + ignore::Walk::new(&manifest_path) + .into_iter() + .filter(|p| { + // Ignore this entry if it is a directory that contains a `Cargo.toml` that is not the + // `Cargo.toml` related to the current package. This is done to ignore sub-crates of a + // crate. If such a sub-crate is a dependency, it will be processed independently + // anyway. + let Ok(p) = p else { return false }; + let p = p.path(); + p == manifest_path || !p.is_dir() || !p.join("Cargo.toml").exists() + }) + .filter_map(|p| p.ok().map(|p| p.into_path())) + .filter(|p| p.extension().map(|e| e == "rs" || e == "toml").unwrap_or_default()) + .for_each(|ref x| rerun_if_changed(x)); +} + +fn build_all_wasm_blobs(actors: &[Actor], manifest_path: &Path, cargo: &Path, out_dir: &Path) -> Result> { + let package_args = actors.iter().map(|actor|format!("-p={}", actor.package_name)); + + // Cargo build command for all test_actors at once. + let mut cmd = Command::new(cargo); + cmd.arg("build") + .args(package_args) + .arg("--target=wasm32-unknown-unknown") + .arg("--profile=wasm") + .arg("--features=fil-actor") + .arg(format!("--manifest-path={}", manifest_path.display())) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + // We are supposed to only generate artifacts under OUT_DIR, + // so set OUT_DIR as the target directory for this build. + .env("CARGO_TARGET_DIR", &out_dir) + // As we are being called inside a build-script, this env variable is set. However, we set + // our own `RUSTFLAGS` and thus, we need to remove this. Otherwise cargo favors this + // env variable. + .env_remove("CARGO_ENCODED_RUSTFLAGS"); + + // Launch the command. + let mut child = cmd.spawn()?; + + // Pipe the output as cargo warnings. Unfortunately this is the only way to + // get cargo build to print the output. + let stdout = child.stdout.take().expect("no stdout"); + let stderr = child.stderr.take().expect("no stderr"); + let j1 = thread::spawn(move || { + for line in BufReader::new(stderr).lines() { + println!("cargo:warning={:?}", line.unwrap()); + } + }); + let j2 = thread::spawn(move || { + for line in BufReader::new(stdout).lines() { + println!("cargo:warning={:?}", line.unwrap()); + } + }); + + let _ = j1.join(); + let _ = j2.join(); + + let exit_status = child.wait()?; + if !exit_status.success() { + bail!("actor build faile"); + } + Ok(actors.to_vec()) +} + +/// Use the `Cargo.lock` file to find the workspace manifest and extract dependency information +fn print_cargo_rerun_if_dependency_instructions( + cargo_manifest: &Path, + out_dir: &Path, +) -> Result<()> { + // Rerun `build.rs` if the `Cargo.lock` changes + let cargo_lock = find_cargo_lock(out_dir).ok_or_eyre("Couldn't find Cargo.lock")?; + rerun_if_changed(&cargo_lock); + + let workspace = cargo_lock.parent().unwrap(); + + let metadata = create_metadata_command(workspace.join("Cargo.toml")) + .exec()?; + + let package = metadata + .packages + .iter() + .find(|p| p.manifest_path == cargo_manifest).ok_or_eyre("Detected a circular dependency. The crate depends on itself..")?; + + // Start with the dependencies of the crate we want to compile for wasm. + let mut dependencies = Vec::from_iter(package.dependencies.iter()); + + // Collect all packages by follow the dependencies of all packages we find. + let mut packages = HashSet::new(); + packages.insert(DeduplicatePackage::from(package)); + + while let Some(dependency) = dependencies.pop() { + // Ignore all dev dependencies + if dependency.kind == DependencyKind::Development { + continue + } + + let path_or_git_dep = + dependency.source.as_ref().map(|s| s.starts_with("git+")).unwrap_or(true); + + let maybe_package = metadata + .packages + .iter() + .filter(|p| !p.manifest_path.starts_with(&workspace)) + .find(|p| { + // Check that the name matches and that the version matches or this is + // a git or path dep. A git or path dependency can only occur once, so we don't + // need to check the version. + (path_or_git_dep || dependency.req.matches(&p.version)) && dependency.name == p.name + }); + + if let Some(package) = maybe_package { + if packages.insert(DeduplicatePackage::from(package)) { + dependencies.extend(package.dependencies.iter()); + } + } + } + + // Make sure that if any file/folder of a dependency change, we need to rerun the `build.rs` + packages.iter().for_each(package_rerun_if_changed); + + Ok(()) +} + + + +fn main() -> Result<()> { + // Cargo executable location. + let cargo = std::env::var_os("CARGO").ok_or_eyre("no CARGO env var")?; + let cargo = Path::new(&cargo); + + // Rust provided out dir, used as intermediate place to store a) wasm target dir b) their outputs + // the final `.car` file will sit under `dst`. + let out_dir = std::env::var_os("OUT_DIR") + .as_ref() + .map(Path::new) + .map(|p| p.join("bundle")) + .ok_or_eyre("Must have OUT_DIR defined, this is only ever run from build.rs")?; + + let manifest_path_dir = std::env::var_os("CARGO_MANIFEST_DIR").ok_or_eyre("CARGO_MANIFEST_DIR unset")?; + let manifest_path_dir = + Path::new(&manifest_path_dir); + let actors_manifest_path = manifest_path_dir + .parent().unwrap() + .join("actors") + .join("Cargo.toml"); + + let wasm_blob_dir = out_dir + .join("wasm32-unknown-unknown") + .join("wasm"); + + let actors = parse_dependencies_of_umbrella_crate(&actors_manifest_path)?; + let actors = Vec::from_iter( + actors + .iter() + .map(|(name, crate_path)| Actor { + package_name: name.as_str().to_owned(), + wasm_blob_path: wasm_blob_dir.join(name.as_str()).with_extension("wasm"), + crate_path: crate_path.clone(), + }) + ); + + print_cargo_rerun_if_dependency_instructions(&actors_manifest_path, &out_dir)?; + + build_all_wasm_blobs(&actors, &actors_manifest_path, &cargo, &out_dir)?; + + let bundle_car_dest_dir = actors_manifest_path.parent().unwrap().join("output"); + + fs_err::create_dir_all(&bundle_car_dest_dir)?; + let bundle_car_path = bundle_car_dest_dir.join("custom_actors_bundle.car"); + + rerun_if_changed(&bundle_car_path); + + bundle_wasm_blobs_into_car(&actors, &bundle_car_path)?; + + Ok(()) +} diff --git a/fendermint/actors-car/src/lib.rs b/fendermint/actors-car/src/lib.rs new file mode 100644 index 000000000..006333c8a --- /dev/null +++ b/fendermint/actors-car/src/lib.rs @@ -0,0 +1,5 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT +mod manifest; + +pub use manifest::Manifest; diff --git a/fendermint/actors/src/manifest.rs b/fendermint/actors-car/src/manifest.rs similarity index 100% rename from fendermint/actors/src/manifest.rs rename to fendermint/actors-car/src/manifest.rs diff --git a/fendermint/actors/Cargo.toml b/fendermint/actors/Cargo.toml index d50176cd0..089f43132 100644 --- a/fendermint/actors/Cargo.toml +++ b/fendermint/actors/Cargo.toml @@ -1,30 +1,18 @@ [package] -name = "fendermint_actors" +name = "actors-umbrella" version = "0.1.0" authors.workspace = true edition.workspace = true license.workspace = true +description = "Depend on all individual actors to be included." +# we use the list of dependencies +# to determine which ones to include in the `.car` file +# and filter for the `fendermint_actor_` prefix +# the all-in-one wasmblob is NOT used, nor should it be compiled, +# neither for the host nor target arch [target.'cfg(target_arch = "wasm32")'.dependencies] fendermint_actor_activity_tracker = { path = "activity-tracker", features = ["fil-actor"] } fendermint_actor_chainmetadata = { path = "chainmetadata", features = ["fil-actor"] } fendermint_actor_gas_market_eip1559 = { path = "gas_market/eip1559", features = ["fil-actor"] } fendermint_actor_eam = { path = "eam", features = ["fil-actor"] } - -[dependencies] -cid = { workspace = true } -anyhow = { workspace = true } -fvm_ipld_blockstore = { workspace = true } -fvm_ipld_encoding = { workspace = true } -fendermint_actor_chainmetadata = { path = "chainmetadata" } -fendermint_actor_eam = { path = "eam" } -fendermint_actor_gas_market_eip1559 = { path = "gas_market/eip1559" } -fs-err = { workspace = true } - -[build-dependencies] -anyhow = { workspace = true } -fil_actors_runtime = { workspace = true, features = ["test_utils"] } -fil_actor_bundler = "6.1.0" -fs-err = { workspace = true } -num-traits = { workspace = true } -toml = "0.8.19" diff --git a/fendermint/actors/build.rs b/fendermint/actors/build.rs deleted file mode 100644 index 6f452be35..000000000 --- a/fendermint/actors/build.rs +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT - -use anyhow::anyhow; -use fil_actor_bundler::Bundler; -use fs_err as fs; -use std::error::Error; -use std::io::{BufRead, BufReader}; -use std::path::Path; -use std::process::{Command, Stdio}; -use std::thread; -use toml::Value; - -fn parse_dependencies_for_wasm32() -> anyhow::Result> { - let manifest = fs::read_to_string("Cargo.toml")?; - let document = manifest.parse::()?; - - let dependencies = document - .get("target") - .and_then(|t| t.get(r#"cfg(target_arch = "wasm32")"#)) - .and_then(|t| t.get("dependencies")) - .and_then(Value::as_table) - .ok_or_else(|| anyhow!("could not find wasm32 dependencies"))?; - - let mut ret = Vec::with_capacity(dependencies.len()); - for (name, details) in dependencies.iter() { - let path = details - .get("path") - .and_then(Value::as_str) - .ok_or_else(|| anyhow!("could not find path for a wasm32 dependency"))?; - ret.push((name.clone(), path.to_string())); - } - - Ok(ret) -} - -const FILES_TO_WATCH: &[&str] = &["Cargo.toml", "src"]; - -fn main() -> Result<(), Box> { - // Cargo executable location. - let cargo = std::env::var_os("CARGO").expect("no CARGO env var"); - - let out_dir = std::env::var_os("OUT_DIR") - .as_ref() - .map(Path::new) - .map(|p| p.join("bundle")) - .expect("no OUT_DIR env var"); - println!("cargo:warning=out_dir: {:?}", &out_dir); - - let manifest_path = - Path::new(&std::env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR unset")) - .join("Cargo.toml"); - - let actors = parse_dependencies_for_wasm32()?; - let actor_files = actors - .iter() - .map(|(name, _)| name.as_str()) - .collect::>(); - - for file in [FILES_TO_WATCH, actor_files.as_slice()].concat() { - println!("cargo:rerun-if-changed={}", file); - } - - // Cargo build command for all test_actors at once. - let mut cmd = Command::new(cargo); - cmd.arg("build") - .args(actors.iter().map(|(pkg, _)| "-p=".to_owned() + pkg)) - .arg("--target=wasm32-unknown-unknown") - .arg("--profile=wasm") - .arg("--features=fil-actor") - .arg(format!("--manifest-path={}", manifest_path.display())) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - // We are supposed to only generate artifacts under OUT_DIR, - // so set OUT_DIR as the target directory for this build. - .env("CARGO_TARGET_DIR", &out_dir) - // As we are being called inside a build-script, this env variable is set. However, we set - // our own `RUSTFLAGS` and thus, we need to remove this. Otherwise cargo favors this - // env variable. - .env_remove("CARGO_ENCODED_RUSTFLAGS"); - - // Print out the command line we're about to run. - println!("cargo:warning=cmd={:?}", &cmd); - - // Launch the command. - let mut child = cmd.spawn().expect("failed to launch cargo build"); - - // Pipe the output as cargo warnings. Unfortunately this is the only way to - // get cargo build to print the output. - let stdout = child.stdout.take().expect("no stdout"); - let stderr = child.stderr.take().expect("no stderr"); - let j1 = thread::spawn(move || { - for line in BufReader::new(stderr).lines() { - println!("cargo:warning={:?}", line.unwrap()); - } - }); - let j2 = thread::spawn(move || { - for line in BufReader::new(stdout).lines() { - println!("cargo:warning={:?}", line.unwrap()); - } - }); - - j1.join().unwrap(); - j2.join().unwrap(); - - let result = child.wait().expect("failed to wait for build to finish"); - if !result.success() { - return Err("actor build failed".into()); - } - - // make sure the output dir exists - fs::create_dir_all("output") - .expect("failed to create output dir for the custom_actors_bundle.car file"); - - let dst = Path::new("output/custom_actors_bundle.car"); - let mut bundler = Bundler::new(dst); - for (pkg, id) in actors.iter().map(|(pkg, _)| pkg).zip(1u32..) { - let bytecode_path = Path::new(&out_dir) - .join("wasm32-unknown-unknown/wasm") - .join(format!("{}.wasm", pkg)); - - // This actor version doesn't force synthetic CIDs; it uses genuine - // content-addressed CIDs. - let forced_cid = None; - - let actor_name = pkg - .to_owned() - .strip_prefix("fendermint_actor_") - .ok_or_else(|| { - format!("expected fendermint_actor_ prefix in actor package name; got: {pkg}") - })? - .to_owned(); - - let cid = bundler - .add_from_file(id, actor_name, forced_cid, &bytecode_path) - .unwrap_or_else(|err| { - panic!( - "failed to add file {:?} to bundle for actor {}: {}", - bytecode_path, id, err - ) - }); - println!( - "cargo:warning=added {} ({}) to bundle with CID {}", - pkg, id, cid - ); - } - bundler.finish().expect("failed to finish bundle"); - - println!("cargo:warning=bundle={}", dst.display()); - - Ok(()) -} diff --git a/fendermint/actors/src/lib.rs b/fendermint/actors/src/lib.rs index 006333c8a..dd82e9a10 100644 --- a/fendermint/actors/src/lib.rs +++ b/fendermint/actors/src/lib.rs @@ -1,5 +1 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT -mod manifest; - -pub use manifest::Manifest; +//! Empty, just a placeholder to dispatch to the dependencies diff --git a/fendermint/vm/interpreter/Cargo.toml b/fendermint/vm/interpreter/Cargo.toml index 1b1aca7ad..d6db17291 100644 --- a/fendermint/vm/interpreter/Cargo.toml +++ b/fendermint/vm/interpreter/Cargo.toml @@ -22,7 +22,7 @@ fendermint_crypto = { path = "../../crypto" } fendermint_eth_hardhat = { path = "../../eth/hardhat" } fendermint_rpc = { path = "../../rpc" } fendermint_tracing = { path = "../../tracing" } -fendermint_actors = { path = "../../actors" } +fendermint_actors = { path = "../../actors-car" } fendermint_actor_chainmetadata = { path = "../../actors/chainmetadata" } fendermint_actor_activity_tracker = { path = "../../actors/activity-tracker" } fendermint_actor_gas_market_eip1559 = { path = "../../actors/gas_market/eip1559" } From 6e3aefb78b5baf582acb75f09638056b4d6283cb Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 27 Feb 2025 09:39:00 +0100 Subject: [PATCH 2/8] x --- Cargo.lock | 88 +++++++++++++++++++++++++++++------- Cargo.toml | 1 + fendermint/actors/src/lib.rs | 2 + 3 files changed, 74 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d81751c25..ca1f7d85c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1548,6 +1548,21 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "color-eyre" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f1885697ee8a177096d42f158922251a41973117f6d8a234cee94b9509157b7" +dependencies = [ + "backtrace", + "color-spantrace 0.1.6", + "eyre", + "indenter", + "once_cell", + "owo-colors 1.3.0", + "tracing-error 0.1.2", +] + [[package]] name = "color-eyre" version = "0.6.3" @@ -1555,12 +1570,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55146f5e46f237f7423d74111267d4597b59b0dad0ffaf7303bce9945d843ad5" dependencies = [ "backtrace", - "color-spantrace", + "color-spantrace 0.2.1", "eyre", "indenter", "once_cell", - "owo-colors", - "tracing-error", + "owo-colors 3.5.0", + "tracing-error 0.2.1", +] + +[[package]] +name = "color-spantrace" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6eee477a4a8a72f4addd4de416eb56d54bc307b284d6601bafdee1f4ea462d1" +dependencies = [ + "once_cell", + "owo-colors 1.3.0", + "tracing-core", + "tracing-error 0.1.2", ] [[package]] @@ -1570,9 +1597,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd6be1b2a7e382e2b98b43b2adcca6bb0e465af0bdd38123873ae61eb17a72c2" dependencies = [ "once_cell", - "owo-colors", + "owo-colors 3.5.0", "tracing-core", - "tracing-error", + "tracing-error 0.2.1", ] [[package]] @@ -2993,7 +3020,7 @@ dependencies = [ "tower", "tower-abci", "tracing", - "tracing-subscriber", + "tracing-subscriber 0.3.18", ] [[package]] @@ -3191,7 +3218,7 @@ dependencies = [ "tower-abci", "tracing", "tracing-appender", - "tracing-subscriber", + "tracing-subscriber 0.3.18", ] [[package]] @@ -3215,7 +3242,7 @@ dependencies = [ "num-traits", "tendermint-rpc", "tracing", - "tracing-subscriber", + "tracing-subscriber 0.3.18", "url", ] @@ -3333,7 +3360,7 @@ dependencies = [ "tokio", "tower-http", "tracing", - "tracing-subscriber", + "tracing-subscriber 0.3.18", ] [[package]] @@ -3444,7 +3471,7 @@ dependencies = [ "tendermint-rpc", "tokio", "tracing", - "tracing-subscriber", + "tracing-subscriber 0.3.18", ] [[package]] @@ -3775,7 +3802,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tracing", - "tracing-subscriber", + "tracing-subscriber 0.3.18", ] [[package]] @@ -5512,7 +5539,7 @@ dependencies = [ "tokio", "tokio-tungstenite 0.18.0", "toml 0.7.8", - "tracing-subscriber", + "tracing-subscriber 0.3.18", "url", "zeroize", ] @@ -5530,7 +5557,7 @@ dependencies = [ "strum", "tracing", "tracing-appender", - "tracing-subscriber", + "tracing-subscriber 0.3.18", ] [[package]] @@ -5642,7 +5669,7 @@ name = "ipc_actors_abis" version = "0.1.0" dependencies = [ "anyhow", - "color-eyre", + "color-eyre 0.6.3", "ethers", "fs-err", "fvm_shared", @@ -6076,7 +6103,7 @@ dependencies = [ "prost-build", "thiserror 1.0.69", "tracing", - "tracing-subscriber", + "tracing-subscriber 0.3.18", "unsigned-varint 0.7.2", ] @@ -7354,6 +7381,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "owo-colors" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2386b4ebe91c2f7f51082d4cefa145d030e33a1842a96b12e4885cc3c01f7a55" + [[package]] name = "owo-colors" version = "3.5.0" @@ -10484,7 +10517,7 @@ dependencies = [ "crossbeam-channel", "thiserror 1.0.69", "time", - "tracing-subscriber", + "tracing-subscriber 0.3.18", ] [[package]] @@ -10508,6 +10541,16 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-error" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4d7c0b83d4a500748fa5879461652b361edf5c9d51ede2a2ac03875ca185e24" +dependencies = [ + "tracing", + "tracing-subscriber 0.2.25", +] + [[package]] name = "tracing-error" version = "0.2.1" @@ -10515,7 +10558,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db" dependencies = [ "tracing", - "tracing-subscriber", + "tracing-subscriber 0.3.18", ] [[package]] @@ -10549,6 +10592,17 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-subscriber" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0d2eaa99c3c2e41547cfa109e910a68ea03823cccad4a0525dcbc9b01e8c71" +dependencies = [ + "sharded-slab", + "thread_local", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.18" diff --git a/Cargo.toml b/Cargo.toml index 50c838285..e7b3b794d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,6 +70,7 @@ blake2b_simd = "1.0" bloom = "0.3" bytes = "1.4" clap = { version = "4.1", features = ["derive", "env", "string"] } +color-eyre = "0.5.11" byteorder = "1.5.0" config = "0.13" dirs = "5.0" diff --git a/fendermint/actors/src/lib.rs b/fendermint/actors/src/lib.rs index dd82e9a10..d9da900cb 100644 --- a/fendermint/actors/src/lib.rs +++ b/fendermint/actors/src/lib.rs @@ -1 +1,3 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT //! Empty, just a placeholder to dispatch to the dependencies From 618e0301376547c568685708d358b049647a2944 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 27 Feb 2025 10:00:34 +0100 Subject: [PATCH 3/8] clippy = fmt --- fendermint/actors-car/build.rs | 316 +++++++++++++++++---------------- 1 file changed, 166 insertions(+), 150 deletions(-) diff --git a/fendermint/actors-car/build.rs b/fendermint/actors-car/build.rs index f92819747..eb125fb3c 100644 --- a/fendermint/actors-car/build.rs +++ b/fendermint/actors-car/build.rs @@ -15,17 +15,17 @@ // limitations under the License. use cargo_metadata::{DependencyKind, MetadataCommand}; +use color_eyre::eyre::{bail, eyre, OptionExt, Result}; use fil_actor_bundler::Bundler; use fs_err as fs; +use std::collections::HashSet; use std::io::{BufRead, BufReader}; +use std::ops::Deref; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::thread; use toml::Value; -use color_eyre::eyre::{bail, eyre, OptionExt, Result}; use tracing::info; -use std::collections::HashSet; -use std::ops::Deref; fn parse_dependencies_of_umbrella_crate(manifest_path: &Path) -> Result> { let manifest = dbg!(fs::read_to_string(manifest_path))?; @@ -43,9 +43,9 @@ fn parse_dependencies_of_umbrella_crate(manifest_path: &Path) -> Result { - package: &'a cargo_metadata::Package, - identifier: String, + package: &'a cargo_metadata::Package, + identifier: String, } impl<'a> From<&'a cargo_metadata::Package> for DeduplicatePackage<'a> { - fn from(package: &'a cargo_metadata::Package) -> Self { - Self { - package, - identifier: format!("{}{}{:?}", package.name, package.version, package.source), - } - } + fn from(package: &'a cargo_metadata::Package) -> Self { + Self { + package, + identifier: format!("{}{}{:?}", package.name, package.version, package.source), + } + } } impl<'a> std::hash::Hash for DeduplicatePackage<'a> { - fn hash(&self, state: &mut H) { - self.identifier.hash(state); - } + fn hash(&self, state: &mut H) { + self.identifier.hash(state); + } } impl<'a> PartialEq for DeduplicatePackage<'a> { - fn eq(&self, other: &Self) -> bool { - self.identifier == other.identifier - } + fn eq(&self, other: &Self) -> bool { + self.identifier == other.identifier + } } impl<'a> Eq for DeduplicatePackage<'a> {} impl<'a> Deref for DeduplicatePackage<'a> { - type Target = cargo_metadata::Package; + type Target = cargo_metadata::Package; - fn deref(&self) -> &Self::Target { - self.package - } + fn deref(&self) -> &Self::Target { + self.package + } } #[derive(Debug, Clone, PartialEq, Eq)] @@ -104,39 +104,49 @@ struct Actor { /// Bundle all indidivual wasm blobs together fn bundle_wasm_blobs_into_car(actor_wasm_blobs: &[Actor], dst: &Path) -> Result { let mut bundler = Bundler::new(dst); - for (Actor { package_name: pkg, wasm_blob_path: path, ..}, id) in actor_wasm_blobs.iter().zip(1u32..) { - + for ( + Actor { + package_name: pkg, + wasm_blob_path: path, + .. + }, + id, + ) in actor_wasm_blobs.iter().zip(1u32..) + { // This actor version doesn't force synthetic CIDs; it uses genuine // content-addressed CIDs. let forced_cid = None; let actor_name = pkg .strip_prefix("fendermint_actor_") - .ok_or_eyre( - format!("expected fendermint_actor_ prefix in actor package name; got: {pkg}") - )? + .ok_or_eyre(format!( + "expected fendermint_actor_ prefix in actor package name; got: {pkg}" + ))? .to_owned(); let cid = bundler - .add_from_file(id, actor_name, forced_cid, &path) - .map_err(|err| eyre!( + .add_from_file(id, actor_name, forced_cid, path) + .map_err(|err| { + eyre!( "failed to add file {:?} to bundle for actor {}: {}", - path, id, err - ))?; - info!( - "added {} ({}) to bundle with CID {}", - pkg, id, cid - ); + path, + id, + err + ) + })?; + info!("added {} ({}) to bundle with CID {}", pkg, id, cid); } - - bundler.finish().map_err(|e| eyre!("Failed to finish bundle builder: {e}"))?; - + + bundler + .finish() + .map_err(|e| eyre!("Failed to finish bundle builder: {e}"))?; + Ok(dst.to_path_buf()) } fn create_metadata_command(path: impl Into) -> MetadataCommand { - let mut metadata_command = MetadataCommand::new(); - metadata_command.manifest_path(path); + let mut metadata_command = MetadataCommand::new(); + metadata_command.manifest_path(path); // metadata_command.other_options(vec!["--offline".to_owned()]); metadata_command } @@ -145,50 +155,60 @@ fn create_metadata_command(path: impl Into) -> MetadataCommand { /// /// If the `Cargo.lock` cannot be found, we emit a warning and return `None`. fn find_cargo_lock(out_dir: &Path) -> Option { - fn find_impl(mut path: PathBuf) -> Option { - loop { - if path.join("Cargo.lock").exists() { - return Some(path.join("Cargo.lock")) - } - - if !path.pop() { - return None - } - } - } - - if let Some(path) = find_impl(out_dir.to_path_buf()) { - return Some(path) - } - - None + fn find_impl(mut path: PathBuf) -> Option { + loop { + if path.join("Cargo.lock").exists() { + return Some(path.join("Cargo.lock")); + } + + if !path.pop() { + return None; + } + } + } + + if let Some(path) = find_impl(out_dir.to_path_buf()) { + return Some(path); + } + + None } /// Track files and paths related to the given package to rerun `build.rs` on any relevant change. fn package_rerun_if_changed(package: &DeduplicatePackage) { - let mut manifest_path = package.manifest_path.clone(); - if manifest_path.ends_with("Cargo.toml") { - manifest_path.pop(); - } - - ignore::Walk::new(&manifest_path) - .into_iter() - .filter(|p| { - // Ignore this entry if it is a directory that contains a `Cargo.toml` that is not the - // `Cargo.toml` related to the current package. This is done to ignore sub-crates of a - // crate. If such a sub-crate is a dependency, it will be processed independently - // anyway. + let mut manifest_path = package.manifest_path.clone(); + if manifest_path.ends_with("Cargo.toml") { + manifest_path.pop(); + } + + ignore::Walk::new(&manifest_path) + .filter(|p| { + // Ignore this entry if it is a directory that contains a `Cargo.toml` that is not the + // `Cargo.toml` related to the current package. This is done to ignore sub-crates of a + // crate. If such a sub-crate is a dependency, it will be processed independently + // anyway. let Ok(p) = p else { return false }; let p = p.path(); - p == manifest_path || !p.is_dir() || !p.join("Cargo.toml").exists() - }) - .filter_map(|p| p.ok().map(|p| p.into_path())) - .filter(|p| p.extension().map(|e| e == "rs" || e == "toml").unwrap_or_default()) - .for_each(|ref x| rerun_if_changed(x)); + p == manifest_path || !p.is_dir() || !p.join("Cargo.toml").exists() + }) + .filter_map(|p| p.ok().map(|p| p.into_path())) + .filter(|p| { + p.extension() + .map(|e| e == "rs" || e == "toml") + .unwrap_or_default() + }) + .for_each(|ref x| rerun_if_changed(x)); } -fn build_all_wasm_blobs(actors: &[Actor], manifest_path: &Path, cargo: &Path, out_dir: &Path) -> Result> { - let package_args = actors.iter().map(|actor|format!("-p={}", actor.package_name)); +fn build_all_wasm_blobs( + actors: &[Actor], + manifest_path: &Path, + cargo: &Path, + out_dir: &Path, +) -> Result> { + let package_args = actors + .iter() + .map(|actor| format!("-p={}", actor.package_name)); // Cargo build command for all test_actors at once. let mut cmd = Command::new(cargo); @@ -202,7 +222,7 @@ fn build_all_wasm_blobs(actors: &[Actor], manifest_path: &Path, cargo: &Path, ou .stderr(Stdio::piped()) // We are supposed to only generate artifacts under OUT_DIR, // so set OUT_DIR as the target directory for this build. - .env("CARGO_TARGET_DIR", &out_dir) + .env("CARGO_TARGET_DIR", out_dir) // As we are being called inside a build-script, this env variable is set. However, we set // our own `RUSTFLAGS` and thus, we need to remove this. Otherwise cargo favors this // env variable. @@ -241,61 +261,62 @@ fn print_cargo_rerun_if_dependency_instructions( cargo_manifest: &Path, out_dir: &Path, ) -> Result<()> { - // Rerun `build.rs` if the `Cargo.lock` changes - let cargo_lock = find_cargo_lock(out_dir).ok_or_eyre("Couldn't find Cargo.lock")?; + // Rerun `build.rs` if the `Cargo.lock` changes + let cargo_lock = find_cargo_lock(out_dir).ok_or_eyre("Couldn't find Cargo.lock")?; rerun_if_changed(&cargo_lock); let workspace = cargo_lock.parent().unwrap(); - let metadata = create_metadata_command(workspace.join("Cargo.toml")) - .exec()?; - - let package = metadata - .packages - .iter() - .find(|p| p.manifest_path == cargo_manifest).ok_or_eyre("Detected a circular dependency. The crate depends on itself..")?; - - // Start with the dependencies of the crate we want to compile for wasm. - let mut dependencies = Vec::from_iter(package.dependencies.iter()); - - // Collect all packages by follow the dependencies of all packages we find. - let mut packages = HashSet::new(); - packages.insert(DeduplicatePackage::from(package)); - - while let Some(dependency) = dependencies.pop() { - // Ignore all dev dependencies - if dependency.kind == DependencyKind::Development { - continue - } - - let path_or_git_dep = - dependency.source.as_ref().map(|s| s.starts_with("git+")).unwrap_or(true); - - let maybe_package = metadata - .packages - .iter() - .filter(|p| !p.manifest_path.starts_with(&workspace)) - .find(|p| { - // Check that the name matches and that the version matches or this is - // a git or path dep. A git or path dependency can only occur once, so we don't - // need to check the version. - (path_or_git_dep || dependency.req.matches(&p.version)) && dependency.name == p.name - }); - - if let Some(package) = maybe_package { - if packages.insert(DeduplicatePackage::from(package)) { - dependencies.extend(package.dependencies.iter()); - } - } - } - - // Make sure that if any file/folder of a dependency change, we need to rerun the `build.rs` - packages.iter().for_each(package_rerun_if_changed); + let metadata = create_metadata_command(workspace.join("Cargo.toml")).exec()?; - Ok(()) -} + let package = metadata + .packages + .iter() + .find(|p| p.manifest_path == cargo_manifest) + .ok_or_eyre("Detected a circular dependency. The crate depends on itself..")?; + // Start with the dependencies of the crate we want to compile for wasm. + let mut dependencies = Vec::from_iter(package.dependencies.iter()); + // Collect all packages by follow the dependencies of all packages we find. + let mut packages = HashSet::new(); + packages.insert(DeduplicatePackage::from(package)); + + while let Some(dependency) = dependencies.pop() { + // Ignore all dev dependencies + if dependency.kind == DependencyKind::Development { + continue; + } + + let path_or_git_dep = dependency + .source + .as_ref() + .map(|s| s.starts_with("git+")) + .unwrap_or(true); + + let maybe_package = metadata + .packages + .iter() + .filter(|p| !p.manifest_path.starts_with(workspace)) + .find(|p| { + // Check that the name matches and that the version matches or this is + // a git or path dep. A git or path dependency can only occur once, so we don't + // need to check the version. + (path_or_git_dep || dependency.req.matches(&p.version)) && dependency.name == p.name + }); + + if let Some(package) = maybe_package { + if packages.insert(DeduplicatePackage::from(package)) { + dependencies.extend(package.dependencies.iter()); + } + } + } + + // Make sure that if any file/folder of a dependency change, we need to rerun the `build.rs` + packages.iter().for_each(package_rerun_if_changed); + + Ok(()) +} fn main() -> Result<()> { // Cargo executable location. @@ -310,40 +331,35 @@ fn main() -> Result<()> { .map(|p| p.join("bundle")) .ok_or_eyre("Must have OUT_DIR defined, this is only ever run from build.rs")?; - let manifest_path_dir = std::env::var_os("CARGO_MANIFEST_DIR").ok_or_eyre("CARGO_MANIFEST_DIR unset")?; let manifest_path_dir = - Path::new(&manifest_path_dir); + std::env::var_os("CARGO_MANIFEST_DIR").ok_or_eyre("CARGO_MANIFEST_DIR unset")?; + let manifest_path_dir = Path::new(&manifest_path_dir); let actors_manifest_path = manifest_path_dir - .parent().unwrap() - .join("actors") - .join("Cargo.toml"); + .parent() + .unwrap() + .join("actors") + .join("Cargo.toml"); - let wasm_blob_dir = out_dir - .join("wasm32-unknown-unknown") - .join("wasm"); + let wasm_blob_dir = out_dir.join("wasm32-unknown-unknown").join("wasm"); let actors = parse_dependencies_of_umbrella_crate(&actors_manifest_path)?; - let actors = Vec::from_iter( - actors - .iter() - .map(|(name, crate_path)| Actor { - package_name: name.as_str().to_owned(), - wasm_blob_path: wasm_blob_dir.join(name.as_str()).with_extension("wasm"), - crate_path: crate_path.clone(), - }) - ); + let actors = Vec::from_iter(actors.iter().map(|(name, crate_path)| Actor { + package_name: name.as_str().to_owned(), + wasm_blob_path: wasm_blob_dir.join(name.as_str()).with_extension("wasm"), + crate_path: crate_path.clone(), + })); print_cargo_rerun_if_dependency_instructions(&actors_manifest_path, &out_dir)?; - - build_all_wasm_blobs(&actors, &actors_manifest_path, &cargo, &out_dir)?; - + + build_all_wasm_blobs(&actors, &actors_manifest_path, cargo, &out_dir)?; + let bundle_car_dest_dir = actors_manifest_path.parent().unwrap().join("output"); - fs_err::create_dir_all(&bundle_car_dest_dir)?; + fs_err::create_dir_all(&bundle_car_dest_dir)?; let bundle_car_path = bundle_car_dest_dir.join("custom_actors_bundle.car"); - + rerun_if_changed(&bundle_car_path); - + bundle_wasm_blobs_into_car(&actors, &bundle_car_path)?; Ok(()) From 031e145b79c4ee0e861f58c95dbbb6df3f9168c8 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 27 Feb 2025 11:44:29 +0100 Subject: [PATCH 4/8] add builtin fetcher --- Cargo.lock | 417 +++++++++++++++--- Cargo.toml | 3 +- fendermint/actors-builtin-car/Cargo.toml | 17 + fendermint/actors-builtin-car/build.rs | 148 +++++++ fendermint/actors-builtin-car/src/lib.rs | 2 + fendermint/actors-car/src/lib.rs | 5 - .../Cargo.toml | 0 .../README.md | 0 .../build.rs | 4 +- fendermint/actors-custom-car/src/lib.rs | 8 + .../src/manifest.rs | 0 fendermint/vm/interpreter/Cargo.toml | 2 +- 12 files changed, 542 insertions(+), 64 deletions(-) create mode 100644 fendermint/actors-builtin-car/Cargo.toml create mode 100644 fendermint/actors-builtin-car/build.rs create mode 100644 fendermint/actors-builtin-car/src/lib.rs delete mode 100644 fendermint/actors-car/src/lib.rs rename fendermint/{actors-car => actors-custom-car}/Cargo.toml (100%) rename fendermint/{actors-car => actors-custom-car}/README.md (100%) rename fendermint/{actors-car => actors-custom-car}/build.rs (99%) create mode 100644 fendermint/actors-custom-car/src/lib.rs rename fendermint/{actors-car => actors-custom-car}/src/manifest.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index ca1f7d85c..ae30e7485 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,6 +12,20 @@ dependencies = [ "regex", ] +[[package]] +name = "actors-builtin-fetch" +version = "0.1.0" +dependencies = [ + "bytes", + "color-eyre 0.5.11", + "fs-err", + "futures-util", + "reqwest 0.12.12", + "sha2 0.10.8", + "tempfile", + "tokio", +] + [[package]] name = "actors-umbrella" version = "0.1.0" @@ -662,7 +676,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d9a9bf8b79a749ee0b911b91b671cc2b6c670bdbc7e3dfd537576ddc94bb2a2" dependencies = [ - "http", + "http 0.2.12", "log", "url", ] @@ -707,9 +721,9 @@ dependencies = [ "bitflags 1.3.2", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.31", "itoa", "matchit", "memchr", @@ -722,10 +736,10 @@ dependencies = [ "serde_path_to_error", "serde_urlencoded", "sha1", - "sync_wrapper", + "sync_wrapper 0.1.2", "tokio", "tokio-tungstenite 0.20.1", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", ] @@ -739,8 +753,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "mime", "rustversion", "tower-layer", @@ -1143,8 +1157,8 @@ dependencies = [ "futures-core", "futures-util", "hex", - "http", - "hyper", + "http 0.2.12", + "hyper 0.14.31", "hyperlocal", "log", "pin-project-lite", @@ -2690,7 +2704,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "reqwest", + "reqwest 0.11.27", "serde", "serde_json", "syn 2.0.87", @@ -2752,7 +2766,7 @@ checksum = "e79e5973c26d4baf0ce55520bd732314328cabe53193286671b47144145b9649" dependencies = [ "chrono", "ethers-core", - "reqwest", + "reqwest 0.11.27", "semver", "serde", "serde_json", @@ -2777,7 +2791,7 @@ dependencies = [ "futures-locks", "futures-util", "instant", - "reqwest", + "reqwest 0.11.27", "serde", "serde_json", "thiserror 1.0.69", @@ -2805,12 +2819,12 @@ dependencies = [ "futures-timer", "futures-util", "hashers", - "http", + "http 0.2.12", "instant", "jsonwebtoken", "once_cell", "pin-project", - "reqwest", + "reqwest 0.11.27", "serde", "serde_json", "thiserror 1.0.69", @@ -3017,7 +3031,7 @@ dependencies = [ "structopt", "tendermint 0.31.1", "tokio", - "tower", + "tower 0.4.13", "tower-abci", "tracing", "tracing-subscriber 0.3.18", @@ -3214,7 +3228,7 @@ dependencies = [ "tendermint-proto 0.31.1", "tendermint-rpc", "tokio", - "tower", + "tower 0.4.13", "tower-abci", "tracing", "tracing-appender", @@ -4675,7 +4689,26 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.12", + "indexmap 2.6.0", + "slab", + "tokio", + "tokio-util 0.7.12", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.2.0", "indexmap 2.6.0", "slab", "tokio", @@ -4741,7 +4774,7 @@ dependencies = [ "base64 0.21.7", "bytes", "headers-core", - "http", + "http 0.2.12", "httpdate", "mime", "sha1", @@ -4753,7 +4786,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" dependencies = [ - "http", + "http 0.2.12", ] [[package]] @@ -4941,6 +4974,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.6" @@ -4948,7 +4992,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.2.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http 1.2.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -4995,9 +5062,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -5009,6 +5076,26 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.8", + "http 1.2.0", + "http-body 1.0.1", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-proxy" version = "0.9.1" @@ -5018,8 +5105,8 @@ dependencies = [ "bytes", "futures", "headers", - "http", - "hyper", + "http 0.2.12", + "hyper 0.14.31", "hyper-rustls 0.22.1", "rustls-native-certs 0.5.0", "tokio", @@ -5036,7 +5123,7 @@ checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" dependencies = [ "ct-logs", "futures-util", - "hyper", + "hyper 0.14.31", "log", "rustls 0.19.1", "rustls-native-certs 0.5.0", @@ -5053,13 +5140,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", - "hyper", + "http 0.2.12", + "hyper 0.14.31", "rustls 0.21.12", "tokio", "tokio-rustls 0.24.1", ] +[[package]] +name = "hyper-rustls" +version = "0.27.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" +dependencies = [ + "futures-util", + "http 1.2.0", + "hyper 1.6.0", + "hyper-util", + "rustls 0.23.16", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.1", + "tower-service", + "webpki-roots 0.26.8", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -5067,12 +5172,47 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper", + "hyper 0.14.31", "native-tls", "tokio", "tokio-native-tls", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.6.0", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.2.0", + "http-body 1.0.1", + "hyper 1.6.0", + "pin-project-lite", + "socket2 0.5.7", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "hyperlocal" version = "0.8.0" @@ -5081,7 +5221,7 @@ checksum = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c" dependencies = [ "futures-util", "hex", - "hyper", + "hyper 0.14.31", "pin-project", "tokio", ] @@ -5295,7 +5435,7 @@ dependencies = [ "log", "rtnetlink", "smol", - "system-configuration", + "system-configuration 0.5.1", "tokio", "windows", ] @@ -5310,8 +5450,8 @@ dependencies = [ "attohttpc", "bytes", "futures", - "http", - "hyper", + "http 0.2.12", + "hyper 0.14.31", "log", "rand", "tokio", @@ -5529,7 +5669,7 @@ dependencies = [ "openssl", "prometheus", "prometheus_exporter", - "reqwest", + "reqwest 0.11.27", "serde", "serde_bytes", "serde_json", @@ -5579,7 +5719,7 @@ dependencies = [ "fvm_ipld_encoding", "fvm_shared", "hex", - "http", + "http 0.2.12", "indoc", "ipc-api", "ipc-observability", @@ -5592,7 +5732,7 @@ dependencies = [ "num-derive 0.3.3", "num-traits", "prometheus", - "reqwest", + "reqwest 0.11.27", "serde", "serde_bytes", "serde_json", @@ -8503,12 +8643,12 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.31", "hyper-rustls 0.24.2", - "hyper-tls", + "hyper-tls 0.5.0", "ipnet", "js-sys", "log", @@ -8518,12 +8658,12 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rustls 0.21.12", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", - "system-configuration", + "sync_wrapper 0.1.2", + "system-configuration 0.5.1", "tokio", "tokio-native-tls", "tokio-rustls 0.24.1", @@ -8536,6 +8676,57 @@ dependencies = [ "winreg", ] +[[package]] +name = "reqwest" +version = "0.12.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" +dependencies = [ + "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.4.8", + "http 1.2.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.6.0", + "hyper-rustls 0.27.5", + "hyper-tls 0.6.0", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls 0.23.16", + "rustls-pemfile 2.2.0", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 1.0.2", + "system-configuration 0.6.1", + "tokio", + "tokio-native-tls", + "tokio-rustls 0.26.1", + "tokio-util 0.7.12", + "tower 0.5.2", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots 0.26.8", + "windows-registry", +] + [[package]] name = "resolv-conf" version = "0.7.0" @@ -8814,7 +9005,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "schannel", "security-framework", ] @@ -8828,6 +9019,15 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.10.0" @@ -9781,7 +9981,7 @@ dependencies = [ "fs2", "hex", "once_cell", - "reqwest", + "reqwest 0.11.27", "semver", "serde", "serde_json", @@ -9819,6 +10019,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.12.6" @@ -9850,7 +10059,18 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", "core-foundation", - "system-configuration-sys", + "system-configuration-sys 0.5.0", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.6.0", + "core-foundation", + "system-configuration-sys 0.6.0", ] [[package]] @@ -9863,6 +10083,16 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tap" version = "1.0.1" @@ -10030,8 +10260,8 @@ dependencies = [ "flex-error", "futures", "getrandom", - "http", - "hyper", + "http 0.2.12", + "hyper 0.14.31", "hyper-proxy", "hyper-rustls 0.22.1", "peg", @@ -10290,6 +10520,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" +dependencies = [ + "rustls 0.23.16", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.16" @@ -10447,6 +10687,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 1.0.2", + "tokio", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-abci" version = "0.7.0" @@ -10462,7 +10717,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-util 0.6.10", - "tower", + "tower 0.4.13", "tracing", ] @@ -10476,8 +10731,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "http-range-header", "pin-project-lite", "tower-layer", @@ -10665,7 +10920,7 @@ dependencies = [ "base64 0.13.1", "byteorder", "bytes", - "http", + "http 0.2.12", "httparse", "log", "native-tls", @@ -10687,7 +10942,7 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http", + "http 0.2.12", "httparse", "log", "rand", @@ -11029,6 +11284,19 @@ dependencies = [ "leb128", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.95.0" @@ -11314,6 +11582,15 @@ version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +[[package]] +name = "webpki-roots" +version = "0.26.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "which" version = "4.4.2" @@ -11391,6 +11668,36 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-registry" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index e7b3b794d..9f08d4949 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,8 @@ members = [ "fendermint/tracing", "fendermint/vm/*", "fendermint/actors", - "fendermint/actors-car", + "fendermint/actors-custom-car", + "fendermint/actors-builtin-car", "fendermint/actors/api", "fendermint/actors/chainmetadata", "fendermint/actors/activity-tracker", diff --git a/fendermint/actors-builtin-car/Cargo.toml b/fendermint/actors-builtin-car/Cargo.toml new file mode 100644 index 000000000..b5e445f1c --- /dev/null +++ b/fendermint/actors-builtin-car/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "actors-builtin-fetch" +version = "0.1.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +license-file.workspace = true + +[build-dependencies] +reqwest = { version = "0.12", features = ["rustls-tls", "stream"] } +tokio = { version = "1", features = ["full"] } +fs-err = { workspace = true } +sha2 = { workspace = true } +tempfile = { workspace = true } +color-eyre = { workspace = true } +futures-util = { workspace = true } +bytes = { workspace = true } diff --git a/fendermint/actors-builtin-car/build.rs b/fendermint/actors-builtin-car/build.rs new file mode 100644 index 000000000..ad1a91a01 --- /dev/null +++ b/fendermint/actors-builtin-car/build.rs @@ -0,0 +1,148 @@ +//! Download filecoin's builtin actors car file + +use std::io::{Write, Read}; +use fs_err as fs; +use color_eyre::eyre::{self, bail, WrapErr, Result}; +use sha2::Digest; +use futures_util::stream::StreamExt; +use std::path::Path; +use tempfile::NamedTempFile; +use bytes::buf::Buf; + +const BUILTIN_ACTORS_TAG: &str = "v15.0.0"; + +const FORCE_RERUN: &str = "IPC_BUILTIN_ACTORS_FORCE_FETCH"; + +const VERSION_OVERRIDE: &str = "IPC_BUILTIN_ACTORS_VERSION_OVERRIDE"; + +/// Handle `Interrupt` errors, call a closure for each read kb piece +fn read_file_piecewise Result<()>>(mut reader: R, mut f: F) -> Result<()> { + let mut buf = [0u8; 1024]; + loop { + match reader.read(&mut buf[..]) { + Ok(0) => { + return Ok(()); + } + Ok(n) => { + f(&buf[..n])?; + } + Err(e) => { + match e.kind() { + std::io::ErrorKind::Interrupted => continue, + _ => return Err(e.into()), + } + } + } + } +} + +/// Calculate the digest of a file +fn file_digest(reader: impl Read) -> Result { + let mut sha = sha2::Sha256::default(); + read_file_piecewise(reader, |bytes| { + sha.write_all(bytes)?; + Ok(()) + })?; + let digest = sha.finalize(); + Checksum::try_from(digest.as_slice()) +} + +/// Convert a slice to an array +/// +/// The slice must have the exact length of the array size `N`. +fn slice_to_array(bytes: &[u8]) -> Result<[u8;N]> { + if bytes.len() != N { + bail!("Length mismatch, execpted {}, but got {}", N, bytes.len()); + } + let mut buf = [0u8;N]; + buf.copy_from_slice(&bytes[..N]); + Ok(buf) +} + +/// Digest wrapping type +#[derive(Debug, PartialEq, Eq, Hash)] +struct Checksum([u8;32]); + +impl TryFrom<&[u8]> for Checksum { + type Error = eyre::Error; + fn try_from(value: &[u8]) -> Result { + Ok(Checksum(slice_to_array::<32>(value)?)) + } +} + +fn tempfile(out_dir: &Path) -> Result { + let t = NamedTempFile::new_in(out_dir)?; + Ok(t) +} + +/// Download the file piecewise to a temporary file and calculate the digest +async fn download_builtin_actors_bundle(tag: impl AsRef, out_dir: &Path) -> Result<(NamedTempFile, Checksum)> { + let tag = tag.as_ref(); + let mut tmp = tempfile(out_dir)?; + + let url = format!("https://github.com/filecoin-project/builtin-actors/releases/download/{tag}/builtin-actors-mainnet.car"); + let url = reqwest::Url::parse(&url)?; + + let response = reqwest::get(url) + .await?; + let mut stream = response.bytes_stream(); + // concurrently compute the sha2 & write to temp file + let mut sha = sha2::Sha256::default(); + while let Some(piece) = stream.next().await { + let piece = piece?; + let piece = piece.slice(..); + let mut reader = piece.clone().reader(); + std::io::copy(&mut reader, &mut sha)?; + + let mut reader = piece.clone().reader(); + std::io::copy(&mut reader, &mut tmp.as_file_mut())?; + } + let digest = sha.finalize(); + let digest = Checksum::try_from(digest.as_slice())?; + Ok((tmp,digest)) + +} + +#[tokio::main] +async fn main() -> color_eyre::eyre::Result<()> { + let out_dir = std::env::var("OUT_DIR").wrap_err("Missing OUT_DIR env")?; + let out_dir = std::path::PathBuf::from(out_dir); + + let builtin_car_path = out_dir.join("builtin_actors.car"); + + println!("cargo:rerun_if_changed=build.rs"); + println!("cargo:rerun_if_env_changed={}", FORCE_RERUN); + println!("cargo:rerun_if_env_changed={}", VERSION_OVERRIDE); + + println!("cargo:rerun_if_changed={}", builtin_car_path.display()); + + let tag = std::env::var(VERSION_OVERRIDE).unwrap_or_else(|_e| BUILTIN_ACTORS_TAG.to_owned()); + + let (tmp, tmp_digest) = download_builtin_actors_bundle(tag, &out_dir).await?; + + match fs::File::open(&builtin_car_path) { + Ok(f) => { + // compare digests, if mismatch, replace existing with the downloaded file + let actual = file_digest(f)?; + if tmp_digest != actual { + fs::remove_file(&builtin_car_path)?; + fs::rename(tmp.path(), &builtin_car_path)?; + } else { + println!("Nothing to do, identical file is already present"); + } + } + Err(e) => { + if let std::io::ErrorKind::NotFound = e.kind() { + // the file is not found, so a simple rename is good enough + fs::rename(tmp.path(), &builtin_car_path)?; + } else { + // other errors always must lead to an error + bail!(e) + } + } + } + + println!("Builtin actors file is ready for inclusion: {}", builtin_car_path.display()); + + Ok(()) +} diff --git a/fendermint/actors-builtin-car/src/lib.rs b/fendermint/actors-builtin-car/src/lib.rs new file mode 100644 index 000000000..154af05e9 --- /dev/null +++ b/fendermint/actors-builtin-car/src/lib.rs @@ -0,0 +1,2 @@ +/// Included builtin actors bundle +pub const BUILTIN_ACTORS_CAR: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/", "builtin_actors.car")); diff --git a/fendermint/actors-car/src/lib.rs b/fendermint/actors-car/src/lib.rs deleted file mode 100644 index 006333c8a..000000000 --- a/fendermint/actors-car/src/lib.rs +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT -mod manifest; - -pub use manifest::Manifest; diff --git a/fendermint/actors-car/Cargo.toml b/fendermint/actors-custom-car/Cargo.toml similarity index 100% rename from fendermint/actors-car/Cargo.toml rename to fendermint/actors-custom-car/Cargo.toml diff --git a/fendermint/actors-car/README.md b/fendermint/actors-custom-car/README.md similarity index 100% rename from fendermint/actors-car/README.md rename to fendermint/actors-custom-car/README.md diff --git a/fendermint/actors-car/build.rs b/fendermint/actors-custom-car/build.rs similarity index 99% rename from fendermint/actors-car/build.rs rename to fendermint/actors-custom-car/build.rs index eb125fb3c..d43ad6119 100644 --- a/fendermint/actors-car/build.rs +++ b/fendermint/actors-custom-car/build.rs @@ -357,9 +357,9 @@ fn main() -> Result<()> { fs_err::create_dir_all(&bundle_car_dest_dir)?; let bundle_car_path = bundle_car_dest_dir.join("custom_actors_bundle.car"); - + rerun_if_changed(&bundle_car_path); - + bundle_wasm_blobs_into_car(&actors, &bundle_car_path)?; Ok(()) diff --git a/fendermint/actors-custom-car/src/lib.rs b/fendermint/actors-custom-car/src/lib.rs new file mode 100644 index 000000000..78bd4c61b --- /dev/null +++ b/fendermint/actors-custom-car/src/lib.rs @@ -0,0 +1,8 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT +mod manifest; + +pub use manifest::Manifest; + +/// Included bytes for custom actor bundle, ~1.3M in size +pub const CUSTOM_ACTORS: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "custom_actors_bundle.car")); diff --git a/fendermint/actors-car/src/manifest.rs b/fendermint/actors-custom-car/src/manifest.rs similarity index 100% rename from fendermint/actors-car/src/manifest.rs rename to fendermint/actors-custom-car/src/manifest.rs diff --git a/fendermint/vm/interpreter/Cargo.toml b/fendermint/vm/interpreter/Cargo.toml index d6db17291..9fba35e81 100644 --- a/fendermint/vm/interpreter/Cargo.toml +++ b/fendermint/vm/interpreter/Cargo.toml @@ -22,7 +22,7 @@ fendermint_crypto = { path = "../../crypto" } fendermint_eth_hardhat = { path = "../../eth/hardhat" } fendermint_rpc = { path = "../../rpc" } fendermint_tracing = { path = "../../tracing" } -fendermint_actors = { path = "../../actors-car" } +fendermint_actors = { path = "../../actors-custom-car" } fendermint_actor_chainmetadata = { path = "../../actors/chainmetadata" } fendermint_actor_activity_tracker = { path = "../../actors/activity-tracker" } fendermint_actor_gas_market_eip1559 = { path = "../../actors/gas_market/eip1559" } From 43e593ab81c609baee424786f92ac6475e1ff066 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 27 Feb 2025 12:36:58 +0100 Subject: [PATCH 5/8] include path fixup --- fendermint/actors-custom-car/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fendermint/actors-custom-car/src/lib.rs b/fendermint/actors-custom-car/src/lib.rs index 78bd4c61b..e7815ab14 100644 --- a/fendermint/actors-custom-car/src/lib.rs +++ b/fendermint/actors-custom-car/src/lib.rs @@ -5,4 +5,4 @@ mod manifest; pub use manifest::Manifest; /// Included bytes for custom actor bundle, ~1.3M in size -pub const CUSTOM_ACTORS: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "custom_actors_bundle.car")); +pub const CUSTOM_ACTORS: &[u8] = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"),"/../actors/output/", "custom_actors_bundle.car")); From 7d2a09a1682c3aa994bc23c994c1bb769e26bb0a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 27 Feb 2025 12:53:48 +0100 Subject: [PATCH 6/8] y --- fendermint/actors-builtin-car/build.rs | 2 ++ fendermint/actors-builtin-car/src/lib.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/fendermint/actors-builtin-car/build.rs b/fendermint/actors-builtin-car/build.rs index ad1a91a01..4263da24e 100644 --- a/fendermint/actors-builtin-car/build.rs +++ b/fendermint/actors-builtin-car/build.rs @@ -1,3 +1,5 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT //! Download filecoin's builtin actors car file use std::io::{Write, Read}; diff --git a/fendermint/actors-builtin-car/src/lib.rs b/fendermint/actors-builtin-car/src/lib.rs index 154af05e9..82e5712b9 100644 --- a/fendermint/actors-builtin-car/src/lib.rs +++ b/fendermint/actors-builtin-car/src/lib.rs @@ -1,2 +1,4 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT /// Included builtin actors bundle pub const BUILTIN_ACTORS_CAR: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/", "builtin_actors.car")); From ff0855ddb67e9efd5062e87c64b8ccf7ac57332a Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 27 Feb 2025 15:04:14 +0100 Subject: [PATCH 7/8] fmt --- fendermint/actors-builtin-car/build.rs | 65 +++++++++++++----------- fendermint/actors-builtin-car/src/lib.rs | 3 +- fendermint/actors-custom-car/build.rs | 4 +- fendermint/actors-custom-car/src/lib.rs | 6 ++- 4 files changed, 44 insertions(+), 34 deletions(-) diff --git a/fendermint/actors-builtin-car/build.rs b/fendermint/actors-builtin-car/build.rs index 4263da24e..6d539e013 100644 --- a/fendermint/actors-builtin-car/build.rs +++ b/fendermint/actors-builtin-car/build.rs @@ -2,14 +2,14 @@ // SPDX-License-Identifier: Apache-2.0, MIT //! Download filecoin's builtin actors car file -use std::io::{Write, Read}; +use bytes::buf::Buf; +use color_eyre::eyre::{self, bail, Result, WrapErr}; use fs_err as fs; -use color_eyre::eyre::{self, bail, WrapErr, Result}; -use sha2::Digest; use futures_util::stream::StreamExt; +use sha2::Digest; +use std::io::{Read, Write}; use std::path::Path; use tempfile::NamedTempFile; -use bytes::buf::Buf; const BUILTIN_ACTORS_TAG: &str = "v15.0.0"; @@ -18,7 +18,10 @@ const FORCE_RERUN: &str = "IPC_BUILTIN_ACTORS_FORCE_FETCH"; const VERSION_OVERRIDE: &str = "IPC_BUILTIN_ACTORS_VERSION_OVERRIDE"; /// Handle `Interrupt` errors, call a closure for each read kb piece -fn read_file_piecewise Result<()>>(mut reader: R, mut f: F) -> Result<()> { +fn read_file_piecewise Result<()>>( + mut reader: R, + mut f: F, +) -> Result<()> { let mut buf = [0u8; 1024]; loop { match reader.read(&mut buf[..]) { @@ -28,12 +31,10 @@ fn read_file_piecewise Result<()>>(mut reader: R, mu Ok(n) => { f(&buf[..n])?; } - Err(e) => { - match e.kind() { - std::io::ErrorKind::Interrupted => continue, - _ => return Err(e.into()), - } - } + Err(e) => match e.kind() { + std::io::ErrorKind::Interrupted => continue, + _ => return Err(e.into()), + }, } } } @@ -50,20 +51,20 @@ fn file_digest(reader: impl Read) -> Result { } /// Convert a slice to an array -/// +/// /// The slice must have the exact length of the array size `N`. -fn slice_to_array(bytes: &[u8]) -> Result<[u8;N]> { +fn slice_to_array(bytes: &[u8]) -> Result<[u8; N]> { if bytes.len() != N { bail!("Length mismatch, execpted {}, but got {}", N, bytes.len()); } - let mut buf = [0u8;N]; + let mut buf = [0u8; N]; buf.copy_from_slice(&bytes[..N]); Ok(buf) } /// Digest wrapping type #[derive(Debug, PartialEq, Eq, Hash)] -struct Checksum([u8;32]); +struct Checksum([u8; 32]); impl TryFrom<&[u8]> for Checksum { type Error = eyre::Error; @@ -76,17 +77,19 @@ fn tempfile(out_dir: &Path) -> Result { let t = NamedTempFile::new_in(out_dir)?; Ok(t) } - + /// Download the file piecewise to a temporary file and calculate the digest -async fn download_builtin_actors_bundle(tag: impl AsRef, out_dir: &Path) -> Result<(NamedTempFile, Checksum)> { +async fn download_builtin_actors_bundle( + tag: impl AsRef, + out_dir: &Path, +) -> Result<(NamedTempFile, Checksum)> { let tag = tag.as_ref(); let mut tmp = tempfile(out_dir)?; - + let url = format!("https://github.com/filecoin-project/builtin-actors/releases/download/{tag}/builtin-actors-mainnet.car"); let url = reqwest::Url::parse(&url)?; - let response = reqwest::get(url) - .await?; + let response = reqwest::get(url).await?; let mut stream = response.bytes_stream(); // concurrently compute the sha2 & write to temp file let mut sha = sha2::Sha256::default(); @@ -101,8 +104,7 @@ async fn download_builtin_actors_bundle(tag: impl AsRef, out_dir: &Path) -> } let digest = sha.finalize(); let digest = Checksum::try_from(digest.as_slice())?; - Ok((tmp,digest)) - + Ok((tmp, digest)) } #[tokio::main] @@ -115,13 +117,13 @@ async fn main() -> color_eyre::eyre::Result<()> { println!("cargo:rerun_if_changed=build.rs"); println!("cargo:rerun_if_env_changed={}", FORCE_RERUN); println!("cargo:rerun_if_env_changed={}", VERSION_OVERRIDE); - + println!("cargo:rerun_if_changed={}", builtin_car_path.display()); - - let tag = std::env::var(VERSION_OVERRIDE).unwrap_or_else(|_e| BUILTIN_ACTORS_TAG.to_owned()); - + + let tag = std::env::var(VERSION_OVERRIDE).unwrap_or_else(|_e| BUILTIN_ACTORS_TAG.to_owned()); + let (tmp, tmp_digest) = download_builtin_actors_bundle(tag, &out_dir).await?; - + match fs::File::open(&builtin_car_path) { Ok(f) => { // compare digests, if mismatch, replace existing with the downloaded file @@ -136,15 +138,18 @@ async fn main() -> color_eyre::eyre::Result<()> { Err(e) => { if let std::io::ErrorKind::NotFound = e.kind() { // the file is not found, so a simple rename is good enough - fs::rename(tmp.path(), &builtin_car_path)?; + fs::rename(tmp.path(), &builtin_car_path)?; } else { // other errors always must lead to an error bail!(e) } } } - - println!("Builtin actors file is ready for inclusion: {}", builtin_car_path.display()); + + println!( + "Builtin actors file is ready for inclusion: {}", + builtin_car_path.display() + ); Ok(()) } diff --git a/fendermint/actors-builtin-car/src/lib.rs b/fendermint/actors-builtin-car/src/lib.rs index 82e5712b9..ea768aed3 100644 --- a/fendermint/actors-builtin-car/src/lib.rs +++ b/fendermint/actors-builtin-car/src/lib.rs @@ -1,4 +1,5 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT /// Included builtin actors bundle -pub const BUILTIN_ACTORS_CAR: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/", "builtin_actors.car")); +pub const BUILTIN_ACTORS_CAR: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/", "builtin_actors.car")); diff --git a/fendermint/actors-custom-car/build.rs b/fendermint/actors-custom-car/build.rs index d43ad6119..eb125fb3c 100644 --- a/fendermint/actors-custom-car/build.rs +++ b/fendermint/actors-custom-car/build.rs @@ -357,9 +357,9 @@ fn main() -> Result<()> { fs_err::create_dir_all(&bundle_car_dest_dir)?; let bundle_car_path = bundle_car_dest_dir.join("custom_actors_bundle.car"); - + rerun_if_changed(&bundle_car_path); - + bundle_wasm_blobs_into_car(&actors, &bundle_car_path)?; Ok(()) diff --git a/fendermint/actors-custom-car/src/lib.rs b/fendermint/actors-custom-car/src/lib.rs index e7815ab14..25e919d67 100644 --- a/fendermint/actors-custom-car/src/lib.rs +++ b/fendermint/actors-custom-car/src/lib.rs @@ -5,4 +5,8 @@ mod manifest; pub use manifest::Manifest; /// Included bytes for custom actor bundle, ~1.3M in size -pub const CUSTOM_ACTORS: &[u8] = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"),"/../actors/output/", "custom_actors_bundle.car")); +pub const CUSTOM_ACTORS: &[u8] = include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../actors/output/", + "custom_actors_bundle.car" +)); From 5f2829bf9d7546acfa8e4cec60cfaba231522dae Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Thu, 27 Feb 2025 16:39:32 +0100 Subject: [PATCH 8/8] z --- Cargo.lock | 4 +++- fendermint/actors-builtin-car/Cargo.toml | 2 +- fendermint/actors-builtin-car/src/lib.rs | 2 +- fendermint/app/Cargo.toml | 5 +++++ fendermint/app/src/main.rs | 7 ++----- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae30e7485..fa9249540 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,7 +13,7 @@ dependencies = [ ] [[package]] -name = "actors-builtin-fetch" +name = "actors-builtin-car" version = "0.1.0" dependencies = [ "bytes", @@ -3165,6 +3165,7 @@ dependencies = [ name = "fendermint_app" version = "0.1.0" dependencies = [ + "actors-builtin-car", "anyhow", "async-stm", "async-trait", @@ -3172,6 +3173,7 @@ dependencies = [ "cid", "fendermint_abci", "fendermint_actor_gas_market_eip1559", + "fendermint_actors", "fendermint_actors_api", "fendermint_app_options", "fendermint_app_settings", diff --git a/fendermint/actors-builtin-car/Cargo.toml b/fendermint/actors-builtin-car/Cargo.toml index b5e445f1c..823cdab7d 100644 --- a/fendermint/actors-builtin-car/Cargo.toml +++ b/fendermint/actors-builtin-car/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "actors-builtin-fetch" +name = "actors-builtin-car" version = "0.1.0" authors.workspace = true edition.workspace = true diff --git a/fendermint/actors-builtin-car/src/lib.rs b/fendermint/actors-builtin-car/src/lib.rs index ea768aed3..3c5cfb92a 100644 --- a/fendermint/actors-builtin-car/src/lib.rs +++ b/fendermint/actors-builtin-car/src/lib.rs @@ -1,5 +1,5 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT /// Included builtin actors bundle -pub const BUILTIN_ACTORS_CAR: &[u8] = +pub const CAR: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/", "builtin_actors.car")); diff --git a/fendermint/app/Cargo.toml b/fendermint/app/Cargo.toml index d2a2f1bff..5fed55d15 100644 --- a/fendermint/app/Cargo.toml +++ b/fendermint/app/Cargo.toml @@ -66,6 +66,11 @@ fendermint_vm_resolver = { path = "../vm/resolver" } fendermint_vm_snapshot = { path = "../vm/snapshot" } fendermint_vm_topdown = { path = "../vm/topdown" } +# .car file wrapped in a crate +actors-builtin-car = { path = "../actors-builtin-car"} +fendermint_actors = { path = "../actors-custom-car"} + + fs-err = { workspace = true } fvm = { workspace = true } fvm_ipld_blockstore = { workspace = true } diff --git a/fendermint/app/src/main.rs b/fendermint/app/src/main.rs index 99ce66b53..398391251 100644 --- a/fendermint/app/src/main.rs +++ b/fendermint/app/src/main.rs @@ -54,7 +54,6 @@ mod tests { use super::fs; use cid::Cid; use fendermint_rocksdb::{RocksDb, RocksDbConfig}; - use fendermint_vm_interpreter::fvm::bundle::bundle_path; use fvm::machine::Manifest; use fvm_ipld_car::load_car_unchecked; use fvm_ipld_encoding::CborStore; @@ -67,9 +66,7 @@ mod tests { // Not loading the actors from the library any more. It would be possible, as long as dependencies are aligned. // let bundle_car = actors_v10::BUNDLE_CAR; - let bundle_path = bundle_path(); - let bundle_car = fs::read(&bundle_path) - .unwrap_or_else(|_| panic!("failed to load bundle CAR from {bundle_path:?}")); + let bundle_car = actors_builtin_car::CAR; let dir = tempfile::Builder::new() .tempdir() @@ -81,7 +78,7 @@ mod tests { }; let db = open_db(); - let cids = load_car_unchecked(&db, bundle_car.as_slice()) + let cids = load_car_unchecked(&db, bundle_car) .await .expect("error loading bundle CAR");