diff --git a/application/apps/indexer/plugins_host/benches/plugin_parser_init.rs b/application/apps/indexer/plugins_host/benches/plugin_parser_init.rs index a030a3590..bcf275b47 100644 --- a/application/apps/indexer/plugins_host/benches/plugin_parser_init.rs +++ b/application/apps/indexer/plugins_host/benches/plugin_parser_init.rs @@ -35,7 +35,7 @@ fn plugin_parser_init(c: &mut Criterion) { .to_async(tokio::runtime::Runtime::new().unwrap()) .iter(|| async { // Creating parser will load compile and validate the provided plugin - let parser = plugins_host::PluginsParser::create( + let parser = plugins_host::PluginsParser::initialize( &settings.plugin_path, &settings.general_settings, settings.plugin_configs.clone(), diff --git a/application/apps/indexer/plugins_host/src/bytesource_shared/mod.rs b/application/apps/indexer/plugins_host/src/bytesource_shared/mod.rs index a6dda0053..43ecffb5a 100644 --- a/application/apps/indexer/plugins_host/src/bytesource_shared/mod.rs +++ b/application/apps/indexer/plugins_host/src/bytesource_shared/mod.rs @@ -129,17 +129,17 @@ impl WasmPlugin for PluginsByteSource { PluginType::ByteSource } - fn plugin_version(&mut self) -> Result { + async fn plugin_version(&mut self) -> Result { match &mut self.source { - PlugVerByteSource::Ver010(source) => source.plugin_version(), + PlugVerByteSource::Ver010(source) => source.plugin_version().await, } } - fn get_config_schemas( + async fn get_config_schemas( &mut self, ) -> Result, PluginError> { match &mut self.source { - PlugVerByteSource::Ver010(source) => source.get_config_schemas(), + PlugVerByteSource::Ver010(source) => source.get_config_schemas().await, } } } diff --git a/application/apps/indexer/plugins_host/src/lib.rs b/application/apps/indexer/plugins_host/src/lib.rs index 2bad89c96..75b638007 100644 --- a/application/apps/indexer/plugins_host/src/lib.rs +++ b/application/apps/indexer/plugins_host/src/lib.rs @@ -23,6 +23,8 @@ pub enum PluginType { ByteSource, } +// Trait is used with Chipmunk only. +#[allow(async_fn_in_trait)] pub trait WasmPlugin { /// Provides the Type of the plugin. fn get_type() -> PluginType; @@ -31,7 +33,7 @@ pub trait WasmPlugin { /// /// # Note /// This version is for the plugin only and is different from the plugin's API version. - fn plugin_version(&mut self) -> Result; + async fn plugin_version(&mut self) -> Result; /// Provides the schemas for the configurations required by the plugin, which /// will be specified by the users. @@ -39,5 +41,5 @@ pub trait WasmPlugin { /// These schemas define the expected structure, types, and constraints /// for plugin-specific configurations. The values of these configurations /// will be passed to the initializing method of the plugin. - fn get_config_schemas(&mut self) -> Result, PluginError>; + async fn get_config_schemas(&mut self) -> Result, PluginError>; } diff --git a/application/apps/indexer/plugins_host/src/parser_shared/mod.rs b/application/apps/indexer/plugins_host/src/parser_shared/mod.rs index cc5b059c3..70dc90901 100644 --- a/application/apps/indexer/plugins_host/src/parser_shared/mod.rs +++ b/application/apps/indexer/plugins_host/src/parser_shared/mod.rs @@ -42,57 +42,21 @@ enum PlugVerParser { } impl PluginsParser { - pub async fn get_info(plugin_path: PathBuf) -> Result { - let engine = get_wasm_host() - .map(|host| &host.engine) - .map_err(|err| err.to_owned())?; - - if !plugin_path.exists() { - return Err(PluginHostInitError::IO("Plugin path doesn't exist".into())); - } - - if !plugin_path.is_file() { - return Err(PluginHostInitError::IO("Plugin path is not a file".into())); - } - - let component = Component::from_file(engine, &plugin_path) - .map_err(|err| PluginHostInitError::PluginInvalid(err.to_string()))?; - - let component_types = component.component_type(); - - let export_info = component_types.exports(engine).next().ok_or_else(|| { - PluginHostInitError::PluginInvalid("Plugin doesn't have exports information".into()) - })?; - - let (interface_name, version) = export_info.0.split_once('@').ok_or_else(|| { - PluginHostInitError::PluginInvalid( - "Plugin package schema doesn't match `wit` file definitions".into(), - ) - })?; - - if interface_name != PARSER_INTERFACE_NAME { - return Err(PluginHostInitError::PluginInvalid( - "Plugin package name doesn't match `wit` file".into(), - )); - } - - let version: SemanticVersion = version.parse().map_err(|err| { - PluginHostInitError::PluginInvalid(format!("Plugin version parsing failed: {err}")) - })?; + /// Loads the plugin and extract the needed plugin info if valid. + pub async fn get_info(plugin_path: PathBuf) -> Result { + let (component, version) = Self::load(&plugin_path).await?; let plug_info = match version { SemanticVersion { major: 0, minor: 1, patch: 0, - } => { - let info = v0_1_0::parser::PluginParser::get_info(component).await?; - info - } + } => v0_1_0::parser::PluginParser::get_info(component).await?, invalid_version => { return Err(PluginHostInitError::PluginInvalid(format!( "Plugin version {invalid_version} is not supported" - ))) + )) + .into()) } }; @@ -107,20 +71,20 @@ impl PluginsParser { Ok(plugin_info) } - pub async fn create( + /// Loads and validate a plugin returning the its [`Component`] and API [`SemanticVersion`] + async fn load( plugin_path: impl AsRef, - general_config: &pl::PluginParserGeneralSetttings, - plugin_configs: Vec, - ) -> Result { + ) -> Result<(Component, SemanticVersion), PluginHostInitError> { let engine = get_wasm_host() .map(|host| &host.engine) - .map_err(|err| err.to_owned())?; + .map_err(|err| PluginHostInitError::from(err.to_owned()))?; + let plugin_path = plugin_path.as_ref(); - if !plugin_path.as_ref().exists() { + if !plugin_path.exists() { return Err(PluginHostInitError::IO("Plugin path doesn't exist".into())); } - if !plugin_path.as_ref().is_file() { + if !plugin_path.is_file() { return Err(PluginHostInitError::IO("Plugin path is not a file".into())); } @@ -149,15 +113,30 @@ impl PluginsParser { PluginHostInitError::PluginInvalid(format!("Plugin version parsing failed: {err}")) })?; + Ok((component, version)) + } + + /// Initialize parser instance with the needed configuration to be used within a parsing + /// session. + pub async fn initialize( + plugin_path: impl AsRef, + general_config: &pl::PluginParserGeneralSetttings, + plugin_configs: Vec, + ) -> Result { + let (component, version) = Self::load(&plugin_path).await?; + match version { SemanticVersion { major: 0, minor: 1, patch: 0, } => { - let parser = - v0_1_0::parser::PluginParser::create(component, general_config, plugin_configs) - .await?; + let parser = v0_1_0::parser::PluginParser::initialize( + component, + general_config, + plugin_configs, + ) + .await?; Ok(Self { parser: PlugVerParser::Ver010(parser), errors_counter: 0, @@ -169,9 +148,9 @@ impl PluginsParser { } } - pub fn get_render_options(&mut self) -> Result { + pub async fn get_render_options(&mut self) -> Result { match &mut self.parser { - PlugVerParser::Ver010(parser) => parser.get_render_options(), + PlugVerParser::Ver010(parser) => parser.get_render_options().await, } } } @@ -181,15 +160,15 @@ impl WasmPlugin for PluginsParser { PluginType::Parser } - fn plugin_version(&mut self) -> Result { + async fn plugin_version(&mut self) -> Result { match &mut self.parser { - PlugVerParser::Ver010(parser) => parser.plugin_version(), + PlugVerParser::Ver010(parser) => parser.plugin_version().await, } } - fn get_config_schemas(&mut self) -> Result, PluginError> { + async fn get_config_schemas(&mut self) -> Result, PluginError> { match &mut self.parser { - PlugVerParser::Ver010(parser) => parser.get_config_schemas(), + PlugVerParser::Ver010(parser) => parser.get_config_schemas().await, } } } diff --git a/application/apps/indexer/plugins_host/src/plugins_manager/load/mod.rs b/application/apps/indexer/plugins_host/src/plugins_manager/load/mod.rs index 362290c43..445ea0e14 100644 --- a/application/apps/indexer/plugins_host/src/plugins_manager/load/mod.rs +++ b/application/apps/indexer/plugins_host/src/plugins_manager/load/mod.rs @@ -3,11 +3,12 @@ mod paths; use std::{ fs::{self, read_to_string}, io, - path::PathBuf, + path::{Path, PathBuf}, }; use crate::{ plugins_manager::{InvalidPluginInfo, PluginState}, + plugins_shared::plugin_errors::PluginError, PluginHostInitError, PluginType, PluginsParser, }; @@ -17,8 +18,8 @@ use super::{InitError, PluginEntity, PluginMetadata}; pub async fn load_plugins() -> Result, InitError> { let plugins_dir = paths::plugins_dir()?; if !plugins_dir.exists() { - log::trace!("Plugins directory doens't exist"); - return Ok(Vec::new()); + log::trace!("Plugins directory doens't exist. Creating it..."); + fs::create_dir_all(plugins_dir)?; } let mut plugins = load_all_parsers().await?; @@ -36,7 +37,8 @@ async fn load_all_parsers() -> Result, InitError> { let parsers_dir = paths::parser_dir()?; if !parsers_dir.exists() { - log::trace!("Parsers directory deosn't exist"); + log::trace!("Parsers directory deosn't exist. Creating it ..."); + fs::create_dir_all(&parsers_dir)?; return Ok(parsers); } @@ -52,7 +54,6 @@ async fn load_all_parsers() -> Result, InitError> { /// Retrieves all directory form the given directory path fn get_dirs(dir_path: &PathBuf) -> Result, io::Error> { let dirs = fs::read_dir(dir_path)? - .into_iter() .filter_map(|entry| entry.ok().map(|e| e.path())) .filter(|path| path.is_dir()); @@ -121,7 +122,9 @@ async fn load_parser(dir: PathBuf) -> Result { let plugin_info = match PluginsParser::get_info(wasm_file).await { Ok(info) => info, // Stop the whole loading on engine errors - Err(PluginHostInitError::EngineError(err)) => return Err(err.into()), + Err(PluginError::HostInitError(PluginHostInitError::EngineError(err))) => { + return Err(err.into()) + } Err(err) => { let err_msg = format!("Loading plugin binray fail. Error: {err}"); let invalid = PluginEntity { @@ -173,7 +176,7 @@ fn parse_metadata(file: &PathBuf) -> Result { } async fn load_all_bytesources() -> Result, InitError> { - return Ok(Vec::new()); + Ok(Vec::new()) // TODO AAZ: Activate when implementing byte source // @@ -181,7 +184,8 @@ async fn load_all_bytesources() -> Result, InitError> { // // let bytesource_dir = paths::bytesource_dir()?; // if !bytesource_dir.exists() { - // log::trace!("Bytesources directory doesn't exist"); + // log::trace!("Bytesources directory doesn't exist. Creating it ..."); + // fs::create_dir_all(&bytesource_dir)?; // return Ok(bytesources); // } // @@ -194,6 +198,6 @@ async fn load_all_bytesources() -> Result, InitError> { } #[allow(unused)] -fn load_bytesource(dir: &PathBuf) -> Result { +fn load_bytesource(dir: &Path) -> Result { todo!() } diff --git a/application/apps/indexer/plugins_host/src/plugins_shared/plugin_errors.rs b/application/apps/indexer/plugins_host/src/plugins_shared/plugin_errors.rs index 930c63d67..c8f27963c 100644 --- a/application/apps/indexer/plugins_host/src/plugins_shared/plugin_errors.rs +++ b/application/apps/indexer/plugins_host/src/plugins_shared/plugin_errors.rs @@ -31,6 +31,8 @@ pub enum PluginGuestInitError { /// Represents general errors in communication with plugins #[derive(Debug, Error)] pub enum PluginError { + #[error("Error while initializing plugin host. {0}")] + HostInitError(#[from] PluginHostInitError), #[error(transparent)] WasmRunTimeError(#[from] anyhow::Error), } diff --git a/application/apps/indexer/plugins_host/src/v0_1_0/bytesource/mod.rs b/application/apps/indexer/plugins_host/src/v0_1_0/bytesource/mod.rs index 67a397b94..ecd296e3b 100644 --- a/application/apps/indexer/plugins_host/src/v0_1_0/bytesource/mod.rs +++ b/application/apps/indexer/plugins_host/src/v0_1_0/bytesource/mod.rs @@ -5,7 +5,6 @@ use std::io; use bindings::BytesourcePlugin; use bytesource_plugin_state::ByteSourcePluginState; -use futures::executor::block_on; use wasmtime::{ component::{Component, Linker}, Store, @@ -67,22 +66,22 @@ impl PluginByteSource { }) } - pub fn get_config_schemas(&mut self) -> Result, PluginError> { - let schemas = block_on( - self.plugin_bindings - .chipmunk_plugin_byte_source() - .call_get_config_schemas(&mut self.store), - )?; + pub async fn get_config_schemas(&mut self) -> Result, PluginError> { + let schemas = self + .plugin_bindings + .chipmunk_plugin_byte_source() + .call_get_config_schemas(&mut self.store) + .await?; Ok(schemas.into_iter().map(|item| item.into()).collect()) } - pub fn plugin_version(&mut self) -> Result { - let version = block_on( - self.plugin_bindings - .chipmunk_plugin_byte_source() - .call_get_version(&mut self.store), - )?; + pub async fn plugin_version(&mut self) -> Result { + let version = self + .plugin_bindings + .chipmunk_plugin_byte_source() + .call_get_version(&mut self.store) + .await?; Ok(version.into()) } diff --git a/application/apps/indexer/plugins_host/src/v0_1_0/parser/mod.rs b/application/apps/indexer/plugins_host/src/v0_1_0/parser/mod.rs index 41c55d788..c34e47610 100644 --- a/application/apps/indexer/plugins_host/src/v0_1_0/parser/mod.rs +++ b/application/apps/indexer/plugins_host/src/v0_1_0/parser/mod.rs @@ -8,7 +8,7 @@ use wasmtime::{ component::{Component, Linker}, Store, }; -use wasmtime_wasi::{ResourceTable, WasiCtxBuilder}; +use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder}; use crate::{ parser_shared::ParserRenderOptions, @@ -36,8 +36,25 @@ pub(crate) struct PluginInfo { impl PluginParser { /// Load wasm file temporally to retrieve the static plugin information defined by `wit` file - pub(crate) async fn get_info(component: Component) -> Result { - //TODO AAZ: Avoid duplicating code with `create()` method + pub(crate) async fn get_info(component: Component) -> Result { + let ctx = WasiCtxBuilder::new().build(); + let mut parser = Self::create(component, ctx).await?; + + let version = parser.plugin_version().await?; + + let render_options = parser.get_render_options().await?; + + let config_schemas = parser.get_config_schemas().await?; + + Ok(PluginInfo { + version, + config_schemas, + render_options, + }) + } + + /// Creates a new parser instance without initializing it with custom configurations. + async fn create(component: Component, ctx: WasiCtx) -> Result { let engine = get_wasm_host() .map(|host| &host.engine) .map_err(|err| err.to_owned())?; @@ -47,99 +64,72 @@ impl PluginParser { ParsePlugin::add_to_linker(&mut linker, |state| state)?; - let mut ctx = WasiCtxBuilder::new(); let resource_table = ResourceTable::new(); - let mut store = Store::new(engine, ParserPluginState::new(ctx.build(), resource_table)); + let mut store = Store::new(engine, ParserPluginState::new(ctx, resource_table)); let (plugin_bindings, _instance) = ParsePlugin::instantiate_async(&mut store, &component, &linker).await?; - let version = plugin_bindings - .chipmunk_plugin_parser() - .call_get_version(&mut store) - .await?; - - let render_options = plugin_bindings - .chipmunk_plugin_parser() - .call_get_render_options(&mut store) - .await?; - - let config_schemas = plugin_bindings - .chipmunk_plugin_parser() - .call_get_config_schemas(&mut store) - .await?; - - Ok(PluginInfo { - version: version.into(), - config_schemas: config_schemas.into_iter().map(|item| item.into()).collect(), - render_options: render_options.into(), + Ok(Self { + store, + plugin_bindings, }) } - pub async fn create( + /// Initialize parser instance with the needed configuration to be used within a parsing + /// session. + pub async fn initialize( component: Component, general_config: &pl::PluginParserGeneralSetttings, plugin_configs: Vec, ) -> Result { - let engine = get_wasm_host() - .map(|host| &host.engine) - .map_err(|err| err.to_owned())?; - - let mut linker: Linker = Linker::new(engine); - wasmtime_wasi::add_to_linker_async(&mut linker)?; - - ParsePlugin::add_to_linker(&mut linker, |state| state)?; - let mut ctx = get_wasi_ctx_builder(&plugin_configs)?; - let resource_table = ResourceTable::new(); + let ctx = ctx.build(); - let mut store = Store::new(engine, ParserPluginState::new(ctx.build(), resource_table)); + let mut parser = Self::create(component, ctx).await?; - let (plugin_bindings, _instance) = - ParsePlugin::instantiate_async(&mut store, &component, &linker).await?; let plugin_configs: Vec<_> = plugin_configs.into_iter().map(|item| item.into()).collect(); - plugin_bindings + parser + .plugin_bindings .chipmunk_plugin_parser() - .call_init(&mut store, general_config.into(), &plugin_configs) + .call_init(&mut parser.store, general_config.into(), &plugin_configs) .await? .map_err(|guest_err| { PluginHostInitError::GuestError(PluginGuestInitError::from(guest_err)) })?; - Ok(Self { - store, - plugin_bindings, - }) + Ok(parser) } - pub fn get_config_schemas(&mut self) -> Result, PluginError> { - let schemas = block_on( - self.plugin_bindings - .chipmunk_plugin_parser() - .call_get_config_schemas(&mut self.store), - )?; + pub async fn get_config_schemas(&mut self) -> Result, PluginError> { + let schemas = self + .plugin_bindings + .chipmunk_plugin_parser() + .call_get_config_schemas(&mut self.store) + .await?; Ok(schemas.into_iter().map(|item| item.into()).collect()) } - pub fn plugin_version(&mut self) -> Result { - let version = block_on( - self.plugin_bindings - .chipmunk_plugin_parser() - .call_get_version(&mut self.store), - )?; + pub async fn plugin_version(&mut self) -> Result { + let version = self + .plugin_bindings + .chipmunk_plugin_parser() + .call_get_version(&mut self.store) + .await?; Ok(version.into()) } - pub fn get_render_options(&mut self) -> Result { - let options = block_on( - self.plugin_bindings - .chipmunk_plugin_parser() - .call_get_render_options(&mut self.store), - )?; + pub async fn get_render_options(&mut self) -> Result { + let options = self + .plugin_bindings + .chipmunk_plugin_parser() + .call_get_render_options(&mut self.store) + .await?; + Ok(options.into()) } } diff --git a/application/apps/indexer/session/src/handlers/export_raw.rs b/application/apps/indexer/session/src/handlers/export_raw.rs index 274c05486..4b79479dd 100644 --- a/application/apps/indexer/session/src/handlers/export_raw.rs +++ b/application/apps/indexer/session/src/handlers/export_raw.rs @@ -143,7 +143,7 @@ async fn export( println!("------------------------------------------------------"); println!("------------- WASM parser used -----------------"); println!("------------------------------------------------------"); - let parser = PluginsParser::create( + let parser = PluginsParser::initialize( &settings.plugin_path, &settings.general_settings, settings.plugin_configs.clone(), diff --git a/application/apps/indexer/session/src/handlers/observing/mod.rs b/application/apps/indexer/session/src/handlers/observing/mod.rs index 79b17911a..0c201b7ab 100644 --- a/application/apps/indexer/session/src/handlers/observing/mod.rs +++ b/application/apps/indexer/session/src/handlers/observing/mod.rs @@ -87,7 +87,7 @@ async fn run_source_intern( println!("------------------------------------------------------"); println!("------------- WASM parser used -----------------"); println!("------------------------------------------------------"); - let parser = PluginsParser::create( + let parser = PluginsParser::initialize( &settings.plugin_path, &settings.general_settings, settings.plugin_configs.clone(), @@ -131,7 +131,7 @@ async fn run_source_intern( pl::PluginParserSettings::prototyping(proto_plugin_path, string_parser_configs); let now = std::time::Instant::now(); - let parser = PluginsParser::create( + let parser = PluginsParser::initialize( &settings.plugin_path, &settings.general_settings, settings.plugin_configs.clone(), diff --git a/application/apps/indexer/sources/benches/plugin_praser_producer.rs b/application/apps/indexer/sources/benches/plugin_praser_producer.rs index 1254300a0..7edbcdc85 100644 --- a/application/apps/indexer/sources/benches/plugin_praser_producer.rs +++ b/application/apps/indexer/sources/benches/plugin_praser_producer.rs @@ -30,7 +30,7 @@ fn plugin_parser_producer(c: &mut Criterion) { .to_async(tokio::runtime::Runtime::new().unwrap()) .iter_batched( || { - let parser = futures::executor::block_on(PluginsParser::create( + let parser = futures::executor::block_on(PluginsParser::initialize( &settings.plugin_path, &settings.general_settings, settings.plugin_configs.clone(), diff --git a/application/apps/rustcore/rs-bindings/Cargo.lock b/application/apps/rustcore/rs-bindings/Cargo.lock index c6d2c7b11..5d9e15fa7 100644 --- a/application/apps/rustcore/rs-bindings/Cargo.lock +++ b/application/apps/rustcore/rs-bindings/Cargo.lock @@ -2540,6 +2540,7 @@ name = "plugins_host" version = "0.1.0" dependencies = [ "anyhow", + "dirs 5.0.1", "futures", "log", "parsers", @@ -2547,6 +2548,7 @@ dependencies = [ "sources", "thiserror 2.0.3", "tokio", + "toml", "wasmtime", "wasmtime-wasi", ]