diff --git a/Cargo.toml b/Cargo.toml index f73c88c9..214c7de8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,4 @@ [workspace] resolver = "2" members = ["crates/*", "terraphim_server", "desktop/src-tauri"] -default-members = ["terraphim_server"] +default-members = ["terraphim_server"] \ No newline at end of file diff --git a/Earthfile b/Earthfile index c08817d5..3d349d34 100644 --- a/Earthfile +++ b/Earthfile @@ -246,8 +246,7 @@ docker-scratch: ENTRYPOINT ["./terraphim_server"] SAVE IMAGE aks/terraphim_server:scratch - -docs-pages: +docs-deps: FROM +install-native RUN cargo install mdbook RUN cargo install mdbook-epub @@ -256,9 +255,15 @@ docs-pages: RUN cargo install --git https://github.com/typst/typst typst-cli RUN cargo install --git https://github.com/terraphim/mdbook-typst.git RUN cargo install mdbook-mermaid + RUN cargo install mdbook-alerts RUN apt update && apt install libvips-dev -y RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash RUN bash -c "source $HOME/.nvm/nvm.sh && nvm install 20 && npm install -g netlify-cli" + + + +docs-pages: + FROM +docs-deps COPY --keep-ts docs /docs WORKDIR /docs RUN mdbook --version diff --git a/Layerfile b/Layerfile index cad60055..61cbad10 100644 --- a/Layerfile +++ b/Layerfile @@ -15,7 +15,7 @@ RUN rustup component add rustfmt RUN cargo install ripgrep WORKDIR /code COPY . . -RUN ./desktop/scripts/yarn_and_build.sh +RUN ./scripts/yarn_and_build.sh ENV TERRAPHIM_SERVER_HOSTNAME=0.0.0.0:8000 RUN cargo build --release RUN git clone --depth 1 https://github.com/terraphim/INCOSE-Systems-Engineering-Handbook.git /tmp/system_operator/ diff --git a/crates/terraphim_config/src/lib.rs b/crates/terraphim_config/src/lib.rs index f0d7fc67..3aad0e52 100644 --- a/crates/terraphim_config/src/lib.rs +++ b/crates/terraphim_config/src/lib.rs @@ -18,7 +18,6 @@ use ulid::Ulid; pub type Result = std::result::Result; use opendal::Result as OpendalResult; -use url::Url; type PersistenceResult = std::result::Result; @@ -66,9 +65,7 @@ pub struct Role { /// The relevance function used to rank search results pub relevance_function: RelevanceFunction, pub theme: String, - #[serde(rename = "serverUrl")] - pub server_url: Url, - pub kg: KnowledgeGraph, + pub kg: Option, pub haystacks: Vec, #[serde(flatten)] pub extra: AHashMap, @@ -282,12 +279,16 @@ impl ConfigState { let mut roles = AHashMap::new(); for (name, role) in &config.roles { let role_name = name.to_lowercase(); - let automata_url = role.kg.automata_path.clone(); - log::info!("Loading Role `{}` - URL: {}", role_name, automata_url); - - let thesaurus = load_thesaurus(&automata_url).await?; - let rolegraph = RoleGraph::new(role_name.clone(), thesaurus).await?; - roles.insert(role_name, RoleGraphSync::from(rolegraph)); + log::info!("Creating role {}", role_name); + if role.kg.is_some() { + let automata_url = role.kg.as_ref().unwrap().automata_path.clone(); + log::info!("Loading Role `{}` - URL: {}", role_name, automata_url); + let thesaurus = load_thesaurus(&automata_url).await?; + let rolegraph = RoleGraph::new(role_name.clone(), thesaurus).await?; + roles.insert(role_name, RoleGraphSync::from(rolegraph)); + } else { + log::info!("Skipping KG due to None settings for role {}", role_name); + } } Ok(ConfigState { @@ -319,22 +320,21 @@ impl ConfigState { Ok(()) } - /// Search documents in rolegraph index, which match the search query + /// Search documents in rolegraph index using matching Knowledge Graph + /// If knowledge graph isn't defined for the role, RoleGraph isn't build for the role pub async fn search_indexed_documents( &self, search_query: &SearchQuery, + role: &Role, ) -> Vec { log::debug!("search_documents: {:?}", search_query); - let current_config_state = self.config.lock().await.clone(); - let default_role = current_config_state.default_role.clone(); - // if role is not provided, use the default role from the config - let role = search_query.role.clone().unwrap_or(default_role); log::debug!("Role for search_documents: {:#?}", role); - let role_name = role.to_lowercase(); + let role_name = role.name.to_lowercase(); + log::debug!("Role name for searching {role_name}"); + log::debug!("All roles defined {:?}", self.roles.clone().into_keys()); let role = self.roles.get(&role_name).unwrap().lock().await; - let documents = role .query_graph( &search_query.search_term, @@ -410,14 +410,7 @@ mod tests { name: "Default".to_string(), relevance_function: RelevanceFunction::TitleScorer, theme: "spacelab".to_string(), - server_url: Url::parse("http://localhost:8000/documents/search").unwrap(), - kg: KnowledgeGraph { - automata_path: AutomataPath::local_example(), - input_type: KnowledgeGraphInputType::Markdown, - path: PathBuf::from("~/pkm"), - public: true, - publish: true, - }, + kg: None, haystacks: vec![Haystack { path: PathBuf::from("localsearch"), service: ServiceType::Ripgrep, @@ -432,14 +425,7 @@ mod tests { name: "Engineer".to_string(), relevance_function: RelevanceFunction::TitleScorer, theme: "lumen".to_string(), - server_url: Url::parse("http://localhost:8000/documents/search").unwrap(), - kg: KnowledgeGraph { - automata_path: AutomataPath::local_example(), - input_type: KnowledgeGraphInputType::Markdown, - path: PathBuf::from("~/pkm"), - public: true, - publish: true, - }, + kg: None, haystacks: vec![Haystack { path: PathBuf::from("localsearch"), service: ServiceType::Ripgrep, @@ -454,14 +440,13 @@ mod tests { name: "System Operator".to_string(), relevance_function: RelevanceFunction::TitleScorer, theme: "superhero".to_string(), - server_url: Url::parse("http://localhost:8000/documents/search").unwrap(), - kg: KnowledgeGraph { + kg: Some(KnowledgeGraph { automata_path: AutomataPath::local_example(), input_type: KnowledgeGraphInputType::Markdown, path: PathBuf::from("~/pkm"), public: true, publish: true, - }, + }), haystacks: vec![Haystack { path: PathBuf::from("/tmp/system_operator/pages/"), service: ServiceType::Ripgrep, @@ -500,14 +485,13 @@ mod tests { name: "Father".to_string(), relevance_function: RelevanceFunction::TitleScorer, theme: "lumen".to_string(), - server_url: Url::parse("http://localhost:8080").unwrap(), - kg: KnowledgeGraph { + kg: Some(KnowledgeGraph { automata_path: AutomataPath::local_example(), input_type: KnowledgeGraphInputType::Markdown, path: PathBuf::from("~/pkm"), public: true, publish: true, - }, + }), haystacks: vec![Haystack { path: PathBuf::from("localsearch"), service: ServiceType::Ripgrep, diff --git a/crates/terraphim_middleware/src/thesaurus/mod.rs b/crates/terraphim_middleware/src/thesaurus/mod.rs index e9cd6f9a..882cd61c 100644 --- a/crates/terraphim_middleware/src/thesaurus/mod.rs +++ b/crates/terraphim_middleware/src/thesaurus/mod.rs @@ -70,7 +70,7 @@ pub async fn build_thesaurus_from_haystack( tokio::fs::write(&thesaurus_path, thesaurus_json).await?; log::debug!("Thesaurus written to {:#?}", thesaurus_path); - role.kg.automata_path = AutomataPath::Local(thesaurus_path); + role.kg.as_mut().unwrap().automata_path = AutomataPath::Local(thesaurus_path); } Ok(()) } diff --git a/crates/terraphim_rolegraph/src/lib.rs b/crates/terraphim_rolegraph/src/lib.rs index 14ee513a..8f902346 100644 --- a/crates/terraphim_rolegraph/src/lib.rs +++ b/crates/terraphim_rolegraph/src/lib.rs @@ -197,7 +197,6 @@ impl RoleGraph { let mut ranked_documents = results.into_iter().collect::>(); ranked_documents.sort_by_key(|(_, doc)| std::cmp::Reverse(doc.rank)); - ranked_documents.sort_by_key(|(_, doc)| std::cmp::Reverse(doc.id.clone())); let documents: Vec<_> = ranked_documents .into_iter() diff --git a/crates/terraphim_service/src/lib.rs b/crates/terraphim_service/src/lib.rs index 45edc273..619194e2 100644 --- a/crates/terraphim_service/src/lib.rs +++ b/crates/terraphim_service/src/lib.rs @@ -74,24 +74,28 @@ impl<'a> TerraphimService { match role.relevance_function { RelevanceFunction::TitleScorer => { log::debug!("Searching haystack with title scorer"); - let indexed_docs: Vec = self - .config_state - .search_indexed_documents(search_query) - .await; - let documents = index.get_documents(indexed_docs); - // Sort the documents by relevance + let documents = index.get_all_documents(); + log::debug!("Sorting documents by relevance"); + // Sort the documents by relevance let documents = score::sort_documents(search_query, documents); - - Ok(documents) + let total_length = documents.len(); + let mut docs_ranked = Vec::new(); + for (idx, doc) in documents.iter().enumerate() { + let document: &mut terraphim_types::Document = &mut doc.clone(); + let rank = terraphim_types::Rank::new((total_length - idx).try_into().unwrap()); + document.rank = Some(rank); + docs_ranked.push(document.clone()); + } + Ok(docs_ranked) } RelevanceFunction::TerraphimGraph => { self.build_thesaurus(search_query).await?; let scored_index_docs: Vec = self .config_state - .search_indexed_documents(search_query) + .search_indexed_documents(search_query, &role) .await; // Apply to ripgrep vector of document output diff --git a/crates/terraphim_service/src/score/mod.rs b/crates/terraphim_service/src/score/mod.rs index 4484c033..72f86b9e 100644 --- a/crates/terraphim_service/src/score/mod.rs +++ b/crates/terraphim_service/src/score/mod.rs @@ -28,9 +28,9 @@ pub fn sort_documents(search_query: &SearchQuery, documents: Vec) -> V // Score the documents let mut results = scorer.score(&query, documents).unwrap(); - + //FIXME: results.score and rank solve the same purpose. Results score shall be normalized into rank and mapped to IndexedDocument results.rescore(|doc| query.similarity.similarity(&query.name, &doc.title)); - + log::debug!("Rescore results {:#?}", results); results .into_vec() .iter() diff --git a/crates/terraphim_types/src/lib.rs b/crates/terraphim_types/src/lib.rs index fbb8865b..df1157f6 100644 --- a/crates/terraphim_types/src/lib.rs +++ b/crates/terraphim_types/src/lib.rs @@ -439,6 +439,11 @@ impl Index { } documents } + /// Returns all documents from the index for scorer without graph embeddings + pub fn get_all_documents(&self) -> Vec { + let documents: Vec = self.values().cloned().collect::>(); + documents + } /// Get a document from the index (if it exists in the index) pub fn get_document(&self, doc: &IndexedDocument) -> Option { diff --git a/desktop/Earthfile b/desktop/Earthfile index 55ffd412..b19c1619 100644 --- a/desktop/Earthfile +++ b/desktop/Earthfile @@ -2,7 +2,7 @@ VERSION --cache-persist-option --global-cache 0.7 PROJECT applied-knowledge-systems/terraphim-project FROM ghcr.io/terraphim/terraphim_builder_native:latest -WORKDIR frontend +WORKDIR /code/desktop deps: # COPY package.json tsconfig.json yarn.lock vite.config.ts tsconfig.node.json index.html ./ @@ -10,9 +10,10 @@ deps: # COPY --dir src src # COPY --dir public public RUN oro apply -q || true - RUN pkgx +yarnpkg.com yarn + RUN /code/desktop/scripts/yarn_and_build.sh build: FROM +deps - RUN pkgx +yarnpkg.com yarn run build + # RUN pkgx +yarnpkg.com yarn run build + SAVE ARTIFACT dist /dist AS LOCAL dist \ No newline at end of file diff --git a/desktop/public/logo_bw_square.png b/desktop/public/logo_bw_square.png new file mode 100644 index 00000000..821f51f7 Binary files /dev/null and b/desktop/public/logo_bw_square.png differ diff --git a/desktop/scripts/yarn_and_build.sh b/desktop/scripts/yarn_and_build.sh index be34d2a9..0ced2a2c 100755 --- a/desktop/scripts/yarn_and_build.sh +++ b/desktop/scripts/yarn_and_build.sh @@ -1,7 +1,9 @@ -!/usr/bin/env bash +current_dir=${PWD} +echo "Current folder ${current_dir}" # this is to install and build front end inside bionic curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash bash -c "source $HOME/.nvm/nvm.sh && nvm install 16.15.1" bash -c "source $HOME/.nvm/nvm.sh && npm install -g yarn" -bash -c "source $HOME/.nvm/nvm.sh && cd /code/desktop && yarn && yarn build" -cp -Rv /code/desktop/dist /code/terraphim_server/ \ No newline at end of file +bash -c "source $HOME/.nvm/nvm.sh && cd ${current_dir} && yarn && yarn build" +echo "${current_dir}/dist ${current_dir}/../terraphim_server/" +cp -Rv ${current_dir}/dist ${current_dir}/../terraphim_server/ \ No newline at end of file diff --git a/desktop/src-tauri/src/config.rs b/desktop/src-tauri/src/config.rs index e974dd0f..47d8d304 100644 --- a/desktop/src-tauri/src/config.rs +++ b/desktop/src-tauri/src/config.rs @@ -1,7 +1,6 @@ use std::path::PathBuf; use ahash::AHashMap; -use tauri::Url; use terraphim_automata::AutomataPath; use terraphim_config::{ Config, ConfigBuilder, Haystack, KnowledgeGraph, Role, ServiceType, TerraphimConfigError, @@ -11,7 +10,8 @@ use terraphim_types::{KnowledgeGraphInputType, RelevanceFunction}; /// The path to the default haystack directory // TODO: Replace this with a file-based config loader based on `twelf` in the // future -const DEFAULT_HAYSTACK_PATH: &str = "../../docs/"; +// const DEFAULT_HAYSTACK_PATH: &str = "docs/src/"; +const DEFAULT_HAYSTACK_PATH: &str = "terraphim_server/fixtures"; /// Load the default config /// @@ -22,7 +22,10 @@ pub(crate) fn load_config() -> Result { // Create the path to the default haystack directory // by concating the current directory with the default haystack path - let docs_path = std::env::current_dir()?.join(DEFAULT_HAYSTACK_PATH); + let mut docs_path = std::env::current_dir().unwrap(); + docs_path.pop(); + docs_path.pop(); + docs_path = docs_path.join(DEFAULT_HAYSTACK_PATH); println!("Docs path: {:?}", docs_path); ConfigBuilder::new() @@ -34,14 +37,7 @@ pub(crate) fn load_config() -> Result { name: "Default".to_string(), relevance_function: RelevanceFunction::TitleScorer, theme: "spacelab".to_string(), - server_url: Url::parse("http://localhost:8000/documents/search").unwrap(), - kg: KnowledgeGraph { - automata_path: automata_path.clone(), - input_type: KnowledgeGraphInputType::Markdown, - path: PathBuf::from("~/pkm"), - public: true, - publish: true, - }, + kg: None, haystacks: vec![Haystack { path: docs_path.clone(), service: ServiceType::Ripgrep, @@ -56,14 +52,7 @@ pub(crate) fn load_config() -> Result { name: "Engineer".to_string(), relevance_function: RelevanceFunction::TitleScorer, theme: "lumen".to_string(), - server_url: Url::parse("http://localhost:8000/documents/search").unwrap(), - kg: KnowledgeGraph { - automata_path: automata_path.clone(), - input_type: KnowledgeGraphInputType::Markdown, - path: PathBuf::from("~/pkm"), - public: true, - publish: true, - }, + kg: None, haystacks: vec![Haystack { path: docs_path.clone(), service: ServiceType::Ripgrep, @@ -78,14 +67,13 @@ pub(crate) fn load_config() -> Result { name: "System Operator".to_string(), relevance_function: RelevanceFunction::TitleScorer, theme: "superhero".to_string(), - server_url: Url::parse("http://localhost:8000/documents/search").unwrap(), - kg: KnowledgeGraph { + kg: Some(KnowledgeGraph { automata_path, input_type: KnowledgeGraphInputType::Markdown, path: PathBuf::from("~/pkm"), public: true, publish: true, - }, + }), haystacks: vec![Haystack { path: docs_path, service: ServiceType::Ripgrep, diff --git a/desktop/src/App.svelte b/desktop/src/App.svelte index 4f20b7c8..144c5859 100644 --- a/desktop/src/App.svelte +++ b/desktop/src/App.svelte @@ -62,7 +62,7 @@ -