diff --git a/source/citnames_rs/src/compilation.rs b/source/citnames_rs/src/compilation.rs index ea218e19..991fbc53 100644 --- a/source/citnames_rs/src/compilation.rs +++ b/source/citnames_rs/src/compilation.rs @@ -31,6 +31,7 @@ impl TryFrom for Vec { fn try_from(value: CompilerCall) -> Result { match value { CompilerCall::Compile { working_dir, compiler, flags, sources, output } => + // todo: should error when sources are empty? sources.iter() .map(|source| -> Result { let mut arguments: Vec = vec![]; diff --git a/source/citnames_rs/src/configuration.rs b/source/citnames_rs/src/configuration.rs index 7701afdf..fff3bf95 100644 --- a/source/citnames_rs/src/configuration.rs +++ b/source/citnames_rs/src/configuration.rs @@ -131,8 +131,6 @@ pub enum DuplicateFilterFields { All, } - - impl TryFrom for DuplicateFilterFields { type Error = String; @@ -150,173 +148,164 @@ impl TryFrom for DuplicateFilterFields { } } -pub mod io { +#[cfg(test)] +mod test { + use crate::{vec_of_pathbuf, vec_of_strings}; use super::*; - /// Load the content of the given stream and parse it as Configuration. - pub fn from_reader(reader: impl std::io::Read) -> Result { - serde_json::from_reader(reader) + #[test] + fn test_full_config() { + let content: &[u8] = br#"{ + "output": { + "format": { + "command_as_array": true, + "drop_output_field": false + }, + "content": { + "include_only_existing_source": false, + "duplicate_filter_fields": "all", + "paths_to_include": ["sources"], + "paths_to_exclude": ["tests"] + } + }, + "compilation": { + "compilers_to_recognize": [ + { + "executable": "/usr/local/bin/clang", + "flags_to_add": ["-Dfoo=bar"], + "flags_to_remove": ["-Wall"] + } + ], + "compilers_to_exclude": [ + "clang" + ] + } + }"#; + + let result = serde_json::from_reader(content).unwrap(); + + let expected = Configuration { + output: Output { + format: Format { + command_as_array: true, + drop_output_field: false, + }, + content: Content { + include_only_existing_source: false, + duplicate_filter_fields: DuplicateFilterFields::All, + paths_to_include: vec_of_pathbuf!["sources"], + paths_to_exclude: vec_of_pathbuf!["tests"], + }, + }, + compilation: Compilation { + compilers_to_recognize: vec![ + CompilerToRecognize { + executable: PathBuf::from("/usr/local/bin/clang"), + flags_to_add: vec_of_strings!["-Dfoo=bar"], + flags_to_remove: vec_of_strings!["-Wall"], + } + ], + compilers_to_exclude: vec_of_pathbuf!["clang"], + }, + }; + + assert_eq!(expected, result); } - #[cfg(test)] - mod test { - use crate::{vec_of_pathbuf, vec_of_strings}; - use super::*; - - #[test] - fn test_full_config() { - let content: &[u8] = br#"{ - "output": { - "format": { - "command_as_array": true, - "drop_output_field": false - }, - "content": { - "include_only_existing_source": false, - "duplicate_filter_fields": "all", - "paths_to_include": ["sources"], - "paths_to_exclude": ["tests"] - } + #[test] + fn test_only_output_config() { + let content: &[u8] = br#"{ + "output": { + "format": { + "command_as_array": false }, - "compilation": { - "compilers_to_recognize": [ - { - "executable": "/usr/local/bin/clang", - "flags_to_add": ["-Dfoo=bar"], - "flags_to_remove": ["-Wall"] - } - ], - "compilers_to_exclude": [ - "clang" - ] + "content": { + "duplicate_filter_fields": "file" } - }"#; + } + }"#; - let result = from_reader(content).unwrap(); + let result = serde_json::from_reader(content).unwrap(); - let expected = Configuration { - output: Output { - format: Format { - command_as_array: true, - drop_output_field: false, - }, - content: Content { - include_only_existing_source: false, - duplicate_filter_fields: DuplicateFilterFields::All, - paths_to_include: vec_of_pathbuf!["sources"], - paths_to_exclude: vec_of_pathbuf!["tests"], - }, + let expected = Configuration { + output: Output { + format: Format { + command_as_array: false, + drop_output_field: false, }, - compilation: Compilation { - compilers_to_recognize: vec![ - CompilerToRecognize { - executable: PathBuf::from("/usr/local/bin/clang"), - flags_to_add: vec_of_strings!["-Dfoo=bar"], - flags_to_remove: vec_of_strings!["-Wall"], - } - ], - compilers_to_exclude: vec_of_pathbuf!["clang"], + content: Content { + include_only_existing_source: false, + duplicate_filter_fields: DuplicateFilterFields::FileOnly, + paths_to_include: vec_of_pathbuf![], + paths_to_exclude: vec_of_pathbuf![], }, - }; + }, + compilation: Compilation::default(), + }; - assert_eq!(expected, result); - } + assert_eq!(expected, result); + } - #[test] - fn test_only_output_config() { - let content: &[u8] = br#"{ - "output": { - "format": { - "command_as_array": false + #[test] + fn test_compilation_only_config() { + let content: &[u8] = br#"{ + "compilation": { + "compilers_to_recognize": [ + { + "executable": "/usr/local/bin/clang" }, - "content": { - "duplicate_filter_fields": "file" + { + "executable": "/usr/local/bin/clang++" } - } - }"#; - - let result = from_reader(content).unwrap(); - - let expected = Configuration { - output: Output { - format: Format { - command_as_array: false, - drop_output_field: false, + ], + "compilers_to_exclude": [ + "clang", "clang++" + ] + } + }"#; + + let result = serde_json::from_reader(content).unwrap(); + + let expected = Configuration { + output: Output::default(), + compilation: Compilation { + compilers_to_recognize: vec![ + CompilerToRecognize { + executable: PathBuf::from("/usr/local/bin/clang"), + flags_to_add: vec![], + flags_to_remove: vec![], }, - content: Content { - include_only_existing_source: false, - duplicate_filter_fields: DuplicateFilterFields::FileOnly, - paths_to_include: vec_of_pathbuf![], - paths_to_exclude: vec_of_pathbuf![], + CompilerToRecognize { + executable: PathBuf::from("/usr/local/bin/clang++"), + flags_to_add: vec![], + flags_to_remove: vec![], }, - }, - compilation: Compilation::default(), - }; + ], + compilers_to_exclude: vec_of_pathbuf!["clang", "clang++"], + }, + }; - assert_eq!(expected, result); - } + assert_eq!(expected, result); + } - #[test] - fn test_compilation_only_config() { - let content: &[u8] = br#"{ - "compilation": { - "compilers_to_recognize": [ - { - "executable": "/usr/local/bin/clang" - }, - { - "executable": "/usr/local/bin/clang++" - } - ], - "compilers_to_exclude": [ - "clang", "clang++" - ] - } - }"#; - - let result = from_reader(content).unwrap(); - - let expected = Configuration { - output: Output::default(), - compilation: Compilation { - compilers_to_recognize: vec![ - CompilerToRecognize { - executable: PathBuf::from("/usr/local/bin/clang"), - flags_to_add: vec![], - flags_to_remove: vec![], - }, - CompilerToRecognize { - executable: PathBuf::from("/usr/local/bin/clang++"), - flags_to_add: vec![], - flags_to_remove: vec![], - }, - ], - compilers_to_exclude: vec_of_pathbuf!["clang", "clang++"], + #[test] + fn test_failing_config() { + let content: &[u8] = br#"{ + "output": { + "format": { + "command_as_array": false }, - }; - - assert_eq!(expected, result); - } - - #[test] - fn test_failing_config() { - let content: &[u8] = br#"{ - "output": { - "format": { - "command_as_array": false - }, - "content": { - "duplicate_filter_fields": "files" - } + "content": { + "duplicate_filter_fields": "files" } - }"#; + } + }"#; - let result = from_reader(content); + let result: Result = serde_json::from_reader(content); - assert!(result.is_err()); + assert!(result.is_err()); - let message = result.unwrap_err().to_string(); - assert_eq!("Unknown value \"files\" for duplicate filter at line 8 column 21", message); - } + let message = result.unwrap_err().to_string(); + assert_eq!("Unknown value \"files\" for duplicate filter at line 8 column 17", message); } } \ No newline at end of file diff --git a/source/citnames_rs/src/events.rs b/source/citnames_rs/src/events.rs index b525d2b2..7d5e6cac 100644 --- a/source/citnames_rs/src/events.rs +++ b/source/citnames_rs/src/events.rs @@ -34,12 +34,12 @@ pub fn from_reader(reader: impl std::io::Read) -> impl Iterator { match into_execution(value) { - None => vec![], - Some(result) => vec![Ok(result)] + None => None, + Some(result) => Some(Ok(result)) } } Err(error) => - vec![Err(error)], + Some(Err(error)), } }) } diff --git a/source/citnames_rs/src/main.rs b/source/citnames_rs/src/main.rs index 215332a4..98750025 100644 --- a/source/citnames_rs/src/main.rs +++ b/source/citnames_rs/src/main.rs @@ -19,21 +19,22 @@ extern crate core; -use std::fs::OpenOptions; -use std::io::stdin; +use std::fs::{File, OpenOptions}; +use std::io::{BufReader, BufWriter, stdin, stdout}; +use std::path::PathBuf; use std::thread; use anyhow::{anyhow, Context, Result}; use clap::{arg, ArgAction, ArgMatches, command}; -use crossbeam_channel::{bounded, Sender, unbounded}; +use crossbeam_channel::{bounded, Sender}; use json_compilation_db::{Entry, read, write}; -use log::{error, LevelFilter}; +use log::LevelFilter; use simple_logger::SimpleLogger; -use crate::configuration::Configuration; -use crate::configuration::io::from_reader; +use crate::configuration::{Compilation, Configuration}; use crate::execution::Execution; use crate::filter::EntryPredicate; +use crate::tools::{RecognitionResult, Semantic, Tool}; mod configuration; mod events; @@ -78,11 +79,9 @@ fn main() -> Result<()> { .unwrap_or(&false); if input == "-" && config.unwrap_or("+") == "-" { - error!("Both input and config reading the standard input."); return Err(anyhow!("Both input and config reading the standard input.")); } if *append && output == "-" { - error!("Append can't applied to the standard output."); return Err(anyhow!("Append can't applied to the standard output.")); } @@ -90,50 +89,72 @@ fn main() -> Result<()> { let config = match config { Some("-") => { let reader = stdin(); - from_reader(reader).context("Failed to read configuration from stdin")? + serde_json::from_reader(reader) + .context("Failed to read configuration from stdin")? } Some(file) => { let reader = OpenOptions::new().read(true).open(file)?; - from_reader(reader) + serde_json::from_reader(reader) .with_context(|| format!("Failed to read configuration from file: {}", file))? } None => Configuration::default(), }; + // todo: When many verbose is requested, should do less parallel in order to debug easier. run(config, input.into(), output.into(), *append) } fn run(config: Configuration, input: String, output: String, append: bool) -> Result<()> { - let (snd, rcv) = bounded::(100); + let (snd, rcv) = bounded::(32); + // Start reading entries (in a new thread), and send them across the channel. + let (compilation_config, output_config) = (config.compilation, config.output); let captured_output = output.to_owned(); thread::spawn(move || { - new_entries_from_events(&snd, input.as_str()).expect(""); - if append { - old_entries_from_previous_run(&snd, captured_output.as_str()).expect(""); + new_entries_from_events(&snd, input.as_str(), &compilation_config) + .expect("Failed to process events."); + + if PathBuf::from(&captured_output).is_file() && append { + old_entries_from_previous_run(&snd, captured_output.as_str()) + .expect("Failed to process existing compilation database"); } drop(snd); }); - // consume the entry streams here - let temp = format!("{}.tmp", &output); - { - let filter: EntryPredicate = config.output.content.into(); - let file = OpenOptions::new().write(true).open(&temp)?; - write(file, rcv.iter().filter(filter))?; - } - std::fs::remove_file(&output)?; - std::fs::rename(&output, &temp)?; + // Start writing the entries (from the channel) to the output. + let filter: EntryPredicate = output_config.content.into(); + let entries = rcv.iter() + .inspect(|entry| log::debug!("{:?}", entry)) + .filter(filter); + match output.as_str() { + "-" | "/dev/stdout" => + write(stdout(), entries)?, + _ => { + let temp = format!("{}.tmp", &output); + // Create scope for the file, so it will be closed when the scope is over. + { + let file = File::create(&temp) + .with_context(|| format!("Failed to create file: {}", temp))?; + let buffer = BufWriter::new(file); + write(buffer, entries)?; + } + std::fs::rename(&temp, &output) + .with_context(|| format!("Failed to rename file from '{}' to '{}'.", temp, output))?; + } + }; Ok(()) } fn old_entries_from_previous_run(sink: &Sender, source: &str) -> Result<()> { let mut count: u32 = 0; - let reader = OpenOptions::new().read(true).open(source)?; - let events = read(reader); - for event in events { + + let file = OpenOptions::new().read(true).open(source) + .with_context(|| format!("Failed to open file: {}", source))?; + let buffer = BufReader::new(file); + + for event in read(buffer) { match event { Ok(value) => { sink.send(value)?; @@ -150,12 +171,65 @@ fn old_entries_from_previous_run(sink: &Sender, source: &str) -> Result<( Ok(()) } -fn new_entries_from_events(_sink: &Sender, _input: &str) -> Result { - let (_exec_snd, _exec_rcv) = unbounded::(); +fn new_entries_from_events(sink: &Sender, input: &str, config: &Compilation) -> Result<()> { + let (snd, rcv) = bounded::(128); + + // Start worker threads, which will process executions and create compilation database entry. + for _ in 0..num_cpus::get() { + let tool: Box = config.into(); + let captured_sink = sink.clone(); + let captured_source = rcv.clone(); + thread::spawn(move || { + for execution in captured_source.into_iter() { + let result = tool.recognize(&execution); + match result { + RecognitionResult::Recognized(Ok(Semantic::Compiler(call))) => { + log::debug!("execution recognized as compiler call, {:?} : {:?}", call, execution); + let entries: Result> = call.try_into(); + match entries { + Ok(entries) => for entry in entries { + captured_sink.send(entry).expect("") + } + Err(error) => + log::debug!("can't convert into compilation entry: {}", error), + } + } + RecognitionResult::Recognized(Ok(_)) => + log::debug!("execution recognized: {:?}", execution), + RecognitionResult::Recognized(Err(reason)) => + log::debug!("execution recognized with failure, {:?} : {:?}", reason, execution), + RecognitionResult::NotRecognized => + log::debug!("execution not recognized: {:?}", execution), + } + } + }); + } - // log::debug!("Found {new_entries} entries"); + // Start sending execution events from the given file. + let buffer: BufReader> = match input { + "-" | "/dev/stdin" => + BufReader::new(Box::new(stdin())), + _ => { + let file = OpenOptions::new().read(true).open(input) + .with_context(|| format!("Failed to open file: {}", input))?; + BufReader::new(Box::new(file)) + } + }; - Ok(0) + for execution in events::from_reader(buffer) { + match execution { + Ok(value) => { + snd.send(value)?; + } + Err(_error) => { + // todo + log::error!("") + } + } + } + drop(snd); + + Ok(()) } fn configure_logging(matches: &ArgMatches) -> Result<()> { diff --git a/source/citnames_rs/src/tools.rs b/source/citnames_rs/src/tools.rs index 58ee197a..847b64c0 100644 --- a/source/citnames_rs/src/tools.rs +++ b/source/citnames_rs/src/tools.rs @@ -30,7 +30,7 @@ mod wrapper; mod matchers; /// This abstraction is representing a tool which is known by us. -pub(crate) trait Tool { +pub(crate) trait Tool: Send { /// A tool has a potential to recognize a command execution and identify /// the semantic of that command. fn recognize(&self, _: &Execution) -> RecognitionResult; diff --git a/source/citnames_rs/src/tools/wrapper.rs b/source/citnames_rs/src/tools/wrapper.rs index 6c001eb7..ee059f95 100644 --- a/source/citnames_rs/src/tools/wrapper.rs +++ b/source/citnames_rs/src/tools/wrapper.rs @@ -17,8 +17,11 @@ along with this program. If not, see . */ +use std::path::PathBuf; use crate::execution::Execution; -use crate::tools::{RecognitionResult, Tool}; +use crate::tools::{CompilerCall, RecognitionResult, Semantic, Tool}; +use crate::tools::matchers::source::looks_like_a_source_file; +use crate::tools::RecognitionResult::{NotRecognized, Recognized}; pub(crate) struct Wrapper {} @@ -29,7 +32,40 @@ impl Wrapper { } impl Tool for Wrapper { - fn recognize(&self, _: &Execution) -> RecognitionResult { - todo!() + // fixme: this is just a quick and dirty implementation. + fn recognize(&self, x: &Execution) -> RecognitionResult { + if x.executable == PathBuf::from("/usr/bin/g++") { + let mut flags = vec![]; + let mut sources = vec![]; + + // find sources and filter out requested flags. + for argument in x.arguments.iter().skip(1) { + if looks_like_a_source_file(argument.as_str()) { + sources.push(PathBuf::from(argument)); + } else { + flags.push(argument.clone()); + } + } + + if sources.is_empty() { + Recognized(Err(String::from("source file is not found"))) + } else { + Recognized( + Ok( + Semantic::Compiler( + CompilerCall::Compile { + working_dir: x.working_dir.clone(), + compiler: x.executable.clone(), + flags, + sources, + output: None, + } + ) + ) + ) + } + } else { + NotRecognized + } } }