From 680afc7040877f69d306aa3eb2aed6e0e87fba89 Mon Sep 17 00:00:00 2001 From: acesyde Date: Sun, 26 Jan 2025 10:49:33 +0100 Subject: [PATCH 1/5] feat: add git provider and detection --- src/task/task_file_providers/mod.rs | 9 +- .../task_file_providers/remote_task_git.rs | 209 ++++++++++++++++++ .../task_file_providers/remote_task_http.rs | 15 +- 3 files changed, 225 insertions(+), 8 deletions(-) create mode 100644 src/task/task_file_providers/remote_task_git.rs diff --git a/src/task/task_file_providers/mod.rs b/src/task/task_file_providers/mod.rs index 37a364bd26..34e619bc67 100644 --- a/src/task/task_file_providers/mod.rs +++ b/src/task/task_file_providers/mod.rs @@ -1,9 +1,11 @@ use std::{fmt::Debug, path::PathBuf}; mod local_task; +mod remote_task_git; mod remote_task_http; -pub use local_task::LocalTask; +use local_task::LocalTask; +use remote_task_git::RemoteTaskGitBuilder; use remote_task_http::RemoteTaskHttpBuilder; pub trait TaskFileProvider: Debug { @@ -41,6 +43,11 @@ impl TaskFileProviders { fn get_providers(&self) -> Vec> { vec![ + Box::new( + RemoteTaskGitBuilder::new() + .with_cache(self.use_cache) + .build(), + ), Box::new( RemoteTaskHttpBuilder::new() .with_cache(self.use_cache) diff --git a/src/task/task_file_providers/remote_task_git.rs b/src/task/task_file_providers/remote_task_git.rs new file mode 100644 index 0000000000..8c31794e58 --- /dev/null +++ b/src/task/task_file_providers/remote_task_git.rs @@ -0,0 +1,209 @@ +use std::path::PathBuf; + +use regex::Regex; + +use crate::{dirs, env}; + +use super::TaskFileProvider; + +#[derive(Debug)] +pub struct RemoteTaskGitBuilder { + store_path: PathBuf, + use_cache: bool, +} + +impl RemoteTaskGitBuilder { + pub fn new() -> Self { + Self { + store_path: env::temp_dir(), + use_cache: false, + } + } + + pub fn with_cache(mut self, use_cache: bool) -> Self { + if use_cache { + self.store_path = dirs::CACHE.join("remote-git-tasks-cache"); + self.use_cache = true; + } + self + } + + pub fn build(self) -> RemoteTaskGit { + RemoteTaskGit { + storage_path: self.store_path, + is_cached: self.use_cache, + } + } +} + +#[derive(Debug)] +pub struct RemoteTaskGit { + storage_path: PathBuf, + is_cached: bool, +} + +struct GitRepoStructure { + url: String, + user: Option, + host: String, + repo: String, + query: Option, + path: String, +} + +impl GitRepoStructure { + pub fn new( + url: &str, + user: Option, + host: &str, + repo: &str, + query: Option, + path: &str, + ) -> Self { + Self { + url: url.to_string(), + user, + host: host.to_string(), + repo: repo.to_string(), + query, + path: path.to_string(), + } + } +} + +impl RemoteTaskGit { + fn get_cache_key(&self, file: &str) -> String { + "".to_string() + } + + fn detect_ssh(&self, file: &str) -> Result> { + let re = Regex::new(r"^git::ssh://((?P[^@]+)@)(?P[^/]+)/(?P[^/]+)\.git//(?P[^?]+)(\?(?P[^?]+))?$").unwrap(); + + if !re.is_match(file) { + return Err("Invalid SSH URL".into()); + } + + let captures = re.captures(file).unwrap(); + + Ok(GitRepoStructure::new( + file, + Some(captures.name("user").unwrap().as_str().to_string()), + captures.name("host").unwrap().as_str(), + captures.name("repo").unwrap().as_str(), + captures.name("query").map(|m| m.as_str().to_string()), + captures.name("path").unwrap().as_str(), + )) + } + + fn detect_https(&self, file: &str) -> Result> { + let re = Regex::new(r"^git::https://(?P[^/]+)/(?P[^/]+(?:/[^/]+)?)\.git//(?P[^?]+)(\?(?P[^?]+))?$").unwrap(); + + if !re.is_match(file) { + return Err("Invalid HTTPS URL".into()); + } + + let captures = re.captures(file).unwrap(); + + Ok(GitRepoStructure::new( + file, + None, + captures.name("host").unwrap().as_str(), + captures.name("repo").unwrap().as_str(), + captures.name("query").map(|m| m.as_str().to_string()), + captures.name("path").unwrap().as_str(), + )) + } +} + +impl TaskFileProvider for RemoteTaskGit { + fn is_match(&self, file: &str) -> bool { + if self.detect_ssh(file).is_ok() { + return true; + } + + if self.detect_https(file).is_ok() { + return true; + } + + false + } + + fn get_local_path(&self, file: &str) -> Result> { + Ok(PathBuf::new()) + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn test_valid_detect_ssh() { + let remote_task_git = RemoteTaskGitBuilder::new().build(); + + let test_cases = vec![ + "git::ssh://git@github.com:myorg/example.git//myfile?ref=v1.0.0", + "git::ssh://git@github.com:myorg/example.git//terraform/myfile?ref=master", + "git::ssh://git@github.com:myorg/example.git//terraform/myfile?depth=1", + "git::ssh://git@myserver.com/example.git//terraform/myfile", + "git::ssh://user@myserver.com/example.git//myfile?ref=master", + ]; + + for url in test_cases { + let result = remote_task_git.detect_ssh(url); + assert!(result.is_ok()); + } + } + + #[test] + fn test_invalid_detect_ssh() { + let remote_task_git = RemoteTaskGitBuilder::new().build(); + + let test_cases = vec![ + "git::ssh://myserver.com/example.git//myfile?ref=master", + "git::ssh://user@myserver.com/example.git?ref=master", + "git::ssh://user@myserver.com/example.git", + "git::https://github.com/myorg/example.git//myfile?ref=v1.0.0", + ]; + + for url in test_cases { + let result = remote_task_git.detect_ssh(url); + assert!(result.is_err()); + } + } + + #[test] + fn test_valid_detect_https() { + let remote_task_git = RemoteTaskGitBuilder::new().build(); + + let test_cases = vec![ + "git::https://github.com/myorg/example.git//myfile?ref=v1.0.0", + "git::https://github.com/myorg/example.git//terraform/myfile?ref=master", + "git::https://github.com/myorg/example.git//terraform/myfile?depth=1", + "git::https://myserver.com/example.git//terraform/myfile", + "git::https://myserver.com/example.git//myfile?ref=master", + ]; + + for url in test_cases { + let result = remote_task_git.detect_https(url); + assert!(result.is_ok()); + } + } + + #[test] + fn test_invalid_detect_https() { + let remote_task_git = RemoteTaskGitBuilder::new().build(); + + let test_cases = vec![ + "git::https://myserver.com/example.git?ref=master", + "git::https://user@myserver.com/example.git", + "git::ssh://git@github.com:myorg/example.git//myfile?ref=v1.0.0", + ]; + + for url in test_cases { + let result = remote_task_git.detect_https(url); + assert!(result.is_err()); + } + } +} diff --git a/src/task/task_file_providers/remote_task_http.rs b/src/task/task_file_providers/remote_task_http.rs index 18aecd0a7b..ada6e71885 100644 --- a/src/task/task_file_providers/remote_task_http.rs +++ b/src/task/task_file_providers/remote_task_http.rs @@ -161,13 +161,13 @@ mod tests { #[test] fn test_http_remote_task_get_local_path_with_cache() { let paths = vec![ - ("/myfile.py", "myfile.py"), - ("/subpath/myfile.sh", "myfile.sh"), - ("/myfile.sh?query=1&sdfsdf=2", "myfile.sh"), + "/myfile.py", + "/subpath/myfile.sh", + "/myfile.sh?query=1&sdfsdf=2", ]; let mut server = mockito::Server::new(); - for (request_path, not_expected_file_name) in paths { + for request_path in paths { let mocked_server = server .mock("GET", request_path) .with_status(200) @@ -176,13 +176,14 @@ mod tests { .create(); let provider = RemoteTaskHttpBuilder::new().with_cache(true).build(); - let mock = format!("{}{}", server.url(), request_path); + let request_url = format!("{}{}", server.url(), request_path); + let cache_key = provider.get_cache_key(&request_url); for _ in 0..2 { - let path = provider.get_local_path(&mock).unwrap(); + let path = provider.get_local_path(&request_url).unwrap(); assert!(path.exists()); assert!(path.is_file()); - assert!(!path.ends_with(not_expected_file_name)); + assert!(path.ends_with(&cache_key)); } mocked_server.assert(); From 632413247975e59221cf47b028f7c9a16b44aba9 Mon Sep 17 00:00:00 2001 From: acesyde Date: Sun, 26 Jan 2025 11:24:52 +0100 Subject: [PATCH 2/5] fix: unit tests --- src/task/task_file_providers/mod.rs | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/task/task_file_providers/mod.rs b/src/task/task_file_providers/mod.rs index 34e619bc67..4e39ff0380 100644 --- a/src/task/task_file_providers/mod.rs +++ b/src/task/task_file_providers/mod.rs @@ -71,7 +71,7 @@ mod tests { fn test_get_providers() { let task_file_providers = TaskFileProvidersBuilder::new().build(); let providers = task_file_providers.get_providers(); - assert_eq!(providers.len(), 2); + assert_eq!(providers.len(), 3); } #[test] @@ -82,7 +82,8 @@ mod tests { for file in cases { let provider = task_file_providers.get_provider(file); assert!(provider.is_some()); - assert!(format!("{:?}", provider.unwrap()).contains("LocalTask")); + let provider_name = format!("{:?}", provider.unwrap()); + assert!(provider_name.contains("LocalTask")); } } @@ -98,7 +99,26 @@ mod tests { for file in cases { let provider = task_file_providers.get_provider(file); assert!(provider.is_some()); - assert!(format!("{:?}", provider.unwrap()).contains("RemoteTaskHttp")); + let provider_name = format!("{:?}", provider.unwrap()); + assert!(provider_name.contains("RemoteTaskHttp")); + } + } + + #[test] + fn test_git_file_match_git_remote_task_provider() { + let task_file_providers = TaskFileProvidersBuilder::new().build(); + let cases = vec![ + "git::ssh://git@github.com:myorg/example.git//myfile?ref=v1.0.0", + "git::https://github.com/myorg/example.git//myfile?ref=v1.0.0", + "git::ssh://user@myserver.com/example.git//subfolder/myfile.py", + "git::https://myserver.com/example.git//subfolder/myfile.sh", + ]; + + for file in cases { + let provider = task_file_providers.get_provider(file); + assert!(provider.is_some()); + let provider_name = format!("{:?}", provider.unwrap()); + assert!(provider_name.contains("RemoteTaskGit")); } } } From 3558e3236085ec25f94786b134658c89ba017356 Mon Sep 17 00:00:00 2001 From: acesyde Date: Sun, 26 Jan 2025 13:29:30 +0100 Subject: [PATCH 3/5] fix: unit tests --- .../task_file_providers/remote_task_git.rs | 155 ++++++++++++++---- 1 file changed, 119 insertions(+), 36 deletions(-) diff --git a/src/task/task_file_providers/remote_task_git.rs b/src/task/task_file_providers/remote_task_git.rs index 8c31794e58..3b83ba792d 100644 --- a/src/task/task_file_providers/remote_task_git.rs +++ b/src/task/task_file_providers/remote_task_git.rs @@ -1,8 +1,9 @@ use std::path::PathBuf; use regex::Regex; +use xx::git; -use crate::{dirs, env}; +use crate::{dirs, env, hash}; use super::TaskFileProvider; @@ -42,38 +43,33 @@ pub struct RemoteTaskGit { is_cached: bool, } +#[derive(Debug, Clone)] struct GitRepoStructure { url: String, - user: Option, - host: String, - repo: String, - query: Option, + url_without_path: String, path: String, } impl GitRepoStructure { - pub fn new( - url: &str, - user: Option, - host: &str, - repo: &str, - query: Option, - path: &str, - ) -> Self { + pub fn new(url: &str, url_without_path: &str, path: &str) -> Self { Self { url: url.to_string(), - user, - host: host.to_string(), - repo: repo.to_string(), - query, + url_without_path: url_without_path.to_string(), path: path.to_string(), } } } impl RemoteTaskGit { - fn get_cache_key(&self, file: &str) -> String { - "".to_string() + fn get_cache_key(&self, repo_structure: &GitRepoStructure) -> String { + hash::hash_sha256_to_str(&repo_structure.url_without_path) + } + + fn get_repo_structure(&self, file: &str) -> GitRepoStructure { + if self.detect_ssh(file).is_ok() { + return self.detect_ssh(file).unwrap(); + } + self.detect_https(file).unwrap() } fn detect_ssh(&self, file: &str) -> Result> { @@ -85,14 +81,9 @@ impl RemoteTaskGit { let captures = re.captures(file).unwrap(); - Ok(GitRepoStructure::new( - file, - Some(captures.name("user").unwrap().as_str().to_string()), - captures.name("host").unwrap().as_str(), - captures.name("repo").unwrap().as_str(), - captures.name("query").map(|m| m.as_str().to_string()), - captures.name("path").unwrap().as_str(), - )) + let path = captures.name("path").unwrap().as_str(); + + Ok(GitRepoStructure::new(file, &file.replace(path, ""), path)) } fn detect_https(&self, file: &str) -> Result> { @@ -104,14 +95,9 @@ impl RemoteTaskGit { let captures = re.captures(file).unwrap(); - Ok(GitRepoStructure::new( - file, - None, - captures.name("host").unwrap().as_str(), - captures.name("repo").unwrap().as_str(), - captures.name("query").map(|m| m.as_str().to_string()), - captures.name("path").unwrap().as_str(), - )) + let path = captures.name("path").unwrap().as_str(); + + Ok(GitRepoStructure::new(file, &file.replace(path, ""), path)) } } @@ -129,7 +115,32 @@ impl TaskFileProvider for RemoteTaskGit { } fn get_local_path(&self, file: &str) -> Result> { - Ok(PathBuf::new()) + let repo_structure = self.get_repo_structure(file); + let cache_key = self.get_cache_key(&repo_structure); + let destination = self.storage_path.join(&cache_key); + let repo_file_path = repo_structure.path.clone(); + let full_path = destination.join(&repo_file_path); + + match self.is_cached { + true => { + trace!("Cache mode enabled"); + + if full_path.exists() { + return Ok(full_path); + } + } + false => { + trace!("Cache mode disabled"); + + if full_path.exists() { + crate::file::remove_dir(full_path)?; + } + } + } + + let git_cloned = git::clone(repo_structure.url.as_str(), destination)?; + + Ok(git_cloned.dir.join(&repo_file_path)) } } @@ -206,4 +217,76 @@ mod tests { assert!(result.is_err()); } } + + #[test] + fn test_compare_ssh_get_cache_key() { + let remote_task_git = RemoteTaskGitBuilder::new().build(); + + let test_cases = vec![ + ( + "git::ssh://git@github.com:myorg/example.git//myfile?ref=v1.0.0", + "git::ssh://git@github.com:myorg/example.git//myfile?ref=v2.0.0", + false, + ), + ( + "git::ssh://git@github.com:myorg/example.git//myfile?ref=v1.0.0", + "git::ssh://user@myserver.com/example.git//myfile?ref=master", + false, + ), + ( + "git::ssh://git@github.com/example.git//myfile?ref=v1.0.0", + "git::ssh://git@github.com/example.git//subfolder/mysecondfile?ref=v1.0.0", + true, + ), + ( + "git::ssh://git@github.com:myorg/example.git//myfile?ref=v1.0.0", + "git::ssh://git@github.com:myorg/example.git//subfolder/mysecondfile?ref=v1.0.0", + true, + ), + ]; + + for (first_url, second_url, expected) in test_cases { + let first_repo = remote_task_git.detect_ssh(first_url).unwrap(); + let second_repo = remote_task_git.detect_ssh(second_url).unwrap(); + let first_cache_key = remote_task_git.get_cache_key(&first_repo); + let second_cache_key = remote_task_git.get_cache_key(&second_repo); + assert_eq!(expected, first_cache_key == second_cache_key); + } + } + + #[test] + fn test_compare_https_get_cache_key() { + let remote_task_git = RemoteTaskGitBuilder::new().build(); + + let test_cases = vec![ + ( + "git::https://github.com/myorg/example.git//myfile?ref=v1.0.0", + "git::https://github.com/myorg/example.git//myfile?ref=v2.0.0", + false, + ), + ( + "git::https://github.com/myorg/example.git//myfile?ref=v1.0.0", + "git::https://bitbucket.com/myorg/example.git//myfile?ref=v1.0.0", + false, + ), + ( + "git::https://github.com/myorg/example.git//myfile?ref=v1.0.0", + "git::https://github.com/myorg/example.git//subfolder/myfile?ref=v1.0.0", + true, + ), + ( + "git::https://github.com/example.git//myfile?ref=v1.0.0", + "git::https://github.com/example.git//subfolder/myfile?ref=v1.0.0", + true, + ), + ]; + + for (first_url, second_url, expected) in test_cases { + let first_repo = remote_task_git.detect_https(first_url).unwrap(); + let second_repo = remote_task_git.detect_https(second_url).unwrap(); + let first_cache_key = remote_task_git.get_cache_key(&first_repo); + let second_cache_key = remote_task_git.get_cache_key(&second_repo); + assert_eq!(expected, first_cache_key == second_cache_key); + } + } } From eb8f27c6bfc3b439e61b9a76ebc13173f4be4556 Mon Sep 17 00:00:00 2001 From: acesyde Date: Sun, 26 Jan 2025 13:48:15 +0100 Subject: [PATCH 4/5] feat: refacto some code --- .../task_file_providers/remote_task_git.rs | 1 + .../task_file_providers/remote_task_http.rs | 36 ++++++++----------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/task/task_file_providers/remote_task_git.rs b/src/task/task_file_providers/remote_task_git.rs index 3b83ba792d..3fbf6337a6 100644 --- a/src/task/task_file_providers/remote_task_git.rs +++ b/src/task/task_file_providers/remote_task_git.rs @@ -126,6 +126,7 @@ impl TaskFileProvider for RemoteTaskGit { trace!("Cache mode enabled"); if full_path.exists() { + debug!("Using cached file: {:?}", full_path); return Ok(full_path); } } diff --git a/src/task/task_file_providers/remote_task_http.rs b/src/task/task_file_providers/remote_task_http.rs index ada6e71885..5c2394127d 100644 --- a/src/task/task_file_providers/remote_task_http.rs +++ b/src/task/task_file_providers/remote_task_http.rs @@ -50,6 +50,7 @@ impl RemoteTaskHttp { file: &str, destination: &PathBuf, ) -> Result<(), Box> { + trace!("Downloading file: {}", file); HTTP.download_file(file, destination, None)?; file::make_executable(destination)?; Ok(()) @@ -71,37 +72,29 @@ impl TaskFileProvider for RemoteTaskHttp { } fn get_local_path(&self, file: &str) -> Result> { + let cache_key = self.get_cache_key(file); + let destination = self.storage_path.join(&cache_key); + match self.is_cached { true => { trace!("Cache mode enabled"); - let cache_key = self.get_cache_key(file); - let destination = self.storage_path.join(&cache_key); if destination.exists() { debug!("Using cached file: {:?}", destination); return Ok(destination); } - - debug!("Downloading file: {}", file); - self.download_file(file, &destination)?; - Ok(destination) } false => { trace!("Cache mode disabled"); - let url = url::Url::parse(file)?; - let filename = url - .path_segments() - .and_then(|segments| segments.last()) - .unwrap(); - let destination = env::temp_dir().join(filename); if destination.exists() { file::remove_file(&destination)?; } - self.download_file(file, &destination)?; - Ok(destination) } } + + self.download_file(file, &destination)?; + Ok(destination) } } @@ -130,13 +123,13 @@ mod tests { #[test] fn test_http_remote_task_get_local_path_without_cache() { let paths = vec![ - ("/myfile.py", "myfile.py"), - ("/subpath/myfile.sh", "myfile.sh"), - ("/myfile.sh?query=1&sdfsdf=2", "myfile.sh"), + "/myfile.py", + "/subpath/myfile.sh", + "/myfile.sh?query=1&sdfsdf=2", ]; let mut server = mockito::Server::new(); - for (request_path, expected_file_name) in paths { + for request_path in paths { let mocked_server: mockito::Mock = server .mock("GET", request_path) .with_status(200) @@ -145,13 +138,14 @@ mod tests { .create(); let provider = RemoteTaskHttpBuilder::new().build(); - let mock = format!("{}{}", server.url(), request_path); + let request_url = format!("{}{}", server.url(), request_path); + let cache_key = provider.get_cache_key(&request_url); for _ in 0..2 { - let local_path = provider.get_local_path(&mock).unwrap(); + let local_path = provider.get_local_path(&request_url).unwrap(); assert!(local_path.exists()); assert!(local_path.is_file()); - assert!(local_path.ends_with(expected_file_name)); + assert!(local_path.ends_with(&cache_key)); } mocked_server.assert(); From 9beb58bd24b878be91256761a3ab588f92b8199e Mon Sep 17 00:00:00 2001 From: acesyde Date: Sun, 26 Jan 2025 14:05:11 +0100 Subject: [PATCH 5/5] feat: documentation --- docs/tasks/toml-tasks.md | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/tasks/toml-tasks.md b/docs/tasks/toml-tasks.md index 4200c15239..dd7be7f4af 100644 --- a/docs/tasks/toml-tasks.md +++ b/docs/tasks/toml-tasks.md @@ -334,13 +334,47 @@ file = 'scripts/release.sh' # execute an external script ### Remote tasks -Task files can be fetched via http: +Task files can be fetched remotely with multiple protocols: + +#### HTTP ```toml [tasks.build] file = "https://example.com/build.sh" ``` +Please note that the file will be downloaded and executed. Make sure you trust the source. + +#### Git + +::: code-group + +```toml [ssh] +[tasks.build] +file = "git::ssh://git@github.com:myorg/example.git//myfile?ref=v1.0.0" +``` + +```toml [https] +[tasks.build] +file = "git::https://github.com/myorg/example.git//myfile?ref=v1.0.0" +``` + +::: + +Url format must follow these patterns `git:::////?` + +Required fields: + +- `protocol`: The git repository URL. +- `url`: The git repository URL. +- `path`: The path to the file in the repository. + +Optional fields: + +- `query`: The git reference (branch, tag, commit). + +#### Cache + Each task file is cached in the `MISE_CACHE_DIR` directory. If the file is updated, it will not be re-downloaded unless the cache is cleared. :::tip