Skip to content

Commit

Permalink
feat: add git remote task provider (#4233)
Browse files Browse the repository at this point in the history
## Description

Add a git remote task provider

This provider handle ssh and https generic format

Specific patterns like github or bitbucket are not translated because
the ROI is too low and add useless complex code.

User must define a generic url with path

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
  • Loading branch information
acesyde and autofix-ci[bot] authored Feb 8, 2025
1 parent ba43871 commit 1bf4a0f
Show file tree
Hide file tree
Showing 15 changed files with 633 additions and 52 deletions.
36 changes: 35 additions & 1 deletion docs/tasks/toml-tasks.md
Original file line number Diff line number Diff line change
Expand Up @@ -334,13 +334,47 @@ file = 'scripts/release.sh' # execute an external script

### Remote tasks

Task files can be fetched via http:
Task files can be fetched remotely with multiple protocols:

#### HTTP

```toml
[tasks.build]
file = "https://example.com/build.sh"
```

Please note that the file will be downloaded and executed. Make sure you trust the source.

#### Git <Badge type="warning" text="experimental" />

::: code-group

```toml [ssh]
[tasks.build]
file = "git::ssh://[email protected]:myorg/example.git//myfile?ref=v1.0.0"
```

```toml [https]
[tasks.build]
file = "git::https://github.com/myorg/example.git//myfile?ref=v1.0.0"
```

:::

Url format must follow these patterns `git::<protocol>://<url>//<path>?<ref>`

Required fields:

- `protocol`: The git repository URL.
- `url`: The git repository URL.
- `path`: The path to the file in the repository.

Optional fields:

- `ref`: The git reference (branch, tag, commit).

#### Cache

Each task file is cached in the `MISE_CACHE_DIR` directory. If the file is updated, it will not be re-downloaded unless the cache is cleared.

:::tip
Expand Down
1 change: 1 addition & 0 deletions e2e/run_test
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ within_isolated_env() {
TEST_ROOT="$SCRIPT_DIR" \
TEST_SCRIPT="$TEST_SCRIPT" \
TMPDIR="$TEST_TMPDIR" \
EXCLUDE_FROM_CI="${CI:-}" \
"$@" || return $?
}

Expand Down
43 changes: 43 additions & 0 deletions e2e/tasks/test_task_remote_git_https
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
#!/usr/bin/env bash

cargo init --name hello_cargo

#################################################################################
# Test remote tasks with no ref
#################################################################################

cat <<EOF >mise.toml
[tasks.remote_lint_https_latest]
file = "git::https://github.com/jdx/mise.git//xtasks/lint/clippy"
EOF

assert_contains "mise tasks" "remote_lint_https_latest"
assert_succeed "mise run remote_lint_https_latest" # Remote task should be downloaded

mise cache clear # Clear cache to force redownload

assert_succeed "MISE_TASK_REMOTE_NO_CACHE=true mise run remote_lint_https_latest" # Remote task should be redownloaded

assert_succeed "mise run remote_lint_https_latest --no-cache" # Remote task should be redownloaded

assert_succeed "mise run remote_lint_https_latest" # Cache should be used

#################################################################################
# Test remote tasks with with ref
#################################################################################

cat <<EOF >mise.toml
[tasks.remote_lint_https_ref]
file = "git::https://github.com/jdx/mise.git//xtasks/lint/clippy?ref=v2025.1.17"
EOF

assert_contains "mise tasks" "remote_lint_https_ref"
assert_succeed "mise run remote_lint_https_ref" # Remote task should be downloaded

mise cache clear # Clear cache to force redownload

assert_succeed "MISE_TASK_REMOTE_NO_CACHE=true mise run remote_lint_https_ref" # Remote task should be redownloaded

assert_succeed "mise run remote_lint_https_ref --no-cache" # Remote task should be redownloaded

assert_succeed "mise run remote_lint_https_ref" # Cache should be used
48 changes: 48 additions & 0 deletions e2e/tasks/test_task_remote_git_ssh
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
#!/usr/bin/env bash

if [ -n "$EXCLUDE_FROM_CI" ]; then
echo "This test is not supported in CI, because it requires a SSH key to be added to the GitHub account"
exit 0
fi

cargo init --name hello_cargo

#################################################################################
# Test remote tasks with no ref
#################################################################################

cat <<EOF >mise.toml
[tasks.remote_lint_ssh_latest]
file = "git::ssh://[email protected]:jdx/mise.git//xtasks/lint/clippy"
EOF

assert_contains "mise tasks" "remote_lint_ssh_latest"
assert_succeed "mise run remote_lint_ssh_latest" # Remote task should be downloaded

mise cache clear # Clear cache to force redownload

assert_succeed "MISE_TASK_REMOTE_NO_CACHE=true mise run remote_lint_ssh_latest" # Remote task should be redownloaded

assert_succeed "mise run remote_lint_ssh_latest --no-cache" # Remote task should be redownloaded

assert_succeed "mise run remote_lint_ssh_latest" # Cache should be used

#################################################################################
# Test remote tasks with with ref
#################################################################################

cat <<EOF >mise.toml
[tasks.remote_lint_ssh_ref]
file = "git::ssh://[email protected]:jdx/mise.git//xtasks/lint/clippy?ref=v2025.1.17"
EOF

assert_contains "mise tasks" "remote_lint_ssh_ref"
assert_succeed "mise run remote_lint_ssh_ref" # Remote task should be downloaded

mise cache clear # Clear cache to force redownload

assert_succeed "MISE_TASK_REMOTE_NO_CACHE=true mise run remote_lint_ssh_ref" # Remote task should be redownloaded

assert_succeed "mise run remote_lint_ssh_ref --no-cache" # Remote task should be redownloaded

assert_succeed "mise run remote_lint_ssh_ref" # Cache should be used
4 changes: 4 additions & 0 deletions mise.lock
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,15 @@ backend = "aqua:rhysd/actionlint"
[tools.actionlint.checksums]
"actionlint_1.7.7_darwin_arm64.tar.gz" = "sha256:2693315b9093aeacb4ebd91a993fea54fc215057bf0da2659056b4bc033873db"
"actionlint_1.7.7_linux_amd64.tar.gz" = "sha256:023070a287cd8cccd71515fedc843f1985bf96c436b7effaecce67290e7e0757"
"actionlint_1.7.7_linux_arm64.tar.gz" = "sha256:401942f9c24ed71e4fe71b76c7d638f66d8633575c4016efd2977ce7c28317d0"

[tools.bun]
version = "1.2.2"
backend = "core:bun"

[tools.bun.checksums]
"bun-darwin-aarch64.zip" = "sha256:c4d58e06c5c33885b526f4d91a38ca9ebdb9fc3fb4cd547f7d3302055c98e41c"
"bun-linux-aarch64.zip" = "sha256:d1dbaa3e9af24549fad92bdbe4fb21fa53302cd048a8f004e85a240984c93d4d"
"bun-linux-x64-baseline.zip" = "sha256:cad7756a6ee16f3432a328f8023fc5cd431106822eacfa6d6d3afbad6fdc24db"

[tools.cargo-binstall]
Expand All @@ -20,6 +22,7 @@ backend = "aqua:cargo-bins/cargo-binstall"

[tools.cargo-binstall.checksums]
"cargo-binstall-aarch64-apple-darwin.zip" = "sha256:97ce4a2f18181f052dda266b042d8bb220e48ffe40ca75e796ae4c5e418b9e01"
"cargo-binstall-aarch64-unknown-linux-musl.tgz" = "sha256:b8c32b1b007482f42f6c4b5f8cfeb168f9674ec6448bfa29ae0c4ba01b7a370b"
"cargo-binstall-x86_64-unknown-linux-musl.tgz" = "sha256:74d7c647c7e60bb8464fa551702fdd38a7241f5cedb2c4edc3b11639cd1dae47"

[tools."cargo:cargo-edit"]
Expand Down Expand Up @@ -115,6 +118,7 @@ version = "2.6.0"
backend = "ubi:slsa-framework/slsa-verifier"

[tools.slsa-verifier.checksums]
slsa-verifier-linux-aarch64 = "sha256:92b28eb2db998f9a6a048336928b29a38cb100076cd587e443ca0a2543d7c93d"
slsa-verifier-linux-x86_64 = "sha256:1c9c0d6a272063f3def6d233fa3372adbaff1f5a3480611a07c744e73246b62d"
slsa-verifier-macos-aarch64 = "sha256:8740e66832fd48bbaa479acd5310986b876ff545460add0cb4a087aec056189c"
"slsa-verifier.exe-windows-x86_64" = "sha256:37ca29ad748e8ea7be76d3ae766e8fa505362240431f6ea7f0648c727e2f2507"
Expand Down
4 changes: 2 additions & 2 deletions src/aqua/aqua_registry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use crate::backend::aqua;
use crate::backend::aqua::{arch, os};
use crate::config::SETTINGS;
use crate::duration::{DAILY, WEEKLY};
use crate::git::Git;
use crate::git::{CloneOptions, Git};
use crate::{dirs, file, hashmap, http};
use expr::{Context, Parser, Program, Value};
use eyre::{eyre, ContextCompat, Result};
Expand Down Expand Up @@ -176,7 +176,7 @@ impl AquaRegistry {
fetch_latest_repo(&repo)?;
} else if let Some(aqua_registry_url) = &SETTINGS.aqua.registry_url {
info!("cloning aqua registry to {path:?}");
repo.clone(aqua_registry_url, None)?;
repo.clone(aqua_registry_url, CloneOptions::default())?;
repo_exists = true;
}
Ok(Self { path, repo_exists })
Expand Down
7 changes: 5 additions & 2 deletions src/backend/spm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use crate::backend::Backend;
use crate::cli::args::BackendArg;
use crate::cmd::CmdLineRunner;
use crate::config::Settings;
use crate::git::Git;
use crate::git::{CloneOptions, Git};
use crate::install_context::InstallContext;
use crate::toolset::ToolVersion;
use crate::{dirs, file, github};
Expand Down Expand Up @@ -95,7 +95,10 @@ impl SPMBackend {
package_repo.url.as_str(),
repo.dir.display(),
);
repo.clone(package_repo.url.as_str(), Some(&ctx.pr))?;
repo.clone(
package_repo.url.as_str(),
CloneOptions::default().pr(&ctx.pr),
)?;
}
debug!("Checking out revision: {revision}");
repo.update_tag(revision.to_string())?;
Expand Down
40 changes: 35 additions & 5 deletions src/git.rs
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ impl Git {
Ok((prev_rev, post_rev))
}

pub fn clone(&self, url: &str, pr: Option<&Box<dyn SingleReport>>) -> Result<()> {
pub fn clone(&self, url: &str, options: CloneOptions) -> Result<()> {
debug!("cloning {} to {}", url, self.dir.display());
if let Some(parent) = self.dir.parent() {
file::mkdirp(parent)?;
Expand All @@ -139,18 +139,30 @@ impl Git {
err
),
}
if let Some(pr) = pr {
if let Some(pr) = &options.pr {
// in order to prevent hiding potential password prompt, just disable the progress bar
pr.abandon();
}
CmdLineRunner::new("git")

let mut cmd = CmdLineRunner::new("git")
.arg("clone")
.arg("-q")
.arg("--depth")
.arg("1")
.arg(url)
.arg(&self.dir)
.execute()?;
.arg(&self.dir);

if let Some(branch) = &options.branch {
cmd = cmd.args([
"-b",
branch,
"--single-branch",
"-c",
"advice.detachedHead=false",
]);
}

cmd.execute()?;
Ok(())
}

Expand Down Expand Up @@ -265,3 +277,21 @@ impl Debug for Git {
f.debug_struct("Git").field("dir", &self.dir).finish()
}
}

#[derive(Default)]
pub struct CloneOptions<'a> {
pr: Option<&'a Box<dyn SingleReport>>,
branch: Option<String>,
}

impl<'a> CloneOptions<'a> {
pub fn pr(mut self, pr: &'a Box<dyn SingleReport>) -> Self {
self.pr = Some(pr);
self
}

pub fn branch(mut self, branch: &str) -> Self {
self.branch = Some(branch.to_string());
self
}
}
4 changes: 2 additions & 2 deletions src/plugins/asdf_plugin.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use crate::config::{Config, Settings, SETTINGS};
use crate::errors::Error::PluginNotInstalled;
use crate::file::{display_path, remove_all};
use crate::git::Git;
use crate::git::{CloneOptions, Git};
use crate::plugins::{Plugin, PluginType, Script, ScriptManager};
use crate::result::Result;
use crate::timeout::run_with_timeout;
Expand Down Expand Up @@ -333,7 +333,7 @@ Plugins could support local directories in the future but for now a symlink is r
}
let git = Git::new(&self.plugin_path);
pr.set_message(format!("clone {repo_url}"));
git.clone(&repo_url, Some(pr))?;
git.clone(&repo_url, CloneOptions::default().pr(pr))?;
if let Some(ref_) = &repo_ref {
pr.set_message(format!("check out {ref_}"));
git.update(Some(ref_.to_string()))?;
Expand Down
8 changes: 6 additions & 2 deletions src/plugins/core/python.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use crate::cli::args::BackendArg;
use crate::cmd::CmdLineRunner;
use crate::config::{Config, SETTINGS};
use crate::file::{display_path, TarOptions};
use crate::git::Git;
use crate::git::{CloneOptions, Git};
use crate::http::{HTTP, HTTP_FETCH};
use crate::install_context::InstallContext;
use crate::toolset::{ToolRequest, ToolVersion, Toolset};
Expand Down Expand Up @@ -66,7 +66,11 @@ impl PythonPlugin {
file::create_dir_all(self.python_build_path().parent().unwrap())?;
let git = Git::new(self.python_build_path());
let pr = ctx.map(|ctx| &ctx.pr);
git.clone(&SETTINGS.python.pyenv_repo, pr)?;
let mut clone_options = CloneOptions::default();
if let Some(pr) = pr {
clone_options = clone_options.pr(pr);
}
git.clone(&SETTINGS.python.pyenv_repo, clone_options)?;
Ok(())
}
fn update_python_build(&self) -> eyre::Result<()> {
Expand Down
14 changes: 11 additions & 3 deletions src/plugins/core/ruby.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use crate::cmd::CmdLineRunner;
use crate::config::{Config, Settings, SETTINGS};
use crate::duration::DAILY;
use crate::env::PATH_KEY;
use crate::git::Git;
use crate::git::{CloneOptions, Git};
use crate::github::GithubRelease;
use crate::http::{HTTP, HTTP_FETCH};
use crate::install_context::InstallContext;
Expand Down Expand Up @@ -81,7 +81,11 @@ impl RubyPlugin {
file::remove_all(&tmp)?;
file::create_dir_all(tmp.parent().unwrap())?;
let git = Git::new(tmp.clone());
git.clone(&SETTINGS.ruby.ruby_build_repo, pr)?;
let mut clone_options = CloneOptions::default();
if let Some(pr) = pr {
clone_options = clone_options.pr(pr);
}
git.clone(&SETTINGS.ruby.ruby_build_repo, clone_options)?;

cmd!("sh", "install.sh")
.env("PREFIX", self.ruby_build_path())
Expand Down Expand Up @@ -123,7 +127,11 @@ impl RubyPlugin {
file::remove_all(&tmp)?;
file::create_dir_all(tmp.parent().unwrap())?;
let git = Git::new(tmp.clone());
git.clone(&settings.ruby.ruby_install_repo, pr)?;
let mut clone_options = CloneOptions::default();
if let Some(pr) = pr {
clone_options = clone_options.pr(pr);
}
git.clone(&settings.ruby.ruby_install_repo, clone_options)?;

cmd!("make", "install")
.env("PREFIX", self.ruby_install_path())
Expand Down
7 changes: 4 additions & 3 deletions src/plugins/vfox_plugin.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use crate::file::{display_path, remove_all};
use crate::git::Git;
use crate::git::{CloneOptions, Git};
use crate::plugins::{Plugin, PluginType};
use crate::result::Result;
use crate::tokio::RUNTIME;
Expand Down Expand Up @@ -133,7 +133,8 @@ impl Plugin for VfoxPlugin {
let url = self.get_repo_url()?;
trace!("Cloning vfox plugin: {url}");
let pr = mpr.add(&format!("clone vfox plugin {}", url));
self.repo().clone(url.as_str(), Some(&pr))?;
self.repo()
.clone(url.as_str(), CloneOptions::default().pr(&pr))?;
}
Ok(())
}
Expand Down Expand Up @@ -208,7 +209,7 @@ Plugins could support local directories in the future but for now a symlink is r
}
let git = Git::new(&self.plugin_path);
pr.set_message(format!("clone {repo_url}"));
git.clone(&repo_url, Some(pr))?;
git.clone(&repo_url, CloneOptions::default().pr(pr))?;
if let Some(ref_) = &repo_ref {
pr.set_message(format!("git update {ref_}"));
git.update(Some(ref_.to_string()))?;
Expand Down
Loading

0 comments on commit 1bf4a0f

Please sign in to comment.