diff --git a/.github/workflows/truss-transfer-cli-release.yaml b/.github/workflows/truss-transfer-cli-release.yaml index bfa50ecbf..8b661580b 100644 --- a/.github/workflows/truss-transfer-cli-release.yaml +++ b/.github/workflows/truss-transfer-cli-release.yaml @@ -15,18 +15,25 @@ defaults: jobs: build-and-upload-cli: - name: Build and Upload CLI Binary - runs-on: ubuntu-latest + name: Build and Upload CLI Binaries + strategy: + matrix: + os: [ubuntu-22.04] + target: + - x86_64-unknown-linux-musl + - aarch64-unknown-linux-musl + # - i686-unknown-linux-musl + # - armv7-unknown-linux-musleabihf + # - powerpc64le-unknown-linux-musl + runs-on: ${{ matrix.os }} env: - VERSION: ${{ github.ref_name }} # or ${GITHUB_REF##*/}, see note below - OS: linux - ARCH: x86_64 + NAME: truss-transfer-cli-${{ github.ref_name }}-linux-${{ matrix.target }} steps: - name: Checkout Code uses: actions/checkout@v4 - name: Install musl-tools - run: sudo apt-get update && sudo apt-get install -y musl-tools + run: sudo apt-get update && sudo apt-get install -y musl-tools libssl-dev libatomic-ops-dev - name: Install Rust with musl target uses: actions-rs/toolchain@v1 @@ -36,20 +43,20 @@ jobs: components: rust-src - name: Add musl Target - run: rustup target add x86_64-unknown-linux-musl + run: rustup target add ${{ matrix.target }} - name: Build Statically Linked CLI Binary run: | - cargo build --release --target x86_64-unknown-linux-musl --features cli --bin truss_transfer_cli + cargo build --release --target ${{ matrix.target }} --features cli --bin truss_transfer_cli mkdir -p dist/cli - cp target/x86_64-unknown-linux-musl/release/truss_transfer_cli dist/cli/truss_transfer_cli-${VERSION}-${OS}-${ARCH} + cp target/${{ matrix.target }}/release/truss_transfer_cli dist/cli/${{ env.NAME }} - name: Test CLI Binary run: | sudo mkdir -p /bptr sudo chown $(whoami):$(whoami) /bptr cp ./example-bptr-manifest.json /bptr/bptr-manifest - dist/cli/truss_transfer_cli-${VERSION}-${OS}-${ARCH} ./example_bptr_resolved + dist/cli/${{ env.NAME }} ./example_bptr_resolved if [ ! -d "./example_bptr_resolved" ]; then echo "❌ Test failed: output directory not created." exit 1 diff --git a/.github/workflows/truss-transfer-test.yml b/.github/workflows/truss-transfer-test.yml index 37a46fa89..71060ace4 100644 --- a/.github/workflows/truss-transfer-test.yml +++ b/.github/workflows/truss-transfer-test.yml @@ -58,7 +58,7 @@ jobs: ## CLI direct install - name: Install musl-tools - run: sudo apt-get update && sudo apt-get install -y musl-tools + run: sudo apt-get update && sudo apt-get install -y musl-tools libssl-dev libatomic-ops-dev - name: Install Rust with musl target uses: actions-rs/toolchain@v1 @@ -72,7 +72,6 @@ jobs: - name: Build Statically Linked CLI Binary run: | - apt-get update && apt-get install -y musl-tools musl-gcc cargo build --release --target x86_64-unknown-linux-musl --features cli --bin truss_transfer_cli chmod +x target/x86_64-unknown-linux-musl/release/truss_transfer_cli ./target/x86_64-unknown-linux-musl/release/truss_transfer_cli "./tmp_cli_no_fs_cache/test" diff --git a/truss-transfer/Cargo.lock b/truss-transfer/Cargo.lock index 150efae4b..dbc8102df 100644 --- a/truss-transfer/Cargo.lock +++ b/truss-transfer/Cargo.lock @@ -83,12 +83,6 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.9.0" @@ -110,12 +104,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "cfg_aliases" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" - [[package]] name = "chrono" version = "0.4.39" @@ -279,10 +267,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", - "js-sys", "libc", "wasi", - "wasm-bindgen", ] [[package]] @@ -397,7 +383,6 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots", ] [[package]] @@ -825,15 +810,6 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" -[[package]] -name = "ppv-lite86" -version = "0.2.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" -dependencies = [ - "zerocopy", -] - [[package]] name = "proc-macro2" version = "1.0.93" @@ -906,58 +882,6 @@ dependencies = [ "syn", ] -[[package]] -name = "quinn" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" -dependencies = [ - "bytes", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror", - "tokio", - "tracing", -] - -[[package]] -name = "quinn-proto" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" -dependencies = [ - "bytes", - "getrandom", - "rand", - "ring", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.52.0", -] - [[package]] name = "quote" version = "1.0.38" @@ -967,36 +891,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] - [[package]] name = "reqwest" version = "0.12.12" @@ -1024,17 +918,13 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "quinn", - "rustls", "rustls-pemfile", - "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", "tokio-native-tls", - "tokio-rustls", "tokio-util", "tower", "tower-service", @@ -1043,7 +933,6 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", "windows-registry", ] @@ -1068,12 +957,6 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" -[[package]] -name = "rustc-hash" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" - [[package]] name = "rustix" version = "0.38.43" @@ -1094,7 +977,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" dependencies = [ "once_cell", - "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -1115,9 +997,6 @@ name = "rustls-pki-types" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" -dependencies = [ - "web-time", -] [[package]] name = "rustls-webpki" @@ -1331,26 +1210,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "thiserror" -version = "2.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tinystr" version = "0.7.6" @@ -1361,21 +1220,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "tinyvec" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "tokio" version = "1.43.0" @@ -1484,7 +1328,7 @@ dependencies = [ [[package]] name = "truss_transfer" -version = "0.0.1-rc3" +version = "0.0.1-rc4" dependencies = [ "anyhow", "bytes", @@ -1666,25 +1510,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki-roots" -version = "0.26.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "windows-core" version = "0.52.0" @@ -1842,27 +1667,6 @@ dependencies = [ "synstructure", ] -[[package]] -name = "zerocopy" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" -dependencies = [ - "byteorder", - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "zerofrom" version = "0.1.5" diff --git a/truss-transfer/Cargo.toml b/truss-transfer/Cargo.toml index fe8e5138c..e57252e1c 100644 --- a/truss-transfer/Cargo.toml +++ b/truss-transfer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "truss_transfer" -version = "0.0.1-rc3" +version = "0.0.1-rc4" edition = "2021" [lib] @@ -17,17 +17,11 @@ serde = { version = "1.0", features = ["derive"] } serde_yaml = "0.9" tokio = { version = "1.24", features = ["rt-multi-thread", "macros", "fs"] } -# Base reqwest dependency without default features -reqwest = { version = "0.12.2", default-features = false, optional = true, features = ["blocking", "stream", "http2"] } -openssl = { version = "0.10", features = ["vendored"], optional = true } +reqwest = { version = "0.12.12", default-features = false, optional = false, features = ["blocking", "stream", "http2", "default-tls"] } +openssl = { version = "0.10", features = ["vendored"], optional = false } [features] -# CLI uses reqwest with rustls-tls, so that it is musl compatible -cli = ["reqwest/rustls-tls"] - -# Python extension uses OpenSSL, as rustls-tls is not supported on aaarch64 -python_extension = ["reqwest/default-tls", "openssl"] -default = ["python_extension"] +cli = [] [[bin]] name = "truss_transfer_cli" diff --git a/truss-transfer/src/lib.rs b/truss-transfer/src/lib.rs index ecb0d08cc..1212e907c 100644 --- a/truss-transfer/src/lib.rs +++ b/truss-transfer/src/lib.rs @@ -22,6 +22,8 @@ static CACHE_DIR: &str = "/cache/org/artifacts"; static BLOB_DOWNLOAD_TIMEOUT_SECS: u64 = 7200; static BASETEN_FS_ENABLED_ENV_VAR: &str = "BASETEN_FS_ENABLED"; static TRUSS_TRANSFER_NUM_WORKERS_DEFAULT: usize = 64; +static TRUSS_TRANSFER_DOWNLOAD_DIR_ENV_VAR: &str = "TRUSS_TRANSFER_DOWNLOAD_DIR"; +static TRUSS_TRANSFER_DOWNLOAD_DIR_FALLBACK: &str = "/tmp/bptr-resolved"; // Global lock to serialize downloads static GLOBAL_DOWNLOAD_LOCK: OnceLock>> = OnceLock::new(); @@ -31,6 +33,22 @@ fn get_global_lock() -> &'static Arc> { GLOBAL_DOWNLOAD_LOCK.get_or_init(|| Arc::new(Mutex::new(()))) } +fn resolve_truss_transfer_download_dir(optional_download_dir: Option) -> String { + // Order: + // 1. optional_download_dir, if provided + // 2. TRUSS_TRANSFER_DOWNLOAD_DIR_ENV_VAR + // 3. TRUSS_TRANSFER_DOWNLOAD_DIR_FALLBACK and print a warning + optional_download_dir + .or_else(|| env::var(TRUSS_TRANSFER_DOWNLOAD_DIR_ENV_VAR).ok()) + .unwrap_or_else(|| { + println!( + "[WARN] No download directory provided. Please set `export {}=/path/to/dir` or pass it as an argument. Using fallback: {}", + TRUSS_TRANSFER_DOWNLOAD_DIR_ENV_VAR, TRUSS_TRANSFER_DOWNLOAD_DIR_FALLBACK + ); + TRUSS_TRANSFER_DOWNLOAD_DIR_FALLBACK.into() + }) +} + /// Corresponds to `Resolution` in the Python code #[derive(Debug, Deserialize)] struct Resolution { @@ -57,17 +75,21 @@ struct BasetenPointerManifest { } /// Python-callable function to read the manifest and download data. -/// By default, uses 64 concurrent workers if you don't specify `num_workers`. +/// By default, it will use the `TRUSS_TRANSFER_DOWNLOAD_DIR` environment variable. #[pyfunction] -#[pyo3(signature = (download_dir))] -fn lazy_data_resolve(download_dir: String) -> PyResult<()> { - lazy_data_resolve_entrypoint(download_dir).map_err(|err| PyException::new_err(err.to_string())) +#[pyo3(signature = (download_dir=None))] +fn lazy_data_resolve(download_dir: Option) -> PyResult { + lazy_data_resolve_entrypoint(download_dir) + .map(|resolved_dir| resolved_dir) + .map_err(|err| PyException::new_err(err.to_string())) } /// Shared entrypoint for both Python and CLI -fn lazy_data_resolve_entrypoint(download_dir: String) -> Result<()> { +fn lazy_data_resolve_entrypoint(download_dir: Option) -> Result { let num_workers = TRUSS_TRANSFER_NUM_WORKERS_DEFAULT; + let download_dir = resolve_truss_transfer_download_dir(download_dir); + // Ensure the global lock is initialized let lock = get_global_lock(); @@ -82,7 +104,8 @@ fn lazy_data_resolve_entrypoint(download_dir: String) -> Result<()> { .context("Failed to build Tokio runtime")?; // Run the async logic within the runtime - rt.block_on(async { lazy_data_resolve_async(download_dir.into(), num_workers).await }) + rt.block_on(async { lazy_data_resolve_async(download_dir.clone().into(), num_workers).await })?; + Ok(download_dir) } /// Asynchronous implementation of the lazy data resolver logic. @@ -335,18 +358,11 @@ fn main() -> anyhow::Result<()> { "[INFO] truss_transfer_cli, version: {}", env!("CARGO_PKG_VERSION") ); - let args: Vec = std::env::args().collect(); - if args.len() < 2 { - println!("Usage: {} ", args[0]); - return Ok(()); - } - - let download_dir = &args[1]; + let download_dir = std::env::args().nth(1); - println!("[INFO] Invoking lazy_data_resolve_async with download_dir='{download_dir}'"); - - lazy_data_resolve_entrypoint(download_dir.into()) + let _ = lazy_data_resolve_entrypoint(download_dir.into()); + Ok(()) } /// Python module definition