From c4b9cfcd52968eba6b22265d21fb5672125c6801 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 17 Dec 2024 13:30:09 +0000 Subject: [PATCH 1/5] tappd: Add guest api --- Cargo.lock | 98 +++++++++++++++++++++ Cargo.toml | 6 +- basefiles/app-compose.service | 2 +- basefiles/app-compose.sh | 2 +- guest-api/Cargo.toml | 21 +++++ guest-api/build.rs | 11 +++ guest-api/proto/guest_api.proto | 41 +++++++++ guest-api/src/client.rs | 8 ++ guest-api/src/generated/mod.rs | 4 + guest-api/src/lib.rs | 8 ++ tappd/Cargo.toml | 5 +- tappd/src/guest_api_routes.rs | 58 +++++++++++++ tappd/src/guest_api_service.rs | 136 ++++++++++++++++++++++++++++++ tappd/src/main.rs | 19 ++++- tappd/tappd.toml | 4 + teepod/rpc/proto/teepod_rpc.proto | 2 + teepod/src/app.rs | 15 +++- teepod/src/app/qemu.rs | 3 + teepod/src/console.html | 19 ++++- 19 files changed, 451 insertions(+), 11 deletions(-) create mode 100644 guest-api/Cargo.toml create mode 100644 guest-api/build.rs create mode 100644 guest-api/proto/guest_api.proto create mode 100644 guest-api/src/client.rs create mode 100644 guest-api/src/generated/mod.rs create mode 100644 guest-api/src/lib.rs create mode 100644 tappd/src/guest_api_routes.rs create mode 100644 tappd/src/guest_api_service.rs diff --git a/Cargo.lock b/Cargo.lock index d71ce725..cab600d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -986,6 +986,23 @@ dependencies = [ "x509-cert", ] +[[package]] +name = "default-net" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c5a6569a908354d49b10db3c516d69aca1eccd97562fd31c98b13f00b73ca66" +dependencies = [ + "dlopen2", + "libc", + "memalloc", + "netlink-packet-core", + "netlink-packet-route", + "netlink-sys", + "once_cell", + "system-configuration", + "windows 0.48.0", +] + [[package]] name = "der" version = "0.7.9" @@ -1144,6 +1161,17 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "dlopen2" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b4f5f101177ff01b8ec4ecc81eead416a8aa42819a2869311b3420fa114ffa" +dependencies = [ + "libc", + "once_cell", + "winapi", +] + [[package]] name = "documented" version = "0.9.1" @@ -1574,6 +1602,19 @@ dependencies = [ "subtle", ] +[[package]] +name = "guest-api" +version = "0.3.2" +dependencies = [ + "anyhow", + "http-client", + "prost 0.13.3", + "prpc", + "prpc-build", + "serde", + "serde_json", +] + [[package]] name = "h2" version = "0.3.26" @@ -2455,6 +2496,12 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "memalloc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df39d232f5c40b0891c10216992c2f250c054105cb1e56f0fc9032db6203ecc1" + [[package]] name = "memchr" version = "2.7.4" @@ -2567,6 +2614,54 @@ dependencies = [ "serde", ] +[[package]] +name = "netlink-packet-core" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72724faf704479d67b388da142b186f916188505e7e0b26719019c525882eda4" +dependencies = [ + "anyhow", + "byteorder", + "netlink-packet-utils", +] + +[[package]] +name = "netlink-packet-route" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053998cea5a306971f88580d0829e90f270f940befd7cf928da179d4187a5a66" +dependencies = [ + "anyhow", + "bitflags 1.3.2", + "byteorder", + "libc", + "netlink-packet-core", + "netlink-packet-utils", +] + +[[package]] +name = "netlink-packet-utils" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ede8a08c71ad5a95cdd0e4e52facd37190977039a4704eb82a283f713747d34" +dependencies = [ + "anyhow", + "byteorder", + "paste", + "thiserror 1.0.65", +] + +[[package]] +name = "netlink-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16c903aa70590cb93691bf97a767c8d1d6122d2cc9070433deb3bbf36ce8bd23" +dependencies = [ + "bytes", + "libc", + "log", +] + [[package]] name = "nix" version = "0.29.0" @@ -4734,9 +4829,12 @@ dependencies = [ "bollard", "chrono", "clap", + "default-net", "fs-err", "git-version", + "guest-api", "hex", + "host-api", "ra-rpc", "ra-tls", "rcgen", diff --git a/Cargo.toml b/Cargo.toml index d8e4821f..539f31d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,8 +28,9 @@ members = [ "supervisor", "supervisor/client", "rocket-vsock-listener", - "host-api", "http-client", + "host-api", + "guest-api", ] resolver = "2" @@ -49,6 +50,7 @@ tdx-attest-sys = { path = "tdx-attest-sys" } certbot = { path = "certbot" } rocket-vsock-listener = { path = "rocket-vsock-listener" } host-api = { path = "host-api", default-features = false } +guest-api = { path = "guest-api", default-features = false } http-client = { path = "http-client", default-features = false } # Core dependencies @@ -95,6 +97,8 @@ rocket = { git = "https://github.com/rwf2/Rocket", branch = "master", features = rocket-apitoken = { git = "https://github.com/kvinwang/rocket-apitoken", branch = "dev" } tokio = { version = "1.42.0" } tokio-vsock = "0.6.0" +sysinfo = "0.33.0" +default-net = "0.22.0" # Cryptography/Security aes-gcm = "0.10.3" diff --git a/basefiles/app-compose.service b/basefiles/app-compose.service index a670b78a..9e5e901a 100644 --- a/basefiles/app-compose.service +++ b/basefiles/app-compose.service @@ -9,7 +9,7 @@ RemainAfterExit=true EnvironmentFile=-/tapp/env WorkingDirectory=/tapp ExecStart=/usr/bin/env app-compose.sh -ExecStop=/usr/bin/env docker compose down +ExecStop=/usr/bin/env docker compose stop StandardOutput=journal+console StandardError=journal+console diff --git a/basefiles/app-compose.sh b/basefiles/app-compose.sh index 2f07890f..44c045a3 100644 --- a/basefiles/app-compose.sh +++ b/basefiles/app-compose.sh @@ -12,4 +12,4 @@ if ! docker compose up -d; then exit 1 fi -tdxctl notify-host -e "boot.progress" -d "containers started" || true +tdxctl notify-host -e "boot.progress" -d "done" || true diff --git a/guest-api/Cargo.toml b/guest-api/Cargo.toml new file mode 100644 index 00000000..84f6a1ce --- /dev/null +++ b/guest-api/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "guest-api" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +prpc.workspace = true +prost.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +http-client = { workspace = true, optional = true, features = ["prpc"] } + +[build-dependencies] +prpc-build.workspace = true + +[features] +default = ["client"] +client = ["dep:http-client"] diff --git a/guest-api/build.rs b/guest-api/build.rs new file mode 100644 index 00000000..43bca3d8 --- /dev/null +++ b/guest-api/build.rs @@ -0,0 +1,11 @@ +fn main() { + prpc_build::configure() + .out_dir("./src/generated") + .mod_prefix("super::") + .build_scale_ext(false) + .disable_service_name_emission() + .disable_package_emission() + .enable_serde_extension() + .compile_dir("./proto") + .expect("failed to compile proto files"); +} diff --git a/guest-api/proto/guest_api.proto b/guest-api/proto/guest_api.proto new file mode 100644 index 00000000..39bb1d80 --- /dev/null +++ b/guest-api/proto/guest_api.proto @@ -0,0 +1,41 @@ + +syntax = "proto3"; + +import "google/protobuf/empty.proto"; + +package guest_api; + +message GuestInfo { + string name = 1; + string version = 2; +} + +message IpAddress { + string address = 1; + uint32 prefix = 2; +} + +message Interface { + string name = 1; + repeated IpAddress addresses = 2; + uint64 rx_bytes = 3; + uint64 tx_bytes = 4; + uint64 rx_errors = 5; + uint64 tx_errors = 6; +} + +message Gateway { + string address = 1; +} + +message NetworkInformation { + repeated string dns_servers = 1; + repeated Gateway gateways = 2; + repeated Interface interfaces = 3; +} + +service GuestApi { + rpc Info(google.protobuf.Empty) returns (GuestInfo); + rpc Shutdown(google.protobuf.Empty) returns (google.protobuf.Empty); + rpc NetworkInfo(google.protobuf.Empty) returns (NetworkInformation); +} diff --git a/guest-api/src/client.rs b/guest-api/src/client.rs new file mode 100644 index 00000000..e75e43a7 --- /dev/null +++ b/guest-api/src/client.rs @@ -0,0 +1,8 @@ +use crate::guest_api_client::GuestApiClient; +use http_client::prpc::PrpcClient; + +pub type DefaultClient = GuestApiClient; + +pub fn new_client(base_url: String) -> DefaultClient { + DefaultClient::new(PrpcClient::new(base_url)) +} diff --git a/guest-api/src/generated/mod.rs b/guest-api/src/generated/mod.rs new file mode 100644 index 00000000..b24a4b98 --- /dev/null +++ b/guest-api/src/generated/mod.rs @@ -0,0 +1,4 @@ +pub use guest_api::*; + +#[allow(async_fn_in_trait)] +mod guest_api; diff --git a/guest-api/src/lib.rs b/guest-api/src/lib.rs new file mode 100644 index 00000000..f0c09bc9 --- /dev/null +++ b/guest-api/src/lib.rs @@ -0,0 +1,8 @@ +extern crate alloc; + +pub use generated::*; + +mod generated; + +#[cfg(feature = "client")] +pub mod client; diff --git a/tappd/Cargo.toml b/tappd/Cargo.toml index 3dae61da..c3b90f9c 100644 --- a/tappd/Cargo.toml +++ b/tappd/Cargo.toml @@ -28,4 +28,7 @@ ra-rpc = { workspace = true, features = ["rocket"] } tappd-rpc.workspace = true ra-tls.workspace = true tdx-attest.workspace = true -sysinfo = "0.33.0" +guest-api = { workspace = true, features = ["client"] } +host-api = { workspace = true, features = ["client"] } +sysinfo.workspace = true +default-net.workspace = true diff --git a/tappd/src/guest_api_routes.rs b/tappd/src/guest_api_routes.rs new file mode 100644 index 00000000..b65455af --- /dev/null +++ b/tappd/src/guest_api_routes.rs @@ -0,0 +1,58 @@ +use crate::{guest_api_service::GuestApiHandler, AppState}; +use ra_rpc::rocket_helper::PrpcHandler; + +use rocket::{ + data::{Data, Limits}, + get, + http::ContentType, + mtls::Certificate, + post, + response::status::Custom, + routes, Route, State, +}; + +#[post("/?", data = "")] +#[allow(clippy::too_many_arguments)] +async fn prpc_post( + state: &State, + cert: Option>, + method: &str, + data: Data<'_>, + limits: &Limits, + content_type: Option<&ContentType>, + json: bool, +) -> Custom> { + PrpcHandler::builder() + .state(&**state) + .maybe_certificate(cert) + .method(method) + .data(data) + .limits(limits) + .maybe_content_type(content_type) + .json(json) + .build() + .handle::() + .await +} + +#[get("/")] +async fn prpc_get( + state: &State, + method: &str, + limits: &Limits, + content_type: Option<&ContentType>, +) -> Custom> { + PrpcHandler::builder() + .state(&**state) + .method(method) + .limits(limits) + .maybe_content_type(content_type) + .json(true) + .build() + .handle::() + .await +} + +pub fn routes() -> Vec { + routes![prpc_post, prpc_get] +} diff --git a/tappd/src/guest_api_service.rs b/tappd/src/guest_api_service.rs new file mode 100644 index 00000000..82c6d0bb --- /dev/null +++ b/tappd/src/guest_api_service.rs @@ -0,0 +1,136 @@ +use std::process::Command; + +use anyhow::Result; +use fs_err as fs; +use guest_api::{ + guest_api_server::{GuestApiRpc, GuestApiServer}, + Gateway, GuestInfo, Interface, IpAddress, NetworkInformation, +}; +use host_api::Notification; +use ra_rpc::{CallContext, RpcCall}; +use serde::Deserialize; + +use crate::AppState; + +#[derive(Deserialize)] +struct LocalConfig { + host_api_url: String, +} + +pub struct GuestApiHandler; + +impl RpcCall for GuestApiHandler { + type PrpcService = GuestApiServer; + + fn into_prpc_service(self) -> Self::PrpcService { + GuestApiServer::new(self) + } + + fn construct(_context: CallContext<'_, AppState>) -> Result + where + Self: Sized, + { + Ok(Self) + } +} + +impl GuestApiRpc for GuestApiHandler { + async fn info(self) -> Result { + let guest_info = GuestInfo { + name: "Tappd".to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + }; + Ok(guest_info) + } + + async fn shutdown(self) -> Result<()> { + tokio::spawn(async move { + notify_q("shutdown.stopapp", "").await.ok(); + run_command("systemctl stop app-compose").ok(); + notify_q("shutdown", "").await.ok(); + run_command("systemctl poweroff").ok(); + }); + Ok(()) + } + + async fn network_info(self) -> Result { + let networks = sysinfo::Networks::new_with_refreshed_list(); + for (interface_name, network) in &networks { + println!("[{interface_name}]: {network:?}"); + } + Ok(NetworkInformation { + dns_servers: get_dns_servers(), + gateways: get_gateways(), + interfaces: get_interfaces(), + }) + } +} + +fn get_interfaces() -> Vec { + sysinfo::Networks::new_with_refreshed_list() + .into_iter() + .map(|(interface_name, network)| Interface { + name: interface_name.clone(), + addresses: network + .ip_networks() + .into_iter() + .map(|ip| IpAddress { + address: ip.addr.to_string(), + prefix: ip.prefix as u32, + }) + .collect(), + rx_bytes: network.total_received(), + tx_bytes: network.total_transmitted(), + rx_errors: network.total_errors_on_received(), + tx_errors: network.total_errors_on_transmitted(), + }) + .collect() +} + +fn get_gateways() -> Vec { + default_net::get_interfaces() + .into_iter() + .flat_map(|iface| { + iface.gateway.map(|gw| Gateway { + address: gw.ip_addr.to_string(), + }) + }) + .collect() +} + +fn get_dns_servers() -> Vec { + let mut dns_servers = Vec::new(); + // read /etc/resolv.conf + let Ok(resolv_conf) = fs::read_to_string("/etc/resolv.conf") else { + return dns_servers; + }; + for line in resolv_conf.lines() { + if line.starts_with("nameserver") { + let Some(ip) = line.split_whitespace().nth(1) else { + continue; + }; + dns_servers.push(ip.to_string()); + } + } + dns_servers +} + +pub async fn notify_q(event: &str, payload: &str) -> Result<()> { + let local_config: LocalConfig = + serde_json::from_str(&fs::read_to_string("/tapp/config.json")?)?; + let nc = host_api::client::new_client(local_config.host_api_url); + nc.notify(Notification { + event: event.to_string(), + payload: payload.to_string(), + }) + .await?; + Ok(()) +} + +fn run_command(command: &str) -> Result<()> { + let output = Command::new("sh").arg("-c").arg(command).output()?; + if !output.status.success() { + return Err(anyhow::anyhow!("Command failed: {}", output.status)); + } + Ok(()) +} diff --git a/tappd/src/main.rs b/tappd/src/main.rs index 713500ee..75deeca7 100644 --- a/tappd/src/main.rs +++ b/tappd/src/main.rs @@ -10,6 +10,8 @@ use rocket::{ use rpc_service::AppState; mod config; +mod guest_api_routes; +mod guest_api_service; mod http_routes; mod models; mod rpc_service; @@ -72,6 +74,17 @@ async fn run_external(state: AppState, figment: Figment) -> Result<()> { Ok(()) } +async fn run_guest_api(state: AppState, figment: Figment) -> Result<()> { + let rocket = rocket::custom(figment) + .mount("/", guest_api_routes::routes()) + .manage(state); + let _ = rocket + .launch() + .await + .map_err(|err| anyhow!("Failed to ignite rocket: {err}"))?; + Ok(()) +} + #[rocket::main] async fn main() -> Result<()> { let args = Args::parse(); @@ -81,11 +94,13 @@ async fn main() -> Result<()> { let internal_figment = figment.clone().select("internal"); let external_figment = figment.clone().select("external"); - let external_https_figment = figment.select("external-https"); + let external_https_figment = figment.clone().select("external-https"); + let guest_api_figment = figment.select("guest-api"); tokio::select!( res = run_internal(state.clone(), internal_figment) => res?, res = run_external(state.clone(), external_figment) => res?, - res = run_external(state, external_https_figment) => res? + res = run_guest_api(state.clone(), guest_api_figment) => res?, + res = run_external(state, external_https_figment) => res?, ); Ok(()) } diff --git a/tappd/tappd.toml b/tappd/tappd.toml index 4022e376..644316f9 100644 --- a/tappd/tappd.toml +++ b/tappd/tappd.toml @@ -25,3 +25,7 @@ port = 8043 [external-https.tls] key = "/etc/tappd/tls.key" certs = "/etc/tappd/tls.cert" + +[guest-api] +address = "vsock:0" +port = 8000 diff --git a/teepod/rpc/proto/teepod_rpc.proto b/teepod/rpc/proto/teepod_rpc.proto index fc0da873..2dcbd963 100644 --- a/teepod/rpc/proto/teepod_rpc.proto +++ b/teepod/rpc/proto/teepod_rpc.proto @@ -27,6 +27,8 @@ message VmInfo { string boot_progress = 10; // Boot error string boot_error = 11; + // Shutdown progress + string shutdown_progress = 12; } message Id { diff --git a/teepod/src/app.rs b/teepod/src/app.rs index f7bf6cc8..950b6f52 100644 --- a/teepod/src/app.rs +++ b/teepod/src/app.rs @@ -151,8 +151,7 @@ impl App { let process_config = vm_state .config .config_qemu(&self.config.qemu_path, &work_dir)?; - vm_state.state.boot_error.clear(); - vm_state.state.boot_progress.clear(); + vm_state.state.clear(); process_config }; self.supervisor @@ -286,6 +285,9 @@ impl App { "boot.error" => { vm.state.boot_error = body; } + "shutdown.progress" => { + vm.state.shutdown_progress = body; + } "instance.info" => { if body.len() > 1024 * 4 { error!("Instance info too large, skipping"); @@ -313,6 +315,15 @@ pub struct VmState { struct VmStateMut { boot_progress: String, boot_error: String, + shutdown_progress: String, +} + +impl VmStateMut { + pub fn clear(&mut self) { + self.boot_progress.clear(); + self.boot_error.clear(); + self.shutdown_progress.clear(); + } } impl VmState { diff --git a/teepod/src/app/qemu.rs b/teepod/src/app/qemu.rs index 3910042e..06f1ce7b 100644 --- a/teepod/src/app/qemu.rs +++ b/teepod/src/app/qemu.rs @@ -32,6 +32,7 @@ pub struct VmInfo { pub instance_id: Option, pub boot_progress: String, pub boot_error: String, + pub shutdown_progress: String, } #[derive(Debug, Builder)] @@ -77,6 +78,7 @@ impl VmInfo { uptime: self.uptime.clone(), boot_progress: self.boot_progress.clone(), boot_error: self.boot_error.clone(), + shutdown_progress: self.shutdown_progress.clone(), configuration: Some(pb::VmConfiguration { name: self.manifest.name.clone(), image: self.manifest.image.clone(), @@ -151,6 +153,7 @@ impl VmState { exited_at: Some(exited_at), boot_progress: self.state.boot_progress.clone(), boot_error: self.state.boot_error.clone(), + shutdown_progress: self.state.shutdown_progress.clone(), } } } diff --git a/teepod/src/console.html b/teepod/src/console.html index 931b50e0..40d5bc90 100644 --- a/teepod/src/console.html +++ b/teepod/src/console.html @@ -633,7 +633,7 @@

VM List

{{ vm.name }} - {{ vm.status }} + {{ vmStatus(vm) }} {{ vm.uptime }} @@ -789,8 +789,7 @@

Update VM Config

- +
@@ -1304,6 +1303,19 @@

Derive VM

} }; + const vmStatus = (vm) => { + if (vm.status != 'running') { + return vm.status; + } + if (vm.state.shutdown_progress) { + return "shutting down"; + } + if (vm.state.boot_progress != 'done') { + return "booting"; + } + return "running"; + }; + return { vms, vmForm, @@ -1336,6 +1348,7 @@

Derive VM

forkDialog, forkVm, loadEnvFile, + vmStatus }; } }).mount('#app'); From 449ba958443694c9f687465256e2f28fc1d884cd Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 17 Dec 2024 13:54:33 +0000 Subject: [PATCH 2/5] teepod: delegate tappd RPCs --- Cargo.lock | 1 + tappd/src/guest_api_service.rs | 35 ++++++++++++--------- tappd/src/main.rs | 2 +- teepod/Cargo.toml | 1 + teepod/rpc/proto/teepod_rpc.proto | 29 ++++++++++++++++++ teepod/src/app.rs | 4 +-- teepod/src/main_service.rs | 51 +++++++++++++++++++++++++++++-- 7 files changed, 104 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cab600d6..b001ad9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4933,6 +4933,7 @@ dependencies = [ "dirs", "fs-err", "git-version", + "guest-api", "hex", "host-api", "humantime", diff --git a/tappd/src/guest_api_service.rs b/tappd/src/guest_api_service.rs index 82c6d0bb..b2eb7c54 100644 --- a/tappd/src/guest_api_service.rs +++ b/tappd/src/guest_api_service.rs @@ -69,20 +69,27 @@ impl GuestApiRpc for GuestApiHandler { fn get_interfaces() -> Vec { sysinfo::Networks::new_with_refreshed_list() .into_iter() - .map(|(interface_name, network)| Interface { - name: interface_name.clone(), - addresses: network - .ip_networks() - .into_iter() - .map(|ip| IpAddress { - address: ip.addr.to_string(), - prefix: ip.prefix as u32, - }) - .collect(), - rx_bytes: network.total_received(), - tx_bytes: network.total_transmitted(), - rx_errors: network.total_errors_on_received(), - tx_errors: network.total_errors_on_transmitted(), + .filter_map(|(interface_name, network)| { + if !(interface_name == "wg0" || interface_name.starts_with("enp")) { + // We only get wg0 and enp interfaces. + // Docker bridge is not included due to privacy concerns. + return None; + } + Some(Interface { + name: interface_name.clone(), + addresses: network + .ip_networks() + .into_iter() + .map(|ip| IpAddress { + address: ip.addr.to_string(), + prefix: ip.prefix as u32, + }) + .collect(), + rx_bytes: network.total_received(), + tx_bytes: network.total_transmitted(), + rx_errors: network.total_errors_on_received(), + tx_errors: network.total_errors_on_transmitted(), + }) }) .collect() } diff --git a/tappd/src/main.rs b/tappd/src/main.rs index 75deeca7..0574eddd 100644 --- a/tappd/src/main.rs +++ b/tappd/src/main.rs @@ -76,7 +76,7 @@ async fn run_external(state: AppState, figment: Figment) -> Result<()> { async fn run_guest_api(state: AppState, figment: Figment) -> Result<()> { let rocket = rocket::custom(figment) - .mount("/", guest_api_routes::routes()) + .mount("/api", guest_api_routes::routes()) .manage(state); let _ = rocket .launch() diff --git a/teepod/Cargo.toml b/teepod/Cargo.toml index 9a5d1903..05dfb613 100644 --- a/teepod/Cargo.toml +++ b/teepod/Cargo.toml @@ -36,3 +36,4 @@ kms-rpc.workspace = true path-absolutize.workspace = true host-api.workspace = true safe-write.workspace = true +guest-api = { workspace = true, features = ["client"] } diff --git a/teepod/rpc/proto/teepod_rpc.proto b/teepod/rpc/proto/teepod_rpc.proto index 2dcbd963..aad361a9 100644 --- a/teepod/rpc/proto/teepod_rpc.proto +++ b/teepod/rpc/proto/teepod_rpc.proto @@ -122,6 +122,30 @@ message ResizeVmRequest { optional uint32 disk_size = 4; } +message IpAddress { + string address = 1; + uint32 prefix = 2; +} + +message Interface { + string name = 1; + repeated IpAddress addresses = 2; + uint64 rx_bytes = 3; + uint64 tx_bytes = 4; + uint64 rx_errors = 5; + uint64 tx_errors = 6; +} + +message Gateway { + string address = 1; +} + +message NetworkInformation { + repeated string dns_servers = 1; + repeated Gateway gateways = 2; + repeated Interface interfaces = 3; +} + // Service definition for Teepod service Teepod { // RPC to create a VM @@ -134,6 +158,8 @@ service Teepod { rpc RemoveVm(Id) returns (google.protobuf.Empty); // RPC to upgrade an app rpc UpgradeApp(UpgradeAppRequest) returns (Id); + // Shutdown a VM + rpc ShutdownVm(Id) returns (google.protobuf.Empty); // RPC to list all VMs rpc Status(google.protobuf.Empty) returns (StatusResponse); @@ -146,6 +172,9 @@ service Teepod { // Get VM info by ID rpc GetInfo(Id) returns (GetInfoResponse); + // Get VM Network Information + rpc GetNetworkInfo(Id) returns (NetworkInformation); + // RPC to resize a VM rpc ResizeVm(ResizeVmRequest) returns (google.protobuf.Empty); } diff --git a/teepod/src/app.rs b/teepod/src/app.rs index 950b6f52..7bcdd7f6 100644 --- a/teepod/src/app.rs +++ b/teepod/src/app.rs @@ -66,7 +66,7 @@ pub struct App { } impl App { - fn lock(&self) -> MutexGuard { + pub(crate) fn lock(&self) -> MutexGuard { self.state.lock().unwrap() } @@ -307,7 +307,7 @@ impl App { #[derive(Clone)] pub struct VmState { - config: Arc, + pub(crate) config: Arc, state: VmStateMut, } diff --git a/teepod/src/main_service.rs b/teepod/src/main_service.rs index 06f38657..60f03236 100644 --- a/teepod/src/main_service.rs +++ b/teepod/src/main_service.rs @@ -3,13 +3,15 @@ use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result}; use fs_err as fs; +use guest_api::client::DefaultClient as GuestClient; use kms_rpc::kms_client::KmsClient; use ra_rpc::client::RaClient; use ra_rpc::{CallContext, RpcCall}; use teepod_rpc::teepod_server::{TeepodRpc, TeepodServer}; use teepod_rpc::{ - AppId, GetInfoResponse, Id, ImageInfo as RpcImageInfo, ImageListResponse, PublicKeyResponse, - ResizeVmRequest, StatusResponse, UpgradeAppRequest, VmConfiguration, + AppId, Gateway, GetInfoResponse, Id, ImageInfo as RpcImageInfo, ImageListResponse, Interface, + IpAddress, NetworkInformation, PublicKeyResponse, ResizeVmRequest, StatusResponse, + UpgradeAppRequest, VmConfiguration, }; use tracing::warn; @@ -103,6 +105,13 @@ impl RpcHandler { let prpc_client = RaClient::new(url, true); Ok(KmsClient::new(prpc_client)) } + + fn tappd_client(&self, id: &str) -> Option { + let cid = self.app.lock().get(id)?.config.cid; + Some(guest_api::client::new_client(format!( + "vsock://{cid}:8000/api" + ))) + } } fn app_id_of(compose_file: &str) -> String { @@ -344,6 +353,44 @@ impl TeepodRpc for RpcHandler { .context("Failed to load VM")?; Ok(()) } + + async fn shutdown_vm(self, request: Id) -> Result<()> { + let tappd_client = self.tappd_client(&request.id).context("vm not found")?; + tappd_client.shutdown().await?; + Ok(()) + } + + async fn get_network_info(self, request: Id) -> Result { + let tappd_client = self.tappd_client(&request.id).context("vm not found")?; + let info = tappd_client.network_info().await?; + Ok(NetworkInformation { + dns_servers: info.dns_servers, + gateways: info + .gateways + .into_iter() + .map(|g| Gateway { address: g.address }) + .collect(), + interfaces: info + .interfaces + .into_iter() + .map(|i| Interface { + name: i.name, + addresses: i + .addresses + .into_iter() + .map(|a| IpAddress { + address: a.address, + prefix: a.prefix, + }) + .collect(), + rx_bytes: i.rx_bytes, + tx_bytes: i.tx_bytes, + rx_errors: i.rx_errors, + tx_errors: i.tx_errors, + }) + .collect(), + }) + } } impl RpcCall for RpcHandler { From 43662f76889d22830609c573ac2fde99bc295923 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 18 Dec 2024 01:49:59 +0000 Subject: [PATCH 3/5] Reuse the same proto of guest api for teepod and tappd --- Cargo.lock | 1 + guest-api/proto/guest_api.proto | 98 +++++++++++++++++++++++++- supervisor/src/process.rs | 4 ++ tappd/Cargo.toml | 1 + tappd/rpc/proto/tappd_rpc.proto | 81 +--------------------- tappd/src/guest_api_service.rs | 111 ++++++++++++++++++++++++++---- tappd/src/http_routes.rs | 6 +- tappd/src/main.rs | 12 +++- tappd/src/models.rs | 2 +- tappd/src/rpc_service.rs | 84 +++------------------- tappd/tappd.toml | 2 +- teepod/rpc/proto/teepod_rpc.proto | 29 +------- teepod/src/app.rs | 18 ++++- teepod/src/app/image.rs | 24 +++++-- teepod/src/app/qemu.rs | 9 ++- teepod/src/console.html | 94 +++++++++++++++++++++++-- teepod/src/guest_api_routes.rs | 58 ++++++++++++++++ teepod/src/guest_api_service.rs | 74 ++++++++++++++++++++ teepod/src/main.rs | 3 + teepod/src/main_service.rs | 77 +++++++-------------- 20 files changed, 522 insertions(+), 266 deletions(-) create mode 100644 teepod/src/guest_api_routes.rs create mode 100644 teepod/src/guest_api_service.rs diff --git a/Cargo.lock b/Cargo.lock index b001ad9c..f136decf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4840,6 +4840,7 @@ dependencies = [ "rcgen", "rinja", "rocket", + "rocket-vsock-listener", "serde", "serde_json", "sha2", diff --git a/guest-api/proto/guest_api.proto b/guest-api/proto/guest_api.proto index 39bb1d80..e698293d 100644 --- a/guest-api/proto/guest_api.proto +++ b/guest-api/proto/guest_api.proto @@ -5,9 +5,21 @@ import "google/protobuf/empty.proto"; package guest_api; +message Id { + string id = 1; +} + message GuestInfo { - string name = 1; - string version = 2; + // Guest software version + string version = 1; + // App ID + string app_id = 2; + // App Instance ID + string instance_id = 3; + // App certificate + string app_cert = 4; + // TCB info + string tcb_info = 5; } message IpAddress { @@ -34,8 +46,88 @@ message NetworkInformation { repeated Interface interfaces = 3; } +message ListContainersResponse { + repeated Container containers = 1; +} + +message Container { + // The ID of this container + string id = 1; + // The names that this container has been given + repeated string names = 2; + // The name of the image used when creating this container + string image = 3; + // The ID of the image that this container was created from + string image_id = 4; + // Command to run when starting the container + string command = 5; + // When the container was created + int64 created = 6; + // The state of this container (e.g. Exited) + string state = 7; + // The status of this container (e.g. Exited) + string status = 8; +} + +// The system info +message SystemInfo { + // Operating system + string os_name = 1; + // Operating system version + string os_version = 2; + // Kernel version + string kernel_version = 3; + // Cpu model + string cpu_model = 4; + // Number of logical CPUs + uint32 num_cpus = 5; + // Total memory + uint64 total_memory = 6; + // Available memory + uint64 available_memory = 7; + // Used memory + uint64 used_memory = 8; + // Free memory + uint64 free_memory = 9; + // Total swap memory + uint64 total_swap = 10; + // Used swap memory + uint64 used_swap = 11; + // Free swap memory + uint64 free_swap = 12; + // Uptime + uint64 uptime = 13; + // Load average + uint32 loadavg_one = 14; + uint32 loadavg_five = 15; + uint32 loadavg_fifteen = 16; + // Disks + repeated DiskInfo disks = 17; +} + +message DiskInfo { + // Device name + string name = 1; + // Mount point + string mount_point = 2; + // Total size + uint64 total_size = 3; + // Free size + uint64 free_size = 5; +} + service GuestApi { rpc Info(google.protobuf.Empty) returns (GuestInfo); - rpc Shutdown(google.protobuf.Empty) returns (google.protobuf.Empty); + rpc SysInfo(google.protobuf.Empty) returns (SystemInfo); rpc NetworkInfo(google.protobuf.Empty) returns (NetworkInformation); + rpc ListContainers(google.protobuf.Empty) returns (ListContainersResponse); + rpc Shutdown(google.protobuf.Empty) returns (google.protobuf.Empty); +} + +service ProxiedGuestApi { + rpc Info(Id) returns (GuestInfo); + rpc SysInfo(Id) returns (SystemInfo); + rpc NetworkInfo(Id) returns (NetworkInformation); + rpc ListContainers(Id) returns (ListContainersResponse); + rpc Shutdown(Id) returns (google.protobuf.Empty); } diff --git a/supervisor/src/process.rs b/supervisor/src/process.rs index 377a3806..6124de18 100644 --- a/supervisor/src/process.rs +++ b/supervisor/src/process.rs @@ -129,6 +129,10 @@ impl ProcessStatus { pub fn is_running(&self) -> bool { matches!(self, ProcessStatus::Running) } + + pub fn is_stopped(&self) -> bool { + matches!(self, ProcessStatus::Stopped) + } } #[derive(Clone)] diff --git a/tappd/Cargo.toml b/tappd/Cargo.toml index c3b90f9c..504e075c 100644 --- a/tappd/Cargo.toml +++ b/tappd/Cargo.toml @@ -32,3 +32,4 @@ guest-api = { workspace = true, features = ["client"] } host-api = { workspace = true, features = ["client"] } sysinfo.workspace = true default-net.workspace = true +rocket-vsock-listener.workspace = true diff --git a/tappd/rpc/proto/tappd_rpc.proto b/tappd/rpc/proto/tappd_rpc.proto index 350a3e67..25962aca 100644 --- a/tappd/rpc/proto/tappd_rpc.proto +++ b/tappd/rpc/proto/tappd_rpc.proto @@ -62,39 +62,6 @@ message TdxQuoteResponse { string event_log = 2; } - -service Worker { - // Get worker info - rpc Info(google.protobuf.Empty) returns (WorkerInfo) {} - // Get system info - rpc SysInfo(google.protobuf.Empty) returns (SystemInfo) {} - // Get worker containers - rpc ListContainers(google.protobuf.Empty) returns (ListContainersResponse) {} -} - -message ListContainersResponse { - repeated Container containers = 1; -} - -message Container { - // The ID of this container - string id = 1; - // The names that this container has been given - repeated string names = 2; - // The name of the image used when creating this container - string image = 3; - // The ID of the image that this container was created from - string image_id = 4; - // Command to run when starting the container - string command = 5; - // When the container was created - int64 created = 6; - // The state of this container (e.g. Exited) - string state = 7; - // The status of this container (e.g. Exited) - string status = 8; -} - // The request to derive a key message WorkerInfo { // App ID @@ -107,49 +74,7 @@ message WorkerInfo { string tcb_info = 4; } -// The system info -message SystemInfo { - // Operating system - string os_name = 1; - // Operating system version - string os_version = 2; - // Kernel version - string kernel_version = 3; - // Cpu model - string cpu_model = 4; - // Number of logical CPUs - uint32 num_cpus = 5; - // Total memory - uint64 total_memory = 6; - // Available memory - uint64 available_memory = 7; - // Used memory - uint64 used_memory = 8; - // Free memory - uint64 free_memory = 9; - // Total swap memory - uint64 total_swap = 10; - // Used swap memory - uint64 used_swap = 11; - // Free swap memory - uint64 free_swap = 12; - // Uptime - uint64 uptime = 13; - // Load average - uint32 loadavg_one = 14; - uint32 loadavg_five = 15; - uint32 loadavg_fifteen = 16; - // Disks - repeated DiskInfo disks = 17; -} - -message DiskInfo { - // Device name - string name = 1; - // Mount point - string mount_point = 2; - // Total size - uint64 total_size = 3; - // Free size - uint64 free_size = 5; +service Worker { + // Get worker info + rpc Info(google.protobuf.Empty) returns (WorkerInfo) {} } diff --git a/tappd/src/guest_api_service.rs b/tappd/src/guest_api_service.rs index b2eb7c54..262d8b9e 100644 --- a/tappd/src/guest_api_service.rs +++ b/tappd/src/guest_api_service.rs @@ -1,23 +1,28 @@ -use std::process::Command; +use std::{path::Path, process::Command}; -use anyhow::Result; +use anyhow::{Context, Result}; +use bollard::{container::ListContainersOptions, Docker}; use fs_err as fs; use guest_api::{ guest_api_server::{GuestApiRpc, GuestApiServer}, - Gateway, GuestInfo, Interface, IpAddress, NetworkInformation, + Container, DiskInfo, Gateway, GuestInfo, Interface, IpAddress, ListContainersResponse, + NetworkInformation, SystemInfo, }; use host_api::Notification; use ra_rpc::{CallContext, RpcCall}; use serde::Deserialize; +use tappd_rpc::worker_server::WorkerRpc as _; -use crate::AppState; +use crate::{rpc_service::ExternalRpcHandler, AppState}; #[derive(Deserialize)] struct LocalConfig { host_api_url: String, } -pub struct GuestApiHandler; +pub struct GuestApiHandler { + state: AppState, +} impl RpcCall for GuestApiHandler { type PrpcService = GuestApiServer; @@ -26,28 +31,34 @@ impl RpcCall for GuestApiHandler { GuestApiServer::new(self) } - fn construct(_context: CallContext<'_, AppState>) -> Result + fn construct(context: CallContext<'_, AppState>) -> Result where Self: Sized, { - Ok(Self) + Ok(Self { + state: context.state.clone(), + }) } } impl GuestApiRpc for GuestApiHandler { async fn info(self) -> Result { - let guest_info = GuestInfo { - name: "Tappd".to_string(), + let ext_rpc = ExternalRpcHandler::new(self.state); + let info = ext_rpc.info().await?; + Ok(GuestInfo { version: env!("CARGO_PKG_VERSION").to_string(), - }; - Ok(guest_info) + app_id: info.app_id, + instance_id: info.instance_id, + app_cert: info.app_cert, + tcb_info: info.tcb_info, + }) } async fn shutdown(self) -> Result<()> { tokio::spawn(async move { - notify_q("shutdown.stopapp", "").await.ok(); + notify_host("shutdown.progress", "stopping app").await.ok(); run_command("systemctl stop app-compose").ok(); - notify_q("shutdown", "").await.ok(); + notify_host("shutdown.progress", "powering off").await.ok(); run_command("systemctl poweroff").ok(); }); Ok(()) @@ -64,6 +75,78 @@ impl GuestApiRpc for GuestApiHandler { interfaces: get_interfaces(), }) } + + async fn sys_info(self) -> Result { + use sysinfo::System; + + let system = System::new_all(); + let cpus = system.cpus(); + + let disks = sysinfo::Disks::new_with_refreshed_list(); + let disks = disks + .list() + .iter() + .filter(|d| d.mount_point() == Path::new("/")) + .map(|d| DiskInfo { + name: d.name().to_string_lossy().to_string(), + mount_point: d.mount_point().to_string_lossy().to_string(), + total_size: d.total_space(), + free_size: d.available_space(), + }) + .collect::>(); + let avg = System::load_average(); + Ok(SystemInfo { + os_name: System::name().unwrap_or_default(), + os_version: System::os_version().unwrap_or_default(), + kernel_version: System::kernel_version().unwrap_or_default(), + cpu_model: cpus.first().map_or("".into(), |cpu| { + format!("{} @{} MHz", cpu.name(), cpu.frequency()) + }), + num_cpus: cpus.len() as _, + total_memory: system.total_memory(), + available_memory: system.available_memory(), + used_memory: system.used_memory(), + free_memory: system.free_memory(), + total_swap: system.total_swap(), + used_swap: system.used_swap(), + free_swap: system.free_swap(), + uptime: System::uptime(), + loadavg_one: (avg.one * 100.0) as u32, + loadavg_five: (avg.five * 100.0) as u32, + loadavg_fifteen: (avg.fifteen * 100.0) as u32, + disks, + }) + } + + async fn list_containers(self) -> Result { + list_containers().await + } +} + +pub(crate) async fn list_containers() -> Result { + let docker = Docker::connect_with_defaults().context("Failed to connect to Docker")?; + let containers = docker + .list_containers::<&str>(Some(ListContainersOptions { + all: true, + ..Default::default() + })) + .await + .context("Failed to list containers")?; + Ok(ListContainersResponse { + containers: containers + .into_iter() + .map(|c| Container { + id: c.id.unwrap_or_default(), + names: c.names.unwrap_or_default(), + image: c.image.unwrap_or_default(), + image_id: c.image_id.unwrap_or_default(), + command: c.command.unwrap_or_default(), + created: c.created.unwrap_or_default(), + state: c.state.unwrap_or_default(), + status: c.status.unwrap_or_default(), + }) + .collect(), + }) } fn get_interfaces() -> Vec { @@ -122,7 +205,7 @@ fn get_dns_servers() -> Vec { dns_servers } -pub async fn notify_q(event: &str, payload: &str) -> Result<()> { +pub async fn notify_host(event: &str, payload: &str) -> Result<()> { let local_config: LocalConfig = serde_json::from_str(&fs::read_to_string("/tapp/config.json")?)?; let nc = host_api::client::new_client(local_config.host_api_url); diff --git a/tappd/src/http_routes.rs b/tappd/src/http_routes.rs index d039c5d2..de589993 100644 --- a/tappd/src/http_routes.rs +++ b/tappd/src/http_routes.rs @@ -1,6 +1,8 @@ -use crate::rpc_service::{list_containers, AppState, ExternalRpcHandler, InternalRpcHandler}; +use crate::guest_api_service::{list_containers, GuestApiHandler}; +use crate::rpc_service::{AppState, ExternalRpcHandler, InternalRpcHandler}; use anyhow::Result; use docker_logs::parse_duration; +use guest_api::guest_api_server::GuestApiRpc; use ra_rpc::rocket_helper::PrpcHandler; use ra_rpc::{CallContext, RpcCall}; use rinja::Template; @@ -74,7 +76,7 @@ async fn index(state: &State) -> Result, String> { .await .map_err(|e| format!("Failed to get worker info: {}", e))?; - let handler = ExternalRpcHandler::construct(context) + let handler = GuestApiHandler::construct(context) .map_err(|e| format!("Failed to construct RPC handler: {}", e))?; let system_info = handler.sys_info().await.unwrap_or_default(); diff --git a/tappd/src/main.rs b/tappd/src/main.rs index 0574eddd..16b49022 100644 --- a/tappd/src/main.rs +++ b/tappd/src/main.rs @@ -7,6 +7,7 @@ use rocket::{ figment::Figment, listener::{Bind, DefaultListener}, }; +use rocket_vsock_listener::VsockListener; use rpc_service::AppState; mod config; @@ -78,10 +79,17 @@ async fn run_guest_api(state: AppState, figment: Figment) -> Result<()> { let rocket = rocket::custom(figment) .mount("/api", guest_api_routes::routes()) .manage(state); - let _ = rocket - .launch() + + let ignite = rocket + .ignite() .await .map_err(|err| anyhow!("Failed to ignite rocket: {err}"))?; + let listener = VsockListener::bind_rocket(&ignite) + .map_err(|err| anyhow!("Failed to bind guest API : {err}"))?; + ignite + .launch_on(listener) + .await + .map_err(|err| anyhow!(err.to_string()))?; Ok(()) } diff --git a/tappd/src/models.rs b/tappd/src/models.rs index 3d54b838..f7d2a13b 100644 --- a/tappd/src/models.rs +++ b/tappd/src/models.rs @@ -1,5 +1,5 @@ +use guest_api::{Container, SystemInfo}; use rinja::Template; -use tappd_rpc::{Container, SystemInfo}; mod filters { use anyhow::Result; diff --git a/tappd/src/rpc_service.rs b/tappd/src/rpc_service.rs index f9465ca4..6d21f521 100644 --- a/tappd/src/rpc_service.rs +++ b/tappd/src/rpc_service.rs @@ -1,7 +1,6 @@ -use std::{path::Path, sync::Arc}; +use std::sync::Arc; use anyhow::{bail, Context, Result}; -use bollard::{container::ListContainersOptions, Docker}; use ra_rpc::{CallContext, RpcCall}; use ra_tls::{ attestation::QuoteContentType, @@ -13,8 +12,7 @@ use serde_json::json; use tappd_rpc::{ tappd_server::{TappdRpc, TappdServer}, worker_server::{WorkerRpc, WorkerServer}, - Container, DeriveKeyArgs, DeriveKeyResponse, DiskInfo, ListContainersResponse, SystemInfo, - TdxQuoteArgs, TdxQuoteResponse, WorkerInfo, + DeriveKeyArgs, DeriveKeyResponse, TdxQuoteArgs, TdxQuoteResponse, WorkerInfo, }; use tdx_attest::eventlog::read_event_logs; @@ -102,6 +100,12 @@ pub struct ExternalRpcHandler { state: AppState, } +impl ExternalRpcHandler { + pub(crate) fn new(state: AppState) -> Self { + Self { state } + } +} + impl WorkerRpc for ExternalRpcHandler { async fn info(self) -> Result { let ca = &self.state.inner.ca; @@ -148,78 +152,6 @@ impl WorkerRpc for ExternalRpcHandler { tcb_info, }) } - - async fn sys_info(self) -> Result { - use sysinfo::System; - - let system = System::new_all(); - let cpus = system.cpus(); - - let disks = sysinfo::Disks::new_with_refreshed_list(); - let disks = disks - .list() - .iter() - .filter(|d| d.mount_point() == Path::new("/")) - .map(|d| DiskInfo { - name: d.name().to_string_lossy().to_string(), - mount_point: d.mount_point().to_string_lossy().to_string(), - total_size: d.total_space(), - free_size: d.available_space(), - }) - .collect::>(); - let avg = System::load_average(); - Ok(SystemInfo { - os_name: System::name().unwrap_or_default(), - os_version: System::os_version().unwrap_or_default(), - kernel_version: System::kernel_version().unwrap_or_default(), - cpu_model: cpus.first().map_or("".into(), |cpu| { - format!("{} @{} MHz", cpu.name(), cpu.frequency()) - }), - num_cpus: cpus.len() as _, - total_memory: system.total_memory(), - available_memory: system.available_memory(), - used_memory: system.used_memory(), - free_memory: system.free_memory(), - total_swap: system.total_swap(), - used_swap: system.used_swap(), - free_swap: system.free_swap(), - uptime: System::uptime(), - loadavg_one: (avg.one * 100.0) as u32, - loadavg_five: (avg.five * 100.0) as u32, - loadavg_fifteen: (avg.fifteen * 100.0) as u32, - disks, - }) - } - - async fn list_containers(self) -> Result { - list_containers().await - } -} - -pub(crate) async fn list_containers() -> Result { - let docker = Docker::connect_with_defaults().context("Failed to connect to Docker")?; - let containers = docker - .list_containers::<&str>(Some(ListContainersOptions { - all: true, - ..Default::default() - })) - .await - .context("Failed to list containers")?; - Ok(ListContainersResponse { - containers: containers - .into_iter() - .map(|c| Container { - id: c.id.unwrap_or_default(), - names: c.names.unwrap_or_default(), - image: c.image.unwrap_or_default(), - image_id: c.image_id.unwrap_or_default(), - command: c.command.unwrap_or_default(), - created: c.created.unwrap_or_default(), - state: c.state.unwrap_or_default(), - status: c.status.unwrap_or_default(), - }) - .collect(), - }) } impl RpcCall for ExternalRpcHandler { diff --git a/tappd/tappd.toml b/tappd/tappd.toml index 644316f9..d5918cab 100644 --- a/tappd/tappd.toml +++ b/tappd/tappd.toml @@ -27,5 +27,5 @@ key = "/etc/tappd/tls.key" certs = "/etc/tappd/tls.cert" [guest-api] -address = "vsock:0" +address = "vsock:0xffffffff" port = 8000 diff --git a/teepod/rpc/proto/teepod_rpc.proto b/teepod/rpc/proto/teepod_rpc.proto index aad361a9..b9ed109a 100644 --- a/teepod/rpc/proto/teepod_rpc.proto +++ b/teepod/rpc/proto/teepod_rpc.proto @@ -29,6 +29,8 @@ message VmInfo { string boot_error = 11; // Shutdown progress string shutdown_progress = 12; + // Image version + string image_version = 13; } message Id { @@ -122,30 +124,6 @@ message ResizeVmRequest { optional uint32 disk_size = 4; } -message IpAddress { - string address = 1; - uint32 prefix = 2; -} - -message Interface { - string name = 1; - repeated IpAddress addresses = 2; - uint64 rx_bytes = 3; - uint64 tx_bytes = 4; - uint64 rx_errors = 5; - uint64 tx_errors = 6; -} - -message Gateway { - string address = 1; -} - -message NetworkInformation { - repeated string dns_servers = 1; - repeated Gateway gateways = 2; - repeated Interface interfaces = 3; -} - // Service definition for Teepod service Teepod { // RPC to create a VM @@ -172,9 +150,6 @@ service Teepod { // Get VM info by ID rpc GetInfo(Id) returns (GetInfoResponse); - // Get VM Network Information - rpc GetNetworkInfo(Id) returns (NetworkInformation); - // RPC to resize a VM rpc ResizeVm(ResizeVmRequest) returns (google.protobuf.Empty); } diff --git a/teepod/src/app.rs b/teepod/src/app.rs index 7bcdd7f6..afa0dba5 100644 --- a/teepod/src/app.rs +++ b/teepod/src/app.rs @@ -151,7 +151,12 @@ impl App { let process_config = vm_state .config .config_qemu(&self.config.qemu_path, &work_dir)?; - vm_state.state.clear(); + // Older images does not support for progress reporting + if vm_state.config.image.info.shared_ro { + vm_state.state.clear(); + } else { + vm_state.state.reset_na(); + } process_config }; self.supervisor @@ -177,7 +182,10 @@ impl App { bail!("VM is running, stop it first"); } - if info.is_some() { + if let Some(info) = info { + if !info.state.status.is_stopped() { + self.supervisor.stop(id).await?; + } self.supervisor.remove(id).await?; } @@ -324,6 +332,12 @@ impl VmStateMut { self.boot_error.clear(); self.shutdown_progress.clear(); } + + pub fn reset_na(&mut self) { + self.boot_progress = "N/A".to_string(); + self.shutdown_progress = "N/A".to_string(); + self.boot_error.clear(); + } } impl VmState { diff --git a/teepod/src/app/image.rs b/teepod/src/app/image.rs index 1efd2379..d09df8f2 100644 --- a/teepod/src/app/image.rs +++ b/teepod/src/app/image.rs @@ -16,6 +16,8 @@ pub struct ImageInfo { pub rootfs_hash: Option, #[serde(default)] pub shared_ro: bool, + #[serde(default)] + pub version: String, } impl ImageInfo { @@ -35,19 +37,21 @@ pub struct Image { pub hda: Option, pub rootfs: Option, pub bios: Option, - pub shared_ro: bool, } impl Image { pub fn load(base_path: impl AsRef) -> Result { let base_path = base_path.as_ref().absolutize()?; - let info = ImageInfo::load(base_path.join("metadata.json"))?; + let mut info = ImageInfo::load(base_path.join("metadata.json"))?; let initrd = base_path.join(&info.initrd); let kernel = base_path.join(&info.kernel); let hda = info.hda.as_ref().map(|hda| base_path.join(hda)); let rootfs = info.rootfs.as_ref().map(|rootfs| base_path.join(rootfs)); let bios = info.bios.as_ref().map(|bios| base_path.join(bios)); - let shared_ro = info.shared_ro; + if info.version.is_empty() { + // Older images does not have version field. Fallback to the version of the image folder name + info.version = guess_version(&base_path).unwrap_or_default(); + } Self { info, hda, @@ -55,7 +59,6 @@ impl Image { kernel, rootfs, bios, - shared_ro, } .ensure_exists() } @@ -85,3 +88,16 @@ impl Image { Ok(self) } } + +fn guess_version(base_path: &Path) -> Option { + // name pattern: dstack-dev-0.2.3 or dstack-0.2.3 + let basename = base_path.file_name()?.to_str()?.to_string(); + let version = if basename.starts_with("dstack-dev-") { + basename.strip_prefix("dstack-dev-")? + } else if basename.starts_with("dstack-") { + basename.strip_prefix("dstack-")? + } else { + return None; + }; + Some(version.to_string()) +} diff --git a/teepod/src/app/qemu.rs b/teepod/src/app/qemu.rs index 06f1ce7b..73d378ff 100644 --- a/teepod/src/app/qemu.rs +++ b/teepod/src/app/qemu.rs @@ -33,6 +33,7 @@ pub struct VmInfo { pub boot_progress: String, pub boot_error: String, pub shutdown_progress: String, + pub image_version: String, } #[derive(Debug, Builder)] @@ -79,6 +80,7 @@ impl VmInfo { boot_progress: self.boot_progress.clone(), boot_error: self.boot_error.clone(), shutdown_progress: self.shutdown_progress.clone(), + image_version: self.image_version.clone(), configuration: Some(pb::VmConfiguration { name: self.manifest.name.clone(), image: self.manifest.image.clone(), @@ -154,6 +156,7 @@ impl VmState { boot_progress: self.state.boot_progress.clone(), boot_error: self.state.boot_error.clone(), shutdown_progress: self.state.shutdown_progress.clone(), + image_version: self.config.image.info.version.clone(), } } } @@ -230,7 +233,11 @@ impl VmConfig { .arg("-device") .arg(format!("vhost-vsock-pci,guest-cid={}", self.cid)); - let ro = if self.image.shared_ro { "on" } else { "off" }; + let ro = if self.image.info.shared_ro { + "on" + } else { + "off" + }; command.arg("-virtfs").arg(format!( "local,path={},mount_tag=host-shared,readonly={ro},security_model=mapped,id=virtfs0", shared_dir.display(), diff --git a/teepod/src/console.html b/teepod/src/console.html index 40d5bc90..ced6f3ab 100644 --- a/teepod/src/console.html +++ b/teepod/src/console.html @@ -641,6 +641,10 @@

VM List

+ @@ -679,6 +683,33 @@

Boot Status

+
+

Network Information

+
+
+ DNS Servers: + {{ networkInfo[vm.id].dns_servers.join(', ') + }} +
+
+ Gateways: + {{ networkInfo[vm.id].gateways.map(gw => + gw.address).join(', ') }} +
+
+ Interface {{ iface.name }}: +
+
IP: {{ iface.addresses.map(addr => + `${addr.address}/${addr.prefix}`).join(', ') }}
+
RX: {{ iface.rx_bytes }} bytes ({{ iface.rx_errors }} errors) +
+
TX: {{ iface.tx_bytes }} bytes ({{ iface.tx_errors }} errors) +
+
+
+
+

VM Configuration

@@ -866,6 +897,7 @@

Derive VM

setup() { const vms = ref([]); const expandedVMs = ref(new Set()); + const networkInfo = ref({}); const vmForm = ref({ name: '', image: '', @@ -915,6 +947,14 @@

Derive VM

const teepodApiToken = ref(localStorage.getItem('teepodApiToken') || ''); const rpcCall = async (method, params) => { + return await baseRpcCall(`/prpc/Teepod.${method}`, params) + } + + const guestRpcCall = async (method, params) => { + return await baseRpcCall(`/guest/${method}`, params) + } + + const baseRpcCall = async (path, params) => { const headers = { 'Content-Type': 'application/json', }; @@ -923,7 +963,7 @@

Derive VM

headers['Authorization'] = `Bearer ${teepodApiToken.value}`; } - const response = await fetch(`/prpc/Teepod.${method}?json`, { + const response = await fetch(`${path}?json`, { method: 'POST', headers, body: JSON.stringify(params || {}), @@ -1064,6 +1104,18 @@

Derive VM

} }; + const shutdownVm = async (id) => { + try { + const _response = await rpcCall('ShutdownVm', { + id + }); + loadVMList(); + } catch (error) { + console.error('Error shutting down VM:', error); + alert('Failed to shut down VM'); + } + }; + const startVm = async (id) => { try { const _response = await rpcCall('StartVm', { @@ -1220,9 +1272,18 @@

Derive VM

expandedVMs.value.delete(vmId); } else { expandedVMs.value.add(vmId); + refreshNetworkInfo(vmId); } }; + const refreshNetworkInfo = async (vmId) => { + const response = await guestRpcCall('NetworkInfo', { + id: vmId + }); + const data = await response.json(); + networkInfo.value[vmId] = data; + }; + const addEncryptedEnv = () => { vmForm.value.encrypted_envs.push({ key: '', @@ -1303,14 +1364,36 @@

Derive VM

} }; + const imageFeatures = (vm) => { + const features = { + progress: false, + graceful_shutdown: false, + network_info: false, + } + if (!vm.image_version) { + return features; + } + const version = vm.image_version.split('.').map(Number); + if (version[0] >= 0 && version[1] >= 3 && version[2] >= 3) { + features.progress = true; + features.graceful_shutdown = true; + features.network_info = true; + } + return features; + }; + const vmStatus = (vm) => { + const features = imageFeatures(vm); + if (!features.progress) { + return vm.status; + } if (vm.status != 'running') { return vm.status; } - if (vm.state.shutdown_progress) { + if (vm.shutdown_progress) { return "shutting down"; } - if (vm.state.boot_progress != 'done') { + if (vm.boot_progress != 'done') { return "booting"; } return "running"; @@ -1339,6 +1422,7 @@

Derive VM

config, toggleDetails, expandedVMs, + networkInfo, dashboardAvailable, addEncryptedEnv, removeEncryptedEnv, @@ -1348,7 +1432,9 @@

Derive VM

forkDialog, forkVm, loadEnvFile, - vmStatus + vmStatus, + imageFeatures, + shutdownVm, }; } }).mount('#app'); diff --git a/teepod/src/guest_api_routes.rs b/teepod/src/guest_api_routes.rs new file mode 100644 index 00000000..f6496f46 --- /dev/null +++ b/teepod/src/guest_api_routes.rs @@ -0,0 +1,58 @@ +use crate::{guest_api_service::GuestApiHandler, App}; +use ra_rpc::rocket_helper::PrpcHandler; + +use rocket::{ + data::{Data, Limits}, + get, + http::ContentType, + mtls::Certificate, + post, + response::status::Custom, + routes, Route, State, +}; + +#[post("/?", data = "")] +#[allow(clippy::too_many_arguments)] +async fn prpc_post( + state: &State, + cert: Option>, + method: &str, + data: Data<'_>, + limits: &Limits, + content_type: Option<&ContentType>, + json: bool, +) -> Custom> { + PrpcHandler::builder() + .state(&**state) + .maybe_certificate(cert) + .method(method) + .data(data) + .limits(limits) + .maybe_content_type(content_type) + .json(json) + .build() + .handle::() + .await +} + +#[get("/")] +async fn prpc_get( + state: &State, + method: &str, + limits: &Limits, + content_type: Option<&ContentType>, +) -> Custom> { + PrpcHandler::builder() + .state(&**state) + .method(method) + .limits(limits) + .maybe_content_type(content_type) + .json(true) + .build() + .handle::() + .await +} + +pub fn routes() -> Vec { + routes![prpc_post, prpc_get] +} diff --git a/teepod/src/guest_api_service.rs b/teepod/src/guest_api_service.rs new file mode 100644 index 00000000..74836039 --- /dev/null +++ b/teepod/src/guest_api_service.rs @@ -0,0 +1,74 @@ +use crate::App as AppState; +use anyhow::Result; +use guest_api::{ + proxied_guest_api_server::{ProxiedGuestApiRpc, ProxiedGuestApiServer}, + GuestInfo, Id, ListContainersResponse, NetworkInformation, SystemInfo, +}; +use ra_rpc::{CallContext, RpcCall}; +use std::ops::Deref; + +pub struct GuestApiHandler { + state: AppState, +} + +impl Deref for GuestApiHandler { + type Target = AppState; + + fn deref(&self) -> &Self::Target { + &self.state + } +} + +impl RpcCall for GuestApiHandler { + type PrpcService = ProxiedGuestApiServer; + + fn into_prpc_service(self) -> Self::PrpcService { + ProxiedGuestApiServer::new(self) + } + + fn construct(context: CallContext<'_, AppState>) -> Result + where + Self: Sized, + { + Ok(Self { + state: context.state.clone(), + }) + } +} + +impl ProxiedGuestApiRpc for GuestApiHandler { + async fn info(self, request: Id) -> Result { + self.tappd_client(&request.id)? + .info() + .await + .map_err(Into::into) + } + + async fn sys_info(self, request: Id) -> Result { + self.tappd_client(&request.id)? + .sys_info() + .await + .map_err(Into::into) + } + + async fn network_info(self, request: Id) -> Result { + self.tappd_client(&request.id)? + .network_info() + .await + .map_err(Into::into) + } + + async fn list_containers(self, request: Id) -> Result { + self.tappd_client(&request.id)? + .list_containers() + .await + .map_err(Into::into) + } + + async fn shutdown(self, request: Id) -> Result<()> { + self.tappd_client(&request.id)? + .shutdown() + .await + .map_err(Into::into) + } +} diff --git a/teepod/src/main.rs b/teepod/src/main.rs index be27cd79..fdd035aa 100644 --- a/teepod/src/main.rs +++ b/teepod/src/main.rs @@ -15,6 +15,8 @@ use supervisor_client::SupervisorClient; mod app; mod config; +mod guest_api_routes; +mod guest_api_service; mod host_api_routes; mod host_api_service; mod main_routes; @@ -41,6 +43,7 @@ struct Args { async fn run_external_api(app: App, figment: Figment, api_auth: ApiToken) -> Result<()> { let external_api = rocket::custom(figment) .mount("/", main_routes::routes()) + .mount("/guest", guest_api_routes::routes()) .manage(app) .manage(api_auth) .attach(AdHoc::on_response("Add app rev header", |_req, res| { diff --git a/teepod/src/main_service.rs b/teepod/src/main_service.rs index 60f03236..1eee61c9 100644 --- a/teepod/src/main_service.rs +++ b/teepod/src/main_service.rs @@ -1,3 +1,4 @@ +use std::ops::Deref; use std::path::PathBuf; use std::time::{SystemTime, UNIX_EPOCH}; @@ -9,9 +10,8 @@ use ra_rpc::client::RaClient; use ra_rpc::{CallContext, RpcCall}; use teepod_rpc::teepod_server::{TeepodRpc, TeepodServer}; use teepod_rpc::{ - AppId, Gateway, GetInfoResponse, Id, ImageInfo as RpcImageInfo, ImageListResponse, Interface, - IpAddress, NetworkInformation, PublicKeyResponse, ResizeVmRequest, StatusResponse, - UpgradeAppRequest, VmConfiguration, + AppId, GetInfoResponse, Id, ImageInfo as RpcImageInfo, ImageListResponse, PublicKeyResponse, + ResizeVmRequest, StatusResponse, UpgradeAppRequest, VmConfiguration, }; use tracing::warn; @@ -28,21 +28,29 @@ pub struct RpcHandler { app: App, } -impl RpcHandler { - fn compose_file_path(&self, id: &str) -> PathBuf { +impl Deref for RpcHandler { + type Target = App; + + fn deref(&self) -> &Self::Target { + &self.app + } +} + +impl App { + pub(crate) fn compose_file_path(&self, id: &str) -> PathBuf { self.shared_dir(id).join("app-compose.json") } - fn encrypted_env_path(&self, id: &str) -> PathBuf { + pub(crate) fn encrypted_env_path(&self, id: &str) -> PathBuf { self.shared_dir(id).join("encrypted-env") } - fn shared_dir(&self, id: &str) -> PathBuf { - self.app.config.run_path.join(id).join("shared") + pub(crate) fn shared_dir(&self, id: &str) -> PathBuf { + self.config.run_path.join(id).join("shared") } - fn prepare_work_dir(&self, id: &str, req: &VmConfiguration) -> Result { - let work_dir = self.app.work_dir(id); + pub(crate) fn prepare_work_dir(&self, id: &str, req: &VmConfiguration) -> Result { + let work_dir = self.work_dir(id); if work_dir.exists() { anyhow::bail!("The instance is already exists at {}", work_dir.display()); } @@ -57,7 +65,7 @@ impl RpcHandler { let certs_dir = shared_dir.join("certs"); fs::create_dir_all(&certs_dir).context("Failed to create certs directory")?; - let cfg = &self.app.config; + let cfg = &self.config; fs::copy(&cfg.cvm.ca_cert, certs_dir.join("ca.cert")).context("Failed to copy ca cert")?; fs::copy(&cfg.cvm.tmp_ca_cert, certs_dir.join("tmp-ca.cert")) .context("Failed to copy tmp ca cert")?; @@ -97,18 +105,18 @@ impl RpcHandler { Ok(work_dir) } - fn kms_client(&self) -> Result> { - if self.app.config.kms_url.is_empty() { + pub(crate) fn kms_client(&self) -> Result> { + if self.config.kms_url.is_empty() { anyhow::bail!("KMS is not configured"); } - let url = format!("{}/prpc", self.app.config.kms_url); + let url = format!("{}/prpc", self.config.kms_url); let prpc_client = RaClient::new(url, true); Ok(KmsClient::new(prpc_client)) } - fn tappd_client(&self, id: &str) -> Option { - let cid = self.app.lock().get(id)?.config.cid; - Some(guest_api::client::new_client(format!( + pub(crate) fn tappd_client(&self, id: &str) -> Result { + let cid = self.lock().get(id).context("vm not found")?.config.cid; + Ok(guest_api::client::new_client(format!( "vsock://{cid}:8000/api" ))) } @@ -355,42 +363,9 @@ impl TeepodRpc for RpcHandler { } async fn shutdown_vm(self, request: Id) -> Result<()> { - let tappd_client = self.tappd_client(&request.id).context("vm not found")?; - tappd_client.shutdown().await?; + self.tappd_client(&request.id)?.shutdown().await?; Ok(()) } - - async fn get_network_info(self, request: Id) -> Result { - let tappd_client = self.tappd_client(&request.id).context("vm not found")?; - let info = tappd_client.network_info().await?; - Ok(NetworkInformation { - dns_servers: info.dns_servers, - gateways: info - .gateways - .into_iter() - .map(|g| Gateway { address: g.address }) - .collect(), - interfaces: info - .interfaces - .into_iter() - .map(|i| Interface { - name: i.name, - addresses: i - .addresses - .into_iter() - .map(|a| IpAddress { - address: a.address, - prefix: a.prefix, - }) - .collect(), - rx_bytes: i.rx_bytes, - tx_bytes: i.tx_bytes, - rx_errors: i.rx_errors, - tx_errors: i.tx_errors, - }) - .collect(), - }) - } } impl RpcCall for RpcHandler { From d60c30e0ba7588e00010b28b1f94823a4846ba72 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 18 Dec 2024 02:05:43 +0000 Subject: [PATCH 4/5] teepod: Adjust buttons layout --- teepod/src/console.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/teepod/src/console.html b/teepod/src/console.html index ced6f3ab..ea8b0477 100644 --- a/teepod/src/console.html +++ b/teepod/src/console.html @@ -641,12 +641,12 @@

VM List

-