Skip to content

Commit

Permalink
teepod: Accept boot progress report
Browse files Browse the repository at this point in the history
  • Loading branch information
kvinwang committed Dec 17, 2024
1 parent 8375c74 commit 02dc2e0
Show file tree
Hide file tree
Showing 5 changed files with 111 additions and 50 deletions.
6 changes: 6 additions & 0 deletions teepod/rpc/proto/teepod_rpc.proto
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,12 @@ message VmInfo {
VmConfiguration configuration = 8;
// Exit time
optional string exited_at = 9;
// Boot progress
string boot_progress = 10;
// Network interface eth0
string eth0 = 11;
// Network interface wg0
string wg0 = 12;
}

message Id {
Expand Down
89 changes: 69 additions & 20 deletions teepod/src/app.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ use teepod_rpc as pb;
use tracing::error;

pub use image::{Image, ImageInfo};
pub use qemu::{TdxConfig, VmConfig, VmWorkDir};
pub use qemu::{VmConfig, VmWorkDir};

mod id_pool;
mod image;
Expand Down Expand Up @@ -116,7 +116,7 @@ impl App {
let vm_config = VmConfig {
manifest,
image,
tdx_config: Some(TdxConfig { cid }),
cid,
networking: self.config.networking.clone(),
};
if vm_config.manifest.disk_size > self.config.cvm.max_disk_size {
Expand All @@ -126,7 +126,7 @@ impl App {
);
}
let vm_id = vm_config.manifest.id.clone();
self.lock().add(vm_config);
self.lock().add(VmState::new(vm_config));
let started = vm_work_dir.started().context("Failed to read VM state")?;
if started {
self.start_vm(&vm_id).await?;
Expand All @@ -135,7 +135,7 @@ impl App {
}

pub async fn start_vm(&self, id: &str) -> Result<()> {
let vm_config = self.lock().get(id).context("VM not found")?;
let vm_state = self.lock().get(id).context("VM not found")?;
let work_dir = self.work_dir(id);
work_dir
.set_started(true)
Expand All @@ -144,7 +144,9 @@ impl App {
// remove the existing pty
fs::remove_file(work_dir.serial_pty()).context("Failed to remove existing pty link")?;
}
let process_config = vm_config.config_qemu(&self.config.qemu_path, &work_dir)?;
let process_config = vm_state
.config
.config_qemu(&self.config.qemu_path, &work_dir)?;
self.supervisor
.deploy(process_config)
.await
Expand Down Expand Up @@ -174,10 +176,8 @@ impl App {

{
let mut state = self.lock();
if let Some(config) = state.remove(id) {
if let Some(cfg) = &config.tdx_config {
state.cid_pool.free(cfg.cid);
}
if let Some(vm_state) = state.remove(id) {
state.cid_pool.free(vm_state.config.cid);
}
}

Expand Down Expand Up @@ -226,7 +226,12 @@ impl App {
let mut infos = self
.lock()
.iter_vms()
.map(|vm| vm.merge_info(vms.get(&vm.manifest.id), &self.work_dir(&vm.manifest.id)))
.map(|vm| {
vm.merged_info(
vms.get(&vm.config.manifest.id),
&self.work_dir(&vm.config.manifest.id),
)
})
.collect::<Vec<_>>();

infos.sort_by(|a, b| a.manifest.created_at_ms.cmp(&b.manifest.created_at_ms));
Expand All @@ -248,37 +253,81 @@ impl App {
.collect())
}

pub async fn get_vm(&self, id: &str) -> Result<Option<pb::VmInfo>> {
pub async fn vm_info(&self, id: &str) -> Result<Option<pb::VmInfo>> {
let proc_state = self.supervisor.info(id).await?;
let Some(cfg) = self.lock().get(id) else {
let Some(vm_state) = self.lock().get(id) else {
return Ok(None);
};
let info = cfg
.merge_info(proc_state.as_ref(), &self.work_dir(id))
let info = vm_state
.merged_info(proc_state.as_ref(), &self.work_dir(id))
.to_pb(&self.config.gateway);
Ok(Some(info))
}

pub(crate) fn vm_event_report(&self, cid: u32, event: &str, message: &str) -> Result<()> {
let mut state = self.lock();
let Some(vm) = state.vms.values_mut().find(|vm| vm.config.cid == cid) else {
bail!("VM not found");
};
match event {
"boot.progress" => {
vm.state.boot_progress = message.to_string();
}
"net.eth0" => {
vm.state.eth0 = message.to_string();
}
"net.wg0" => {
vm.state.wg0 = message.to_string();
}
_ => {
error!("Guest reported unknown event: {event}");
}
}
Ok(())
}
}

#[derive(Clone)]
pub struct VmState {
config: Arc<VmConfig>,
state: VmStateMut,
}

#[derive(Debug, Clone, Default)]
struct VmStateMut {
boot_progress: String,
eth0: String,
wg0: String,
}

impl VmState {
pub fn new(config: VmConfig) -> Self {
Self {
config: Arc::new(config),
state: VmStateMut::default(),
}
}
}

pub(crate) struct AppState {
cid_pool: IdPool<u32>,
vms: HashMap<String, Arc<VmConfig>>,
vms: HashMap<String, VmState>,
}

impl AppState {
pub fn add(&mut self, vm: VmConfig) {
self.vms.insert(vm.manifest.id.clone(), Arc::new(vm));
pub fn add(&mut self, vm: VmState) {
self.vms.insert(vm.config.manifest.id.clone(), vm);
}

pub fn get(&self, id: &str) -> Option<Arc<VmConfig>> {
pub fn get(&self, id: &str) -> Option<VmState> {
self.vms.get(id).cloned()
}

pub fn remove(&mut self, id: &str) -> Option<Arc<VmConfig>> {
pub fn remove(&mut self, id: &str) -> Option<VmState> {
self.vms.remove(id)
}

pub fn iter_vms(&self) -> impl Iterator<Item = &Arc<VmConfig>> {
pub fn iter_vms(&self) -> impl Iterator<Item = &VmState> {
self.vms.values()
}
}
59 changes: 32 additions & 27 deletions teepod/src/app/qemu.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ use std::{
time::{Duration, SystemTime},
};

use super::image::Image;
use super::{image::Image, VmState};
use anyhow::{Context, Result};
use bon::Builder;
use fs_err as fs;
Expand All @@ -30,19 +30,16 @@ pub struct VmInfo {
pub uptime: String,
pub exited_at: Option<String>,
pub instance_id: Option<String>,
}

#[derive(Debug)]
pub struct TdxConfig {
/// Guest CID for vhost-vsock
pub cid: u32,
pub boot_progress: String,
pub eth0: String,
pub wg0: String,
}

#[derive(Debug, Builder)]
pub struct VmConfig {
pub manifest: Manifest,
pub image: Image,
pub tdx_config: Option<TdxConfig>,
pub cid: u32,
pub networking: Networking,
}

Expand Down Expand Up @@ -74,13 +71,16 @@ impl VmInfo {
pub fn to_pb(&self, gw: &GatewayConfig) -> pb::VmInfo {
let workdir = VmWorkDir::new(&self.workdir);
pb::VmInfo {
id: self.manifest.id.as_str().into(),
name: self.manifest.name.as_str().into(),
id: self.manifest.id.clone(),
name: self.manifest.name.clone(),
status: self.status.into(),
uptime: self.uptime.as_str().into(),
uptime: self.uptime.clone(),
boot_progress: self.boot_progress.clone(),
eth0: self.eth0.clone(),
wg0: self.wg0.clone(),
configuration: Some(pb::VmConfiguration {
name: self.manifest.name.as_str().into(),
image: self.manifest.image.as_str().into(),
name: self.manifest.name.clone(),
image: self.manifest.image.clone(),
compose_file: {
fs::read_to_string(workdir.app_compose_path()).unwrap_or_default()
},
Expand All @@ -106,15 +106,15 @@ impl VmInfo {
gw.tappd_port, gw.base_domain, gw.port
)
}),
app_id: self.manifest.app_id.as_str().into(),
app_id: self.manifest.app_id.clone(),
instance_id: self.instance_id.as_deref().map(Into::into),
exited_at: self.exited_at.clone(),
}
}
}

impl VmConfig {
pub fn merge_info(&self, proc_state: Option<&ProcessInfo>, workdir: &VmWorkDir) -> VmInfo {
impl VmState {
pub fn merged_info(&self, proc_state: Option<&ProcessInfo>, workdir: &VmWorkDir) -> VmInfo {
fn truncate(d: Duration) -> Duration {
Duration::from_secs(d.as_secs())
}
Expand Down Expand Up @@ -144,15 +144,20 @@ impl VmConfig {
let exited_at = display_ts(proc_state.and_then(|info| info.state.stopped_at.as_ref()));
let instance_id = workdir.instance_info().ok().map(|info| info.instance_id);
VmInfo {
manifest: self.manifest.clone(),
manifest: self.config.manifest.clone(),
workdir: workdir.path().to_path_buf(),
instance_id,
status,
uptime,
exited_at: Some(exited_at),
boot_progress: self.state.boot_progress.clone(),
eth0: self.state.eth0.clone(),
wg0: self.state.wg0.clone(),
}
}
}

impl VmConfig {
pub fn config_qemu(&self, qemu: &Path, workdir: impl AsRef<Path>) -> Result<ProcessConfig> {
let workdir = VmWorkDir::new(workdir);
let serial_file = workdir.serial_file();
Expand Down Expand Up @@ -215,15 +220,15 @@ impl VmConfig {
};
command.arg("-netdev").arg(netdev);
command.arg("-device").arg("virtio-net-pci,netdev=net0");
if let Some(tdx) = &self.tdx_config {
command
.arg("-machine")
.arg("q35,kernel-irqchip=split,confidential-guest-support=tdx,hpet=off");
command.arg("-object").arg("tdx-guest,id=tdx");
command
.arg("-device")
.arg(format!("vhost-vsock-pci,guest-cid={}", tdx.cid));
}

command
.arg("-machine")
.arg("q35,kernel-irqchip=split,confidential-guest-support=tdx,hpet=off");
command.arg("-object").arg("tdx-guest,id=tdx");
command
.arg("-device")
.arg(format!("vhost-vsock-pci,guest-cid={}", self.cid));

command.arg("-virtfs").arg(format!(
"local,path={},mount_tag=host-shared,readonly=off,security_model=mapped,id=virtfs0",
shared_dir.display()
Expand Down Expand Up @@ -252,7 +257,7 @@ impl VmConfig {
stdout: stdout_path.to_string_lossy().to_string(),
stderr: stderr_path.to_string_lossy().to_string(),
pidfile: pidfile_path.to_string_lossy().to_string(),
cid: self.tdx_config.as_ref().map(|cfg| cfg.cid),
cid: Some(self.cid),
note: "".into(),
};
Ok(process_config)
Expand Down
3 changes: 2 additions & 1 deletion teepod/src/host_api_service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ impl HostApiRpc for HostApiHandler {
}

async fn notify(self, request: Notification) -> Result<()> {
todo!()
self.app
.vm_event_report(self.endpoint.cid, &request.event, &request.message)
}
}
4 changes: 2 additions & 2 deletions teepod/src/main_service.rs
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ impl TeepodRpc for RpcHandler {
}

async fn get_info(self, request: Id) -> Result<GetInfoResponse> {
if let Some(vm) = self.app.get_vm(&request.id).await? {
if let Some(vm) = self.app.vm_info(&request.id).await? {
Ok(GetInfoResponse {
found: true,
info: Some(vm),
Expand All @@ -313,7 +313,7 @@ impl TeepodRpc for RpcHandler {
async fn resize_vm(self, request: ResizeVmRequest) -> Result<()> {
let vm = self
.app
.get_vm(&request.id)
.vm_info(&request.id)
.await?
.ok_or_else(|| anyhow::anyhow!("vm not found: {}", request.id))?;
if vm.status != "stopped" {
Expand Down

0 comments on commit 02dc2e0

Please sign in to comment.