From acd7389c75847d9af0d0995cc3d00bf3b402bd59 Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Sun, 31 Dec 2023 17:25:55 +0100 Subject: [PATCH 01/23] Rename `StartCommand` to `RunCommand` --- src/podman.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/podman.rs b/src/podman.rs index dd06ffd..0da6b06 100644 --- a/src/podman.rs +++ b/src/podman.rs @@ -89,8 +89,8 @@ impl Podman { Ok(()) } - pub(crate) fn run(&self, image_url: &str) -> StartCommand { - StartCommand { + pub(crate) fn run(&self, image_url: &str) -> RunCommand { + RunCommand { podman: self, image_url: image_url.to_owned(), rm: false, @@ -129,7 +129,7 @@ impl Podman { } } -pub(crate) struct StartCommand<'a> { +pub(crate) struct RunCommand<'a> { podman: &'a Podman, env: Vec<(String, String)>, image_url: String, @@ -140,7 +140,7 @@ pub(crate) struct StartCommand<'a> { publish: Vec, } -impl<'a> StartCommand<'a> { +impl<'a> RunCommand<'a> { pub fn env, S2: Into>(&mut self, var: S1, value: S2) -> &mut Self { self.env.push((var.into(), value.into())); self From 6dcae6290f1e912071a513427ef77b05a2a12f33 Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Sun, 31 Dec 2023 17:29:48 +0100 Subject: [PATCH 02/23] Add partial annotations support --- src/podman.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/podman.rs b/src/podman.rs index 0da6b06..a3c7181 100644 --- a/src/podman.rs +++ b/src/podman.rs @@ -91,6 +91,7 @@ impl Podman { pub(crate) fn run(&self, image_url: &str) -> RunCommand { RunCommand { + annotations: Vec::new(), podman: self, image_url: image_url.to_owned(), rm: false, @@ -130,6 +131,7 @@ impl Podman { } pub(crate) struct RunCommand<'a> { + annotations: Vec<(String, String)>, podman: &'a Podman, env: Vec<(String, String)>, image_url: String, @@ -146,6 +148,16 @@ impl<'a> RunCommand<'a> { self } + #[inline] + pub fn annotate, S2: Into>( + &mut self, + key: S1, + value: S2, + ) -> &mut Self { + self.annotations.push((key.into(), value.into())); + self + } + #[inline] pub fn name>(&mut self, name: S) -> &mut Self { self.name = Some(name.into()); From 26f19e77b337c7529c4d65edc8d5498ccbbbe64e Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Sun, 31 Dec 2023 17:29:52 +0100 Subject: [PATCH 03/23] Revert "Add partial annotations support" This reverts commit 6dcae6290f1e912071a513427ef77b05a2a12f33. --- src/podman.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/podman.rs b/src/podman.rs index a3c7181..0da6b06 100644 --- a/src/podman.rs +++ b/src/podman.rs @@ -91,7 +91,6 @@ impl Podman { pub(crate) fn run(&self, image_url: &str) -> RunCommand { RunCommand { - annotations: Vec::new(), podman: self, image_url: image_url.to_owned(), rm: false, @@ -131,7 +130,6 @@ impl Podman { } pub(crate) struct RunCommand<'a> { - annotations: Vec<(String, String)>, podman: &'a Podman, env: Vec<(String, String)>, image_url: String, @@ -148,16 +146,6 @@ impl<'a> RunCommand<'a> { self } - #[inline] - pub fn annotate, S2: Into>( - &mut self, - key: S1, - value: S2, - ) -> &mut Self { - self.annotations.push((key.into(), value.into())); - self - } - #[inline] pub fn name>(&mut self, name: S) -> &mut Self { self.name = Some(name.into()); From df46e9d81c9352ba285119f6598b4b4aaf2cd2c3 Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Mon, 1 Jan 2024 02:00:11 +0100 Subject: [PATCH 04/23] Rename `RegistryHooks` to `ContainerOrchestrator` --- src/main.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index 82bc784..7587068 100644 --- a/src/main.rs +++ b/src/main.rs @@ -38,14 +38,14 @@ macro_rules! try_quiet { }; } -struct PodmanHook { +struct ContainerOrchestrator { podman: Podman, reverse_proxy: Arc, local_addr: SocketAddr, registry_credentials: (String, Secret), } -impl PodmanHook { +impl ContainerOrchestrator { fn new>( podman_path: P, reverse_proxy: Arc, @@ -161,7 +161,7 @@ impl PortMapping { } #[async_trait] -impl RegistryHooks for PodmanHook { +impl RegistryHooks for ContainerOrchestrator { async fn on_manifest_uploaded(&self, manifest_reference: &ManifestReference) { // TODO: Make configurable? let production_tag = "prod"; @@ -286,7 +286,7 @@ async fn main() -> anyhow::Result<()> { "rockslide-podman".to_owned(), cfg.rockslide.master_key.as_secret_string(), ); - let hooks = PodmanHook::new( + let hooks = ContainerOrchestrator::new( &cfg.containers.podman_path, reverse_proxy.clone(), local_addr, From 6b77bd30ece955528f4fdfc060a8a93bb7bf2af7 Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Mon, 1 Jan 2024 02:11:22 +0100 Subject: [PATCH 05/23] Move `ContainerOrchestrator` to its own module --- src/config.rs | 2 +- src/container_orchestrator.rs | 211 ++++++++++++++++++++++++++++++++ src/main.rs | 221 ++-------------------------------- src/podman.rs | 5 + 4 files changed, 225 insertions(+), 214 deletions(-) create mode 100644 src/container_orchestrator.rs diff --git a/src/config.rs b/src/config.rs index dc10fef..b72d16d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -6,7 +6,7 @@ use sec::Secret; use serde::Deserialize; use crate::{ - podman_is_remote, + podman::podman_is_remote, registry::{AuthProvider, UnverifiedCredentials}, }; diff --git a/src/container_orchestrator.rs b/src/container_orchestrator.rs new file mode 100644 index 0000000..2043a2a --- /dev/null +++ b/src/container_orchestrator.rs @@ -0,0 +1,211 @@ +use std::net::Ipv4Addr; +use std::str::FromStr; +use std::{net::SocketAddr, path::Path, sync::Arc}; + +use crate::podman::podman_is_remote; +use crate::{ + podman::Podman, + registry::{storage::ImageLocation, ManifestReference, Reference, RegistryHooks}, + reverse_proxy::{PublishedContainer, ReverseProxy}, +}; + +use axum::async_trait; +use sec::Secret; +use serde::{Deserialize, Deserializer}; +use tracing::{debug, error, info}; + +macro_rules! try_quiet { + ($ex:expr, $msg:expr) => { + match $ex { + Ok(v) => v, + Err(err) => { + error!(%err, $msg); + return; + } + } + }; +} + +pub(crate) struct ContainerOrchestrator { + podman: Podman, + reverse_proxy: Arc, + local_addr: SocketAddr, + registry_credentials: (String, Secret), +} + +impl ContainerOrchestrator { + pub(crate) fn new>( + podman_path: P, + reverse_proxy: Arc, + local_addr: SocketAddr, + registry_credentials: (String, Secret), + ) -> Self { + let podman = Podman::new(podman_path, podman_is_remote()); + Self { + podman, + reverse_proxy, + local_addr, + registry_credentials, + } + } + + async fn fetch_running_containers(&self) -> anyhow::Result> { + debug!("refreshing running containers"); + + let value = self.podman.ps(false).await?; + let rv: Vec = serde_json::from_value(value)?; + + debug!(?rv, "fetched containers"); + + Ok(rv) + } + + pub(crate) async fn updated_published_set(&self) { + let running: Vec<_> = try_quiet!( + self.fetch_running_containers().await, + "could not fetch running containers" + ) + .iter() + .filter_map(ContainerJson::published_container) + .collect(); + + info!(?running, "updating running container set"); + self.reverse_proxy + .update_containers(running.into_iter()) + .await; + } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "PascalCase")] +#[allow(dead_code)] +struct ContainerJson { + id: String, + names: Vec, + #[serde(deserialize_with = "nullable_array")] + ports: Vec, +} + +impl ContainerJson { + fn image_location(&self) -> Option { + const PREFIX: &str = "rockslide-"; + + for name in &self.names { + if let Some(subname) = name.strip_prefix(PREFIX) { + if let Some((left, right)) = subname.split_once('-') { + return Some(ImageLocation::new(left.to_owned(), right.to_owned())); + } + } + } + + None + } + + fn active_published_port(&self) -> Option<&PortMapping> { + self.ports.get(0) + } + + fn published_container(&self) -> Option { + let image_location = self.image_location()?; + let port_mapping = self.active_published_port()?; + + Some(PublishedContainer::new( + port_mapping.get_host_listening_addr()?, + image_location, + )) + } +} + +#[async_trait] +impl RegistryHooks for ContainerOrchestrator { + async fn on_manifest_uploaded(&self, manifest_reference: &ManifestReference) { + // TODO: Make configurable? + let production_tag = "prod"; + + if matches!(manifest_reference.reference(), Reference::Tag(tag) if tag == production_tag) { + let location = manifest_reference.location(); + let name = format!("rockslide-{}-{}", location.repository(), location.image()); + + info!(%name, "removing (potentially nonexistant) container"); + try_quiet!( + self.podman.rm(&name, true).await, + "failed to remove container" + ); + + let image_url = format!( + "{}/{}/{}:{}", + self.local_addr, + location.repository(), + location.image(), + production_tag + ); + + info!(%name, "loggging in"); + try_quiet!( + self.podman + .login( + &self.registry_credentials.0, + self.registry_credentials.1.as_str(), + self.local_addr.to_string().as_ref(), + false + ) + .await, + "failed to login to local registry" + ); + + // We always pull the container to ensure we have the latest version. + info!(%name, "pulling container"); + try_quiet!( + self.podman.pull(&image_url).await, + "failed to pull container" + ); + + info!(%name, "starting container"); + try_quiet!( + self.podman + .run(&image_url) + .rm() + .rmi() + .name(name) + .tls_verify(false) + .publish("127.0.0.1::8000") + .env("PORT", "8000") + .execute() + .await, + "failed to launch container" + ); + + info!(?manifest_reference, "new production image uploaded"); + + self.updated_published_set().await; + } + } +} + +fn nullable_array<'de, D, T>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de>, +{ + let opt: Option> = Deserialize::deserialize(deserializer)?; + + Ok(opt.unwrap_or_default()) +} + +#[derive(Debug, Deserialize)] +#[allow(dead_code)] +struct PortMapping { + host_ip: String, + container_port: u16, + host_port: u16, + range: u16, + protocol: String, +} + +impl PortMapping { + fn get_host_listening_addr(&self) -> Option { + let ip = Ipv4Addr::from_str(&self.host_ip).ok()?; + + Some((ip, self.host_port).into()) + } +} diff --git a/src/main.rs b/src/main.rs index 7587068..6230e0b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,230 +1,25 @@ mod config; -mod podman; +mod container_orchestrator; +pub(crate) mod podman; pub(crate) mod registry; mod reverse_proxy; use std::{ env, fs, - net::{IpAddr, Ipv4Addr, SocketAddr, ToSocketAddrs}, - path::Path, - str::FromStr, - sync::Arc, + net::{IpAddr, SocketAddr, ToSocketAddrs}, }; use anyhow::Context; -use axum::{async_trait, Router}; +use axum::Router; use config::Config; use gethostname::gethostname; -use podman::Podman; -use registry::{ - storage::ImageLocation, ContainerRegistry, ManifestReference, Reference, RegistryHooks, -}; -use reverse_proxy::{PublishedContainer, ReverseProxy}; -use sec::Secret; -use serde::{Deserialize, Deserializer}; +use registry::ContainerRegistry; +use reverse_proxy::ReverseProxy; use tower_http::trace::TraceLayer; -use tracing::{debug, error, info}; +use tracing::{debug, info}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; -macro_rules! try_quiet { - ($ex:expr, $msg:expr) => { - match $ex { - Ok(v) => v, - Err(err) => { - error!(%err, $msg); - return; - } - } - }; -} - -struct ContainerOrchestrator { - podman: Podman, - reverse_proxy: Arc, - local_addr: SocketAddr, - registry_credentials: (String, Secret), -} - -impl ContainerOrchestrator { - fn new>( - podman_path: P, - reverse_proxy: Arc, - local_addr: SocketAddr, - registry_credentials: (String, Secret), - ) -> Self { - let podman = Podman::new(podman_path, podman_is_remote()); - Self { - podman, - reverse_proxy, - local_addr, - registry_credentials, - } - } - - async fn fetch_running_containers(&self) -> anyhow::Result> { - debug!("refreshing running containers"); - - let value = self.podman.ps(false).await?; - let rv: Vec = serde_json::from_value(value)?; - - debug!(?rv, "fetched containers"); - - Ok(rv) - } - - async fn updated_published_set(&self) { - let running: Vec<_> = try_quiet!( - self.fetch_running_containers().await, - "could not fetch running containers" - ) - .iter() - .filter_map(ContainerJson::published_container) - .collect(); - - info!(?running, "updating running container set"); - self.reverse_proxy - .update_containers(running.into_iter()) - .await; - } -} - -pub(crate) fn podman_is_remote() -> bool { - env::var("PODMAN_IS_REMOTE").unwrap_or_default() == "true" -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "PascalCase")] -#[allow(dead_code)] -struct ContainerJson { - id: String, - names: Vec, - #[serde(deserialize_with = "nullable_array")] - ports: Vec, -} - -impl ContainerJson { - fn image_location(&self) -> Option { - const PREFIX: &str = "rockslide-"; - - for name in &self.names { - if let Some(subname) = name.strip_prefix(PREFIX) { - if let Some((left, right)) = subname.split_once('-') { - return Some(ImageLocation::new(left.to_owned(), right.to_owned())); - } - } - } - - None - } - - fn active_published_port(&self) -> Option<&PortMapping> { - self.ports.get(0) - } - - fn published_container(&self) -> Option { - let image_location = self.image_location()?; - let port_mapping = self.active_published_port()?; - - Some(PublishedContainer::new( - port_mapping.get_host_listening_addr()?, - image_location, - )) - } -} - -fn nullable_array<'de, D, T>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, - T: Deserialize<'de>, -{ - let opt: Option> = Deserialize::deserialize(deserializer)?; - - Ok(opt.unwrap_or_default()) -} - -#[derive(Debug, Deserialize)] -#[allow(dead_code)] -struct PortMapping { - host_ip: String, - container_port: u16, - host_port: u16, - range: u16, - protocol: String, -} - -impl PortMapping { - fn get_host_listening_addr(&self) -> Option { - let ip = Ipv4Addr::from_str(&self.host_ip).ok()?; - - Some((ip, self.host_port).into()) - } -} - -#[async_trait] -impl RegistryHooks for ContainerOrchestrator { - async fn on_manifest_uploaded(&self, manifest_reference: &ManifestReference) { - // TODO: Make configurable? - let production_tag = "prod"; - - if matches!(manifest_reference.reference(), Reference::Tag(tag) if tag == production_tag) { - let location = manifest_reference.location(); - let name = format!("rockslide-{}-{}", location.repository(), location.image()); - - info!(%name, "removing (potentially nonexistant) container"); - try_quiet!( - self.podman.rm(&name, true).await, - "failed to remove container" - ); - - let image_url = format!( - "{}/{}/{}:{}", - self.local_addr, - location.repository(), - location.image(), - production_tag - ); - - info!(%name, "loggging in"); - try_quiet!( - self.podman - .login( - &self.registry_credentials.0, - self.registry_credentials.1.as_str(), - self.local_addr.to_string().as_ref(), - false - ) - .await, - "failed to login to local registry" - ); - - // We always pull the container to ensure we have the latest version. - info!(%name, "pulling container"); - try_quiet!( - self.podman.pull(&image_url).await, - "failed to pull container" - ); - - info!(%name, "starting container"); - try_quiet!( - self.podman - .run(&image_url) - .rm() - .rmi() - .name(name) - .tls_verify(false) - .publish("127.0.0.1::8000") - .env("PORT", "8000") - .execute() - .await, - "failed to launch container" - ); - - info!(?manifest_reference, "new production image uploaded"); - - self.updated_published_set().await; - } - } -} +use crate::{container_orchestrator::ContainerOrchestrator, podman::podman_is_remote}; fn load_config() -> anyhow::Result { match env::args().len() { diff --git a/src/podman.rs b/src/podman.rs index 0da6b06..ffeb250 100644 --- a/src/podman.rs +++ b/src/podman.rs @@ -1,4 +1,5 @@ use std::{ + env, fmt::Display, io::{self, Seek, SeekFrom, Write}, path::{Path, PathBuf}, @@ -286,3 +287,7 @@ async fn fetch_json(cmd: Command) -> Result { Ok(parsed) } + +pub(crate) fn podman_is_remote() -> bool { + env::var("PODMAN_IS_REMOTE").unwrap_or_default() == "true" +} From 762240979eae582590fa908b98b40fb5fe18f826 Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Mon, 1 Jan 2024 02:14:17 +0100 Subject: [PATCH 06/23] Fix clippy errors --- src/podman.rs | 2 +- src/reverse_proxy.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/podman.rs b/src/podman.rs index ffeb250..a54bd4c 100644 --- a/src/podman.rs +++ b/src/podman.rs @@ -56,7 +56,7 @@ impl Podman { let mut pw_file = tempfile()?; - pw_file.write(password.reveal().as_bytes())?; + pw_file.write_all(password.reveal().as_bytes())?; pw_file.seek(SeekFrom::Start(0))?; cmd.stdin(Stdio::from(pw_file)); diff --git a/src/reverse_proxy.rs b/src/reverse_proxy.rs index 8415e3f..614b3fa 100644 --- a/src/reverse_proxy.rs +++ b/src/reverse_proxy.rs @@ -79,7 +79,7 @@ impl RoutingTable { let mut domain_maps = HashMap::new(); for container in containers { - if let Some(domain) = Domain::new(&container.image_location.repository()) { + if let Some(domain) = Domain::new(container.image_location.repository()) { domain_maps.insert(domain, container.clone()); } @@ -295,7 +295,7 @@ async fn route_request( /// HTTP/1.1 hop-by-hop headers mod hop_by_hop { use reqwest::header::HeaderName; - pub(super) const HOP_BY_HOP: [HeaderName; 8] = [ + pub(super) static HOP_BY_HOP: [HeaderName; 8] = [ HeaderName::from_static("keep-alive"), HeaderName::from_static("transfer-encoding"), HeaderName::from_static("te"), From 3400d94edad80b3ba0265021f6698874f82d2dc3 Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Mon, 1 Jan 2024 02:16:14 +0100 Subject: [PATCH 07/23] Fix name of `orchestrator` in various other types --- src/main.rs | 11 +++++++---- src/registry.rs | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main.rs b/src/main.rs index 6230e0b..cd76499 100644 --- a/src/main.rs +++ b/src/main.rs @@ -81,16 +81,19 @@ async fn main() -> anyhow::Result<()> { "rockslide-podman".to_owned(), cfg.rockslide.master_key.as_secret_string(), ); - let hooks = ContainerOrchestrator::new( + let orchestrator = ContainerOrchestrator::new( &cfg.containers.podman_path, reverse_proxy.clone(), local_addr, credentials, ); - hooks.updated_published_set().await; + orchestrator.updated_published_set().await; - let registry = - ContainerRegistry::new(&cfg.registry.storage_path, hooks, cfg.rockslide.master_key)?; + let registry = ContainerRegistry::new( + &cfg.registry.storage_path, + orchestrator, + cfg.rockslide.master_key, + )?; let app = Router::new() .merge(registry.make_router()) diff --git a/src/registry.rs b/src/registry.rs index 6a4a4fd..0c3f8fa 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -104,14 +104,14 @@ impl ContainerRegistry { A: AuthProvider + 'static, >( storage_path: P, - hooks: T, + orchestrator: T, auth_provider: A, ) -> Result, FilesystemStorageError> { Ok(Arc::new(ContainerRegistry { realm: "ContainerRegistry".to_string(), auth_provider: Box::new(auth_provider), storage: Box::new(FilesystemStorage::new(storage_path)?), - hooks: Box::new(hooks), + hooks: Box::new(orchestrator), })) } From 5a63e9d760274ab0cc0844a70adca5a4de491b3c Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Mon, 1 Jan 2024 21:59:21 +0100 Subject: [PATCH 08/23] Refactor code handling running container set --- src/container_orchestrator.rs | 117 ++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 56 deletions(-) diff --git a/src/container_orchestrator.rs b/src/container_orchestrator.rs index 2043a2a..65a57b9 100644 --- a/src/container_orchestrator.rs +++ b/src/container_orchestrator.rs @@ -49,76 +49,33 @@ impl ContainerOrchestrator { } } - async fn fetch_running_containers(&self) -> anyhow::Result> { + async fn fetch_managed_containers(&self, all: bool) -> anyhow::Result> { debug!("refreshing running containers"); - let value = self.podman.ps(false).await?; - let rv: Vec = serde_json::from_value(value)?; + let value = self.podman.ps(all).await?; + let all_containers: Vec = serde_json::from_value(value)?; - debug!(?rv, "fetched containers"); + debug!(?all_containers, "fetched containers"); - Ok(rv) + Ok(all_containers + .iter() + .filter_map(ContainerJson::published_container) + .collect()) } pub(crate) async fn updated_published_set(&self) { let running: Vec<_> = try_quiet!( - self.fetch_running_containers().await, + self.fetch_managed_containers(false).await, "could not fetch running containers" - ) - .iter() - .filter_map(ContainerJson::published_container) - .collect(); + ); info!(?running, "updating running container set"); self.reverse_proxy .update_containers(running.into_iter()) .await; } -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "PascalCase")] -#[allow(dead_code)] -struct ContainerJson { - id: String, - names: Vec, - #[serde(deserialize_with = "nullable_array")] - ports: Vec, -} - -impl ContainerJson { - fn image_location(&self) -> Option { - const PREFIX: &str = "rockslide-"; - - for name in &self.names { - if let Some(subname) = name.strip_prefix(PREFIX) { - if let Some((left, right)) = subname.split_once('-') { - return Some(ImageLocation::new(left.to_owned(), right.to_owned())); - } - } - } - None - } - - fn active_published_port(&self) -> Option<&PortMapping> { - self.ports.get(0) - } - - fn published_container(&self) -> Option { - let image_location = self.image_location()?; - let port_mapping = self.active_published_port()?; - - Some(PublishedContainer::new( - port_mapping.get_host_listening_addr()?, - image_location, - )) - } -} - -#[async_trait] -impl RegistryHooks for ContainerOrchestrator { - async fn on_manifest_uploaded(&self, manifest_reference: &ManifestReference) { + async fn update_container_running_state(&self, manifest_reference: &ManifestReference) { // TODO: Make configurable? let production_tag = "prod"; @@ -175,10 +132,58 @@ impl RegistryHooks for ContainerOrchestrator { "failed to launch container" ); - info!(?manifest_reference, "new production image uploaded"); + info!(?manifest_reference, "new production image running"); + } + } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "PascalCase")] +#[allow(dead_code)] +struct ContainerJson { + id: String, + names: Vec, + #[serde(deserialize_with = "nullable_array")] + ports: Vec, +} - self.updated_published_set().await; +impl ContainerJson { + fn image_location(&self) -> Option { + const PREFIX: &str = "rockslide-"; + + for name in &self.names { + if let Some(subname) = name.strip_prefix(PREFIX) { + if let Some((left, right)) = subname.split_once('-') { + return Some(ImageLocation::new(left.to_owned(), right.to_owned())); + } + } } + + None + } + + fn active_published_port(&self) -> Option<&PortMapping> { + self.ports.get(0) + } + + fn published_container(&self) -> Option { + let image_location = self.image_location()?; + let port_mapping = self.active_published_port()?; + + Some(PublishedContainer::new( + port_mapping.get_host_listening_addr()?, + image_location, + )) + } +} + +#[async_trait] +impl RegistryHooks for ContainerOrchestrator { + async fn on_manifest_uploaded(&self, manifest_reference: &ManifestReference) { + self.update_container_running_state(manifest_reference) + .await; + + self.updated_published_set().await; } } From 5d118f4caf8a3031a8ae6c984b009a28674686c4 Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Mon, 1 Jan 2024 22:17:14 +0100 Subject: [PATCH 09/23] Restart all containers at startup. Closes #9. --- src/container_orchestrator.rs | 33 ++++++++++++++++++++++++++++----- src/main.rs | 2 ++ src/registry/storage.rs | 4 ++-- src/reverse_proxy.rs | 17 +++++++++++------ 4 files changed, 43 insertions(+), 13 deletions(-) diff --git a/src/container_orchestrator.rs b/src/container_orchestrator.rs index 65a57b9..8e1b5d4 100644 --- a/src/container_orchestrator.rs +++ b/src/container_orchestrator.rs @@ -75,7 +75,7 @@ impl ContainerOrchestrator { .await; } - async fn update_container_running_state(&self, manifest_reference: &ManifestReference) { + async fn synchronize_container_state(&self, manifest_reference: &ManifestReference) { // TODO: Make configurable? let production_tag = "prod"; @@ -135,6 +135,15 @@ impl ContainerOrchestrator { info!(?manifest_reference, "new production image running"); } } + + pub(crate) async fn synchronize_all(&self) -> anyhow::Result<()> { + for container in self.fetch_managed_containers(true).await? { + self.synchronize_container_state(container.manifest_reference()) + .await; + } + + Ok(()) + } } #[derive(Debug, Deserialize)] @@ -142,6 +151,7 @@ impl ContainerOrchestrator { #[allow(dead_code)] struct ContainerJson { id: String, + image: String, names: Vec, #[serde(deserialize_with = "nullable_array")] ports: Vec, @@ -162,17 +172,31 @@ impl ContainerJson { None } + fn image_tag(&self) -> Option { + let idx = self.image.rfind(':')?; + + // TODO: Handle Reference::Digest here. + Some(Reference::Tag(self.image[idx..].to_owned())) + } + + fn manifest_reference(&self) -> Option { + Some(ManifestReference::new( + self.image_location()?, + self.image_tag()?, + )) + } + fn active_published_port(&self) -> Option<&PortMapping> { self.ports.get(0) } fn published_container(&self) -> Option { - let image_location = self.image_location()?; + let manifest_reference = self.manifest_reference()?; let port_mapping = self.active_published_port()?; Some(PublishedContainer::new( port_mapping.get_host_listening_addr()?, - image_location, + manifest_reference, )) } } @@ -180,8 +204,7 @@ impl ContainerJson { #[async_trait] impl RegistryHooks for ContainerOrchestrator { async fn on_manifest_uploaded(&self, manifest_reference: &ManifestReference) { - self.update_container_running_state(manifest_reference) - .await; + self.synchronize_container_state(manifest_reference).await; self.updated_published_set().await; } diff --git a/src/main.rs b/src/main.rs index cd76499..dcf88e6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -87,6 +87,8 @@ async fn main() -> anyhow::Result<()> { local_addr, credentials, ); + // TODO: Probably should not fail if synchronization fails. + orchestrator.synchronize_all().await?; orchestrator.updated_published_set().await; let registry = ContainerRegistry::new( diff --git a/src/registry/storage.rs b/src/registry/storage.rs index c1b125a..2847d8b 100644 --- a/src/registry/storage.rs +++ b/src/registry/storage.rs @@ -55,7 +55,7 @@ pub(crate) struct ImageLocation { image: String, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub(crate) struct ManifestReference { #[serde(flatten)] location: ImageLocation, @@ -97,7 +97,7 @@ impl ImageLocation { } } -#[derive(Debug)] +#[derive(Clone, Debug)] pub(crate) enum Reference { Tag(String), Digest(Digest), diff --git a/src/reverse_proxy.rs b/src/reverse_proxy.rs index 614b3fa..c1f2218 100644 --- a/src/reverse_proxy.rs +++ b/src/reverse_proxy.rs @@ -22,7 +22,7 @@ use itertools::Itertools; use tokio::sync::RwLock; use tracing::{trace, warn}; -use crate::registry::storage::ImageLocation; +use crate::registry::{storage::ImageLocation, ManifestReference}; pub(crate) struct ReverseProxy { client: reqwest::Client, @@ -32,7 +32,7 @@ pub(crate) struct ReverseProxy { #[derive(Clone, Debug)] pub(crate) struct PublishedContainer { host_addr: SocketAddr, - image_location: ImageLocation, + manifest_reference: ManifestReference, } #[derive(Debug, Default)] @@ -79,11 +79,12 @@ impl RoutingTable { let mut domain_maps = HashMap::new(); for container in containers { - if let Some(domain) = Domain::new(container.image_location.repository()) { + if let Some(domain) = Domain::new(container.manifest_reference.location().repository()) + { domain_maps.insert(domain, container.clone()); } - path_maps.insert(container.image_location.clone(), container); + path_maps.insert(container.manifest_reference.location().clone(), container); } Self { @@ -199,12 +200,16 @@ impl IntoResponse for AppError { } impl PublishedContainer { - pub(crate) fn new(host_addr: SocketAddr, image_location: ImageLocation) -> Self { + pub(crate) fn new(host_addr: SocketAddr, manifest_reference: ManifestReference) -> Self { Self { host_addr, - image_location, + manifest_reference, } } + + pub(crate) fn manifest_reference(&self) -> &ManifestReference { + &self.manifest_reference + } } impl ReverseProxy { From f45fed72a29f6b9b0fb370e3c368039640d7e7fa Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Mon, 1 Jan 2024 22:22:14 +0100 Subject: [PATCH 10/23] Add a runtime configuration --- src/reverse_proxy.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/reverse_proxy.rs b/src/reverse_proxy.rs index c1f2218..8736af1 100644 --- a/src/reverse_proxy.rs +++ b/src/reverse_proxy.rs @@ -19,6 +19,7 @@ use axum::{ Router, }; use itertools::Itertools; +use serde::{Deserialize, Serialize}; use tokio::sync::RwLock; use tracing::{trace, warn}; @@ -33,6 +34,13 @@ pub(crate) struct ReverseProxy { pub(crate) struct PublishedContainer { host_addr: SocketAddr, manifest_reference: ManifestReference, + config: RuntimeConfig, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +pub(crate) struct RuntimeConfig { + #[serde(default)] + http_access: Option>, } #[derive(Debug, Default)] @@ -204,6 +212,7 @@ impl PublishedContainer { Self { host_addr, manifest_reference, + config: Default::default(), } } From 70fb91271cfdac2009835ff7d4391959b1a87f73 Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Mon, 1 Jan 2024 22:25:37 +0100 Subject: [PATCH 11/23] Moved `PublishedContainer` and associated configuration type to `container_orchestrator` --- src/container_orchestrator.rs | 35 +++++++++++++++++++++++++----- src/reverse_proxy.rs | 40 ++++++----------------------------- 2 files changed, 36 insertions(+), 39 deletions(-) diff --git a/src/container_orchestrator.rs b/src/container_orchestrator.rs index 8e1b5d4..7831c2d 100644 --- a/src/container_orchestrator.rs +++ b/src/container_orchestrator.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::net::Ipv4Addr; use std::str::FromStr; use std::{net::SocketAddr, path::Path, sync::Arc}; @@ -6,12 +7,12 @@ use crate::podman::podman_is_remote; use crate::{ podman::Podman, registry::{storage::ImageLocation, ManifestReference, Reference, RegistryHooks}, - reverse_proxy::{PublishedContainer, ReverseProxy}, + reverse_proxy::ReverseProxy, }; use axum::async_trait; use sec::Secret; -use serde::{Deserialize, Deserializer}; +use serde::{Deserialize, Deserializer, Serialize}; use tracing::{debug, error, info}; macro_rules! try_quiet { @@ -33,6 +34,29 @@ pub(crate) struct ContainerOrchestrator { registry_credentials: (String, Secret), } +#[derive(Clone, Debug)] +pub(crate) struct PublishedContainer { + host_addr: SocketAddr, + manifest_reference: ManifestReference, + config: RuntimeConfig, +} + +impl PublishedContainer { + pub(crate) fn manifest_reference(&self) -> &ManifestReference { + &self.manifest_reference + } + + pub(crate) fn host_addr(&self) -> SocketAddr { + self.host_addr + } +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +pub(crate) struct RuntimeConfig { + #[serde(default)] + http_access: Option>, +} + impl ContainerOrchestrator { pub(crate) fn new>( podman_path: P, @@ -194,10 +218,11 @@ impl ContainerJson { let manifest_reference = self.manifest_reference()?; let port_mapping = self.active_published_port()?; - Some(PublishedContainer::new( - port_mapping.get_host_listening_addr()?, + Some(PublishedContainer { + host_addr: port_mapping.get_host_listening_addr()?, manifest_reference, - )) + config: Default::default(), // TODO + }) } } diff --git a/src/reverse_proxy.rs b/src/reverse_proxy.rs index 8736af1..8bce123 100644 --- a/src/reverse_proxy.rs +++ b/src/reverse_proxy.rs @@ -2,7 +2,6 @@ use std::{ collections::HashMap, fmt::{self, Display}, mem, - net::SocketAddr, str::{self, FromStr}, sync::Arc, }; @@ -19,30 +18,16 @@ use axum::{ Router, }; use itertools::Itertools; -use serde::{Deserialize, Serialize}; use tokio::sync::RwLock; use tracing::{trace, warn}; -use crate::registry::{storage::ImageLocation, ManifestReference}; +use crate::{container_orchestrator::PublishedContainer, registry::storage::ImageLocation}; pub(crate) struct ReverseProxy { client: reqwest::Client, routing_table: RwLock, } -#[derive(Clone, Debug)] -pub(crate) struct PublishedContainer { - host_addr: SocketAddr, - manifest_reference: ManifestReference, - config: RuntimeConfig, -} - -#[derive(Clone, Debug, Default, Deserialize, Serialize)] -pub(crate) struct RuntimeConfig { - #[serde(default)] - http_access: Option>, -} - #[derive(Debug, Default)] pub(crate) struct RoutingTable { path_maps: HashMap, @@ -87,12 +72,13 @@ impl RoutingTable { let mut domain_maps = HashMap::new(); for container in containers { - if let Some(domain) = Domain::new(container.manifest_reference.location().repository()) + if let Some(domain) = + Domain::new(container.manifest_reference().location().repository()) { domain_maps.insert(domain, container.clone()); } - path_maps.insert(container.manifest_reference.location().clone(), container); + path_maps.insert(container.manifest_reference().location().clone(), container); } Self { @@ -127,7 +113,7 @@ impl RoutingTable { let mut parts = req_uri.clone().into_parts(); parts.scheme = Some(Scheme::HTTP); parts.authority = Some( - Authority::from_str(&pc.host_addr.to_string()) + Authority::from_str(&pc.host_addr().to_string()) .expect("SocketAddr should never fail to convert to Authority"), ); return Some(Uri::from_parts(parts).expect("should not have invalidated Uri")); @@ -137,7 +123,7 @@ impl RoutingTable { // Reconstruct image location from path segments, keeping remainder intact. if let Some((image_location, remainder)) = split_path_base_url(req_uri) { if let Some(pc) = self.get_path_route(&image_location) { - let container_addr = pc.host_addr; + let container_addr = pc.host_addr(); let mut dest_path_and_query = remainder; @@ -207,20 +193,6 @@ impl IntoResponse for AppError { } } -impl PublishedContainer { - pub(crate) fn new(host_addr: SocketAddr, manifest_reference: ManifestReference) -> Self { - Self { - host_addr, - manifest_reference, - config: Default::default(), - } - } - - pub(crate) fn manifest_reference(&self) -> &ManifestReference { - &self.manifest_reference - } -} - impl ReverseProxy { pub(crate) fn new() -> Arc { Arc::new(ReverseProxy { From 192005fe26fd4cd23deafc1d6c5931ae8e2580d1 Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Fri, 5 Jan 2024 01:22:51 +0100 Subject: [PATCH 12/23] Support config loading from disk --- src/container_orchestrator.rs | 99 ++++++++++++++++++++++++++++------- src/main.rs | 5 +- src/registry/storage.rs | 9 ++++ 3 files changed, 94 insertions(+), 19 deletions(-) diff --git a/src/container_orchestrator.rs b/src/container_orchestrator.rs index 7831c2d..e9f156c 100644 --- a/src/container_orchestrator.rs +++ b/src/container_orchestrator.rs @@ -1,5 +1,7 @@ use std::collections::HashMap; +use std::fs; use std::net::Ipv4Addr; +use std::path::PathBuf; use std::str::FromStr; use std::{net::SocketAddr, path::Path, sync::Arc}; @@ -10,6 +12,7 @@ use crate::{ reverse_proxy::ReverseProxy, }; +use anyhow::Context; use axum::async_trait; use sec::Secret; use serde::{Deserialize, Deserializer, Serialize}; @@ -32,6 +35,7 @@ pub(crate) struct ContainerOrchestrator { reverse_proxy: Arc, local_addr: SocketAddr, registry_credentials: (String, Secret), + configs_dir: PathBuf, } #[derive(Clone, Debug)] @@ -58,19 +62,58 @@ pub(crate) struct RuntimeConfig { } impl ContainerOrchestrator { - pub(crate) fn new>( + pub(crate) fn new, Q: AsRef>( podman_path: P, reverse_proxy: Arc, local_addr: SocketAddr, registry_credentials: (String, Secret), - ) -> Self { + runtime_dir: Q, + ) -> anyhow::Result { let podman = Podman::new(podman_path, podman_is_remote()); - Self { + + let configs_dir = runtime_dir + .as_ref() + .canonicalize() + .context("could not canonicalize runtime config dir")? + .join("configs"); + + if !configs_dir.exists() { + fs::create_dir(&configs_dir).context("could not create config dir")?; + } + + Ok(Self { podman, reverse_proxy, local_addr, registry_credentials, + configs_dir, + }) + } + + fn config_path(&self, manifest_reference: &ManifestReference) -> PathBuf { + let location = manifest_reference.location(); + + self.configs_dir + .join(location.repository()) + .join(location.image()) + .join(manifest_reference.reference().to_string()) + } + + pub(crate) async fn load_config( + &self, + manifest_reference: &ManifestReference, + ) -> anyhow::Result { + let config_path = self.config_path(manifest_reference); + + if !config_path.exists() { + return Ok(Default::default()); } + + let raw = tokio::fs::read_to_string(config_path) + .await + .context("could not read config")?; + + toml::from_str(&raw).context("could not parse configuration") } async fn fetch_managed_containers(&self, all: bool) -> anyhow::Result> { @@ -81,10 +124,41 @@ impl ContainerOrchestrator { debug!(?all_containers, "fetched containers"); - Ok(all_containers - .iter() - .filter_map(ContainerJson::published_container) - .collect()) + let mut rv = Vec::new(); + for container in all_containers { + // TODO: Just log error instead of returning. + if let Some(pc) = self.load_managed_container(container).await? { + rv.push(pc); + } + } + Ok(rv) + } + + async fn load_managed_container( + &self, + container_json: ContainerJson, + ) -> anyhow::Result> { + let manifest_reference = if let Some(val) = container_json.manifest_reference() { + val + } else { + return Ok(None); + }; + + let port_mapping = if let Some(val) = container_json.active_published_port() { + val + } else { + return Ok(None); + }; + + let config = self.load_config(&manifest_reference).await?; + + Ok(Some(PublishedContainer { + host_addr: port_mapping + .get_host_listening_addr() + .context("could not get host listening address")?, + manifest_reference, + config, + })) } pub(crate) async fn updated_published_set(&self) { @@ -213,17 +287,6 @@ impl ContainerJson { fn active_published_port(&self) -> Option<&PortMapping> { self.ports.get(0) } - - fn published_container(&self) -> Option { - let manifest_reference = self.manifest_reference()?; - let port_mapping = self.active_published_port()?; - - Some(PublishedContainer { - host_addr: port_mapping.get_host_listening_addr()?, - manifest_reference, - config: Default::default(), // TODO - }) - } } #[async_trait] diff --git a/src/main.rs b/src/main.rs index dcf88e6..a1f1313 100644 --- a/src/main.rs +++ b/src/main.rs @@ -73,6 +73,8 @@ async fn main() -> anyhow::Result<()> { }; let local_addr = SocketAddr::from((local_ip, cfg.reverse_proxy.http_bind.port())); + // TODO: Fix (see #34). + let local_addr = SocketAddr::from(([127, 0, 0, 1], cfg.reverse_proxy.http_bind.port())); info!(%local_addr, "guessing local registry address"); let reverse_proxy = ReverseProxy::new(); @@ -86,7 +88,8 @@ async fn main() -> anyhow::Result<()> { reverse_proxy.clone(), local_addr, credentials, - ); + &cfg.registry.storage_path, + )?; // TODO: Probably should not fail if synchronization fails. orchestrator.synchronize_all().await?; orchestrator.updated_published_set().await; diff --git a/src/registry/storage.rs b/src/registry/storage.rs index 2847d8b..8d9a1ed 100644 --- a/src/registry/storage.rs +++ b/src/registry/storage.rs @@ -150,6 +150,15 @@ impl Reference { } } +impl Display for Reference { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Reference::Tag(tag) => Display::fmt(tag, f), + Reference::Digest(digest) => Display::fmt(digest, f), + } + } +} + #[derive(Debug, Error)] pub(crate) enum Error { #[error("given upload does not exist")] From 127add9ce24265816200e12cc9af03a094e88397 Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Fri, 5 Jan 2024 02:11:31 +0100 Subject: [PATCH 13/23] Support config loading via get --- src/container_orchestrator.rs | 5 +- src/main.rs | 7 ++- src/reverse_proxy.rs | 92 +++++++++++++++++++++++++++++++---- 3 files changed, 91 insertions(+), 13 deletions(-) diff --git a/src/container_orchestrator.rs b/src/container_orchestrator.rs index e9f156c..98e4f73 100644 --- a/src/container_orchestrator.rs +++ b/src/container_orchestrator.rs @@ -30,6 +30,7 @@ macro_rules! try_quiet { }; } +#[derive(Debug)] pub(crate) struct ContainerOrchestrator { podman: Podman, reverse_proxy: Arc, @@ -58,7 +59,7 @@ impl PublishedContainer { #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub(crate) struct RuntimeConfig { #[serde(default)] - http_access: Option>, + http_access: HashMap, } impl ContainerOrchestrator { @@ -290,7 +291,7 @@ impl ContainerJson { } #[async_trait] -impl RegistryHooks for ContainerOrchestrator { +impl RegistryHooks for Arc { async fn on_manifest_uploaded(&self, manifest_reference: &ManifestReference) { self.synchronize_container_state(manifest_reference).await; diff --git a/src/main.rs b/src/main.rs index a1f1313..40a175c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,6 +7,7 @@ mod reverse_proxy; use std::{ env, fs, net::{IpAddr, SocketAddr, ToSocketAddrs}, + sync::Arc, }; use anyhow::Context; @@ -83,13 +84,15 @@ async fn main() -> anyhow::Result<()> { "rockslide-podman".to_owned(), cfg.rockslide.master_key.as_secret_string(), ); - let orchestrator = ContainerOrchestrator::new( + let orchestrator = Arc::new(ContainerOrchestrator::new( &cfg.containers.podman_path, reverse_proxy.clone(), local_addr, credentials, &cfg.registry.storage_path, - )?; + )?); + reverse_proxy.set_orchestrator(orchestrator.clone()); + // TODO: Probably should not fail if synchronization fails. orchestrator.synchronize_all().await?; orchestrator.updated_published_set().await; diff --git a/src/reverse_proxy.rs b/src/reverse_proxy.rs index 8bce123..7ee188a 100644 --- a/src/reverse_proxy.rs +++ b/src/reverse_proxy.rs @@ -3,14 +3,14 @@ use std::{ fmt::{self, Display}, mem, str::{self, FromStr}, - sync::Arc, + sync::{Arc, OnceLock}, }; use axum::{ body::Body, extract::{Request, State}, http::{ - header::HOST, + header::{CONTENT_TYPE, HOST}, uri::{Authority, Parts, PathAndQuery, Scheme}, StatusCode, Uri, }, @@ -21,11 +21,16 @@ use itertools::Itertools; use tokio::sync::RwLock; use tracing::{trace, warn}; -use crate::{container_orchestrator::PublishedContainer, registry::storage::ImageLocation}; +use crate::{ + container_orchestrator::{ContainerOrchestrator, PublishedContainer}, + registry::{storage::ImageLocation, ManifestReference, Reference}, +}; +#[derive(Debug)] pub(crate) struct ReverseProxy { client: reqwest::Client, routing_table: RwLock, + orchestrator: OnceLock>, } #[derive(Debug, Default)] @@ -66,6 +71,13 @@ impl PartialEq for Domain { } } +#[derive(Debug)] +enum Destination { + ReverseProxied(Uri), + Internal(Uri), + NotFound, +} + impl RoutingTable { fn from_containers(containers: impl IntoIterator) -> Self { let mut path_maps = HashMap::new(); @@ -87,8 +99,7 @@ impl RoutingTable { } } - // TODO: Consider return a `Uri`` instead. - fn get_destination_uri_from_request(&self, request: &Request) -> Option { + fn get_destination_uri_from_request(&self, request: &Request) -> Destination { let req_uri = request.uri(); // First, attempt to match a domain. @@ -116,10 +127,17 @@ impl RoutingTable { Authority::from_str(&pc.host_addr().to_string()) .expect("SocketAddr should never fail to convert to Authority"), ); - return Some(Uri::from_parts(parts).expect("should not have invalidated Uri")); + return Destination::ReverseProxied( + Uri::from_parts(parts).expect("should not have invalidated Uri"), + ); } // Matching a domain did not succeed, let's try with a path. + // First, we attempt to match a special `_rockslide` path: + if req_uri.path().starts_with("/_rockslide") { + return Destination::Internal(req_uri.to_owned()); + } + // Reconstruct image location from path segments, keeping remainder intact. if let Some((image_location, remainder)) = split_path_base_url(req_uri) { if let Some(pc) = self.get_path_route(&image_location) { @@ -141,17 +159,18 @@ impl RoutingTable { parts.authority = Some(Authority::from_str(&container_addr.to_string()).unwrap()); parts.path_and_query = Some(PathAndQuery::from_str(&dest_path_and_query).unwrap()); - return Some(Uri::from_parts(parts).unwrap()); + return Destination::ReverseProxied(Uri::from_parts(parts).unwrap()); } } - None + Destination::NotFound } } #[derive(Debug)] enum AppError { NoSuchContainer, + InternalUrlInvalid, AssertionFailed(&'static str), NonUtf8Header, Internal(anyhow::Error), @@ -162,6 +181,7 @@ impl Display for AppError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { AppError::NoSuchContainer => f.write_str("no such container"), + AppError::InternalUrlInvalid => f.write_str("internal url invalid"), AppError::AssertionFailed(msg) => f.write_str(msg), AppError::NonUtf8Header => f.write_str("a header contained non-utf8 data"), AppError::Internal(err) => Display::fmt(err, f), @@ -184,6 +204,7 @@ impl IntoResponse for AppError { fn into_response(self) -> Response { match self { AppError::NoSuchContainer => StatusCode::NOT_FOUND.into_response(), + AppError::InternalUrlInvalid => StatusCode::NOT_FOUND.into_response(), AppError::AssertionFailed(msg) => (StatusCode::NOT_FOUND, msg).into_response(), AppError::NonUtf8Header => StatusCode::BAD_REQUEST.into_response(), AppError::Internal(err) => { @@ -198,6 +219,7 @@ impl ReverseProxy { Arc::new(ReverseProxy { client: reqwest::Client::new(), routing_table: RwLock::new(Default::default()), + orchestrator: OnceLock::new(), }) } @@ -214,6 +236,13 @@ impl ReverseProxy { let mut guard = self.routing_table.write().await; mem::swap(&mut *guard, &mut routing_table); } + + pub(crate) fn set_orchestrator(&self, orchestrator: Arc) -> &Self { + self.orchestrator + .set(orchestrator) + .expect("set already set orchestrator"); + self + } } fn split_path_base_url(uri: &Uri) -> Option<(ImageLocation, String)> { @@ -239,7 +268,52 @@ async fn route_request( routing_table.get_destination_uri_from_request(&request) }; - let dest = dest_uri.ok_or(AppError::NoSuchContainer)?; + let dest = match dest_uri { + Destination::ReverseProxied(dest) => dest, + Destination::Internal(uri) => { + let remainder = uri + .path() + .strip_prefix("/_rockslide/config/") + .ok_or(AppError::InternalUrlInvalid)?; + + let parts = remainder.split('/').collect::>(); + if parts.len() != 3 { + return Err(AppError::InternalUrlInvalid); + } + + if parts[2] != "prod" { + return Err(AppError::InternalUrlInvalid); + } + + let manifest_reference = ManifestReference::new( + ImageLocation::new(parts[0].to_owned(), parts[1].to_owned()), + Reference::new_tag(parts[2]), + ); + + // TODO: Match GET/POST. + let orchestrator = rp + .orchestrator + .get() + .ok_or_else(|| AppError::AssertionFailed("no orchestrator configured"))?; + + // GET: + let config = orchestrator + .load_config(&manifest_reference) + .await + .map_err(AppError::Internal)?; + let config_toml = + toml::to_string_pretty(&config).map_err(|err| AppError::Internal(err.into()))?; + + return Response::builder() + .status(StatusCode::OK) + .header(CONTENT_TYPE, "application/toml") + .body(Body::from(config_toml)) + .map_err(|_| AppError::AssertionFailed("should not fail to build response")); + } + Destination::NotFound => { + return Err(AppError::NoSuchContainer); + } + }; trace!(%dest, "reverse proxying"); // Note: `reqwest` and `axum` currently use different versions of `http` From 28b9d9a2e37f805493a3a293bf1cf38bb5c3e5a6 Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Fri, 5 Jan 2024 02:20:03 +0100 Subject: [PATCH 14/23] Stub out remainder of implementation for config access --- src/reverse_proxy.rs | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/src/reverse_proxy.rs b/src/reverse_proxy.rs index 7ee188a..4d22537 100644 --- a/src/reverse_proxy.rs +++ b/src/reverse_proxy.rs @@ -12,7 +12,7 @@ use axum::{ http::{ header::{CONTENT_TYPE, HOST}, uri::{Authority, Parts, PathAndQuery, Scheme}, - StatusCode, Uri, + Method, StatusCode, Uri, }, response::{IntoResponse, Response}, Router, @@ -271,6 +271,8 @@ async fn route_request( let dest = match dest_uri { Destination::ReverseProxied(dest) => dest, Destination::Internal(uri) => { + todo!("check access (master password)"); + let remainder = uri .path() .strip_prefix("/_rockslide/config/") @@ -290,25 +292,36 @@ async fn route_request( Reference::new_tag(parts[2]), ); - // TODO: Match GET/POST. let orchestrator = rp .orchestrator .get() .ok_or_else(|| AppError::AssertionFailed("no orchestrator configured"))?; - // GET: - let config = orchestrator - .load_config(&manifest_reference) - .await - .map_err(AppError::Internal)?; - let config_toml = - toml::to_string_pretty(&config).map_err(|err| AppError::Internal(err.into()))?; - - return Response::builder() - .status(StatusCode::OK) - .header(CONTENT_TYPE, "application/toml") - .body(Body::from(config_toml)) - .map_err(|_| AppError::AssertionFailed("should not fail to build response")); + return match *request.method() { + Method::GET => { + let config = orchestrator + .load_config(&manifest_reference) + .await + .map_err(AppError::Internal)?; + let config_toml = toml::to_string_pretty(&config) + .map_err(|err| AppError::Internal(err.into()))?; + + Response::builder() + .status(StatusCode::OK) + .header(CONTENT_TYPE, "application/toml") + .body(Body::from(config_toml)) + .map_err(|_| AppError::AssertionFailed("should not fail to build response")) + } + Method::PUT => { + todo!("handle PUT"); + Response::builder() + .status(StatusCode::OK) + // .header(CONTENT_TYPE, "application/toml") + .body(Body::from("TODO: Replace")) + .map_err(|_| AppError::AssertionFailed("should not fail to build response")) + } + _ => Err(AppError::InternalUrlInvalid), + }; } Destination::NotFound => { return Err(AppError::NoSuchContainer); From bc280d99d151ef438a35d43429ef47d45a57a2cf Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Fri, 5 Jan 2024 19:38:03 +0100 Subject: [PATCH 15/23] Make reverse proxy check credentials when accessing internal URLs --- src/container_orchestrator.rs | 1 - src/main.rs | 16 ++++++---------- src/registry.rs | 12 ++++-------- src/reverse_proxy.rs | 30 ++++++++++++++++++++++++------ 4 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/container_orchestrator.rs b/src/container_orchestrator.rs index 98e4f73..7f85a9e 100644 --- a/src/container_orchestrator.rs +++ b/src/container_orchestrator.rs @@ -30,7 +30,6 @@ macro_rules! try_quiet { }; } -#[derive(Debug)] pub(crate) struct ContainerOrchestrator { podman: Podman, reverse_proxy: Arc, diff --git a/src/main.rs b/src/main.rs index 40a175c..e4dc80f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -55,6 +55,9 @@ async fn main() -> anyhow::Result<()> { debug!(?cfg, "loaded configuration"); + let rockslide_pw = cfg.rockslide.master_key.as_secret_string(); + let auth_provider = Arc::new(cfg.rockslide.master_key); + let local_ip: IpAddr = if podman_is_remote() { info!("podman is remote, trying to guess IP address"); let local_hostname = gethostname(); @@ -78,12 +81,9 @@ async fn main() -> anyhow::Result<()> { let local_addr = SocketAddr::from(([127, 0, 0, 1], cfg.reverse_proxy.http_bind.port())); info!(%local_addr, "guessing local registry address"); - let reverse_proxy = ReverseProxy::new(); + let reverse_proxy = ReverseProxy::new(auth_provider.clone()); - let credentials = ( - "rockslide-podman".to_owned(), - cfg.rockslide.master_key.as_secret_string(), - ); + let credentials = ("rockslide-podman".to_owned(), rockslide_pw); let orchestrator = Arc::new(ContainerOrchestrator::new( &cfg.containers.podman_path, reverse_proxy.clone(), @@ -97,11 +97,7 @@ async fn main() -> anyhow::Result<()> { orchestrator.synchronize_all().await?; orchestrator.updated_published_set().await; - let registry = ContainerRegistry::new( - &cfg.registry.storage_path, - orchestrator, - cfg.rockslide.master_key, - )?; + let registry = ContainerRegistry::new(&cfg.registry.storage_path, orchestrator, auth_provider)?; let app = Router::new() .merge(registry.make_router()) diff --git a/src/registry.rs b/src/registry.rs index 0c3f8fa..d2af01e 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -92,24 +92,20 @@ impl IntoResponse for AppError { pub(crate) struct ContainerRegistry { realm: String, - auth_provider: Box, + auth_provider: Arc, storage: Box, hooks: Box, } impl ContainerRegistry { - pub(crate) fn new< - P: AsRef, - T: RegistryHooks + 'static, - A: AuthProvider + 'static, - >( + pub(crate) fn new, T: RegistryHooks + 'static>( storage_path: P, orchestrator: T, - auth_provider: A, + auth_provider: Arc, ) -> Result, FilesystemStorageError> { Ok(Arc::new(ContainerRegistry { realm: "ContainerRegistry".to_string(), - auth_provider: Box::new(auth_provider), + auth_provider: auth_provider, storage: Box::new(FilesystemStorage::new(storage_path)?), hooks: Box::new(orchestrator), })) diff --git a/src/reverse_proxy.rs b/src/reverse_proxy.rs index 4d22537..18da953 100644 --- a/src/reverse_proxy.rs +++ b/src/reverse_proxy.rs @@ -15,7 +15,7 @@ use axum::{ Method, StatusCode, Uri, }, response::{IntoResponse, Response}, - Router, + RequestExt, Router, }; use itertools::Itertools; use tokio::sync::RwLock; @@ -23,11 +23,13 @@ use tracing::{trace, warn}; use crate::{ container_orchestrator::{ContainerOrchestrator, PublishedContainer}, - registry::{storage::ImageLocation, ManifestReference, Reference}, + registry::{ + storage::ImageLocation, AuthProvider, ManifestReference, Reference, UnverifiedCredentials, + }, }; -#[derive(Debug)] pub(crate) struct ReverseProxy { + auth_provider: Arc, client: reqwest::Client, routing_table: RwLock, orchestrator: OnceLock>, @@ -173,6 +175,7 @@ enum AppError { InternalUrlInvalid, AssertionFailed(&'static str), NonUtf8Header, + AuthFailure(StatusCode), Internal(anyhow::Error), } @@ -184,6 +187,7 @@ impl Display for AppError { AppError::InternalUrlInvalid => f.write_str("internal url invalid"), AppError::AssertionFailed(msg) => f.write_str(msg), AppError::NonUtf8Header => f.write_str("a header contained non-utf8 data"), + AppError::AuthFailure(_) => f.write_str("authentication missing or not present"), AppError::Internal(err) => Display::fmt(err, f), } } @@ -207,6 +211,7 @@ impl IntoResponse for AppError { AppError::InternalUrlInvalid => StatusCode::NOT_FOUND.into_response(), AppError::AssertionFailed(msg) => (StatusCode::NOT_FOUND, msg).into_response(), AppError::NonUtf8Header => StatusCode::BAD_REQUEST.into_response(), + AppError::AuthFailure(status) => status.into_response(), AppError::Internal(err) => { (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() } @@ -215,8 +220,9 @@ impl IntoResponse for AppError { } impl ReverseProxy { - pub(crate) fn new() -> Arc { + pub(crate) fn new(auth_provider: Arc) -> Arc { Arc::new(ReverseProxy { + auth_provider, client: reqwest::Client::new(), routing_table: RwLock::new(Default::default()), orchestrator: OnceLock::new(), @@ -240,6 +246,7 @@ impl ReverseProxy { pub(crate) fn set_orchestrator(&self, orchestrator: Arc) -> &Self { self.orchestrator .set(orchestrator) + .map_err(|_| ()) .expect("set already set orchestrator"); self } @@ -271,7 +278,18 @@ async fn route_request( let dest = match dest_uri { Destination::ReverseProxied(dest) => dest, Destination::Internal(uri) => { - todo!("check access (master password)"); + let method = request.method().clone(); + // Note: The auth functionality has been lifted from `registry`. It may need to be + // refactored out because of that. + let creds = request + .extract::() + .await + .map_err(AppError::AuthFailure)?; + + // Any internal URL is subject to requiring auth through the master key. + if !rp.auth_provider.check_credentials(&creds).await { + todo!("return 403"); + } let remainder = uri .path() @@ -297,7 +315,7 @@ async fn route_request( .get() .ok_or_else(|| AppError::AssertionFailed("no orchestrator configured"))?; - return match *request.method() { + return match method { Method::GET => { let config = orchestrator .load_config(&manifest_reference) From 15def0b5f5e25e5923e6580ff5ad1148339c3e05 Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Sat, 6 Jan 2024 01:14:26 +0100 Subject: [PATCH 16/23] Return proper HTTP 403 error when accessing internal methods --- src/reverse_proxy.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/reverse_proxy.rs b/src/reverse_proxy.rs index 18da953..5c51473 100644 --- a/src/reverse_proxy.rs +++ b/src/reverse_proxy.rs @@ -288,7 +288,10 @@ async fn route_request( // Any internal URL is subject to requiring auth through the master key. if !rp.auth_provider.check_credentials(&creds).await { - todo!("return 403"); + return Response::builder() + .status(StatusCode::FORBIDDEN) + .body(Body::empty()) + .map_err(|_| AppError::AssertionFailed("should not fail to build response")); } let remainder = uri From bebd3ea71ce69b17ca3049122a5fe1009d63ede9 Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Sat, 6 Jan 2024 01:38:34 +0100 Subject: [PATCH 17/23] Finish pure store/load functionality for runtime configuration --- src/container_orchestrator.rs | 46 +++++++++++++++++++++++++++++++++++ src/reverse_proxy.rs | 38 ++++++++++++++--------------- 2 files changed, 65 insertions(+), 19 deletions(-) diff --git a/src/container_orchestrator.rs b/src/container_orchestrator.rs index 7f85a9e..336a3b8 100644 --- a/src/container_orchestrator.rs +++ b/src/container_orchestrator.rs @@ -14,6 +14,10 @@ use crate::{ use anyhow::Context; use axum::async_trait; +use axum::body::Body; +use axum::http::header::CONTENT_TYPE; +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; use sec::Secret; use serde::{Deserialize, Deserializer, Serialize}; use tracing::{debug, error, info}; @@ -61,6 +65,21 @@ pub(crate) struct RuntimeConfig { http_access: HashMap, } +impl IntoResponse for RuntimeConfig { + fn into_response(self) -> axum::response::Response { + toml::to_string_pretty(&self) + .ok() + .and_then(|config_toml| { + Response::builder() + .status(StatusCode::OK) + .header(CONTENT_TYPE, "application/toml") + .body(Body::from(config_toml)) + .ok() + }) + .unwrap_or_else(|| StatusCode::INTERNAL_SERVER_ERROR.into_response()) + } +} + impl ContainerOrchestrator { pub(crate) fn new, Q: AsRef>( podman_path: P, @@ -116,6 +135,33 @@ impl ContainerOrchestrator { toml::from_str(&raw).context("could not parse configuration") } + pub(crate) async fn save_config( + &self, + manifest_reference: &ManifestReference, + config: &RuntimeConfig, + ) -> anyhow::Result { + let config_path = self.config_path(manifest_reference); + let parent_dir = config_path + .parent() + .context("could not determine parent path")?; + + if !parent_dir.exists() { + tokio::fs::create_dir_all(parent_dir) + .await + .context("could not create parent path")?; + } + + let toml = toml::to_string_pretty(config).context("could not serialize new config")?; + + // TODO: Do atomic replace. + tokio::fs::write(config_path, toml) + .await + .context("failed to write new toml config")?; + + // Read back to verify. + self.load_config(manifest_reference).await + } + async fn fetch_managed_containers(&self, all: bool) -> anyhow::Result> { debug!("refreshing running containers"); diff --git a/src/reverse_proxy.rs b/src/reverse_proxy.rs index 5c51473..80e22d5 100644 --- a/src/reverse_proxy.rs +++ b/src/reverse_proxy.rs @@ -10,7 +10,7 @@ use axum::{ body::Body, extract::{Request, State}, http::{ - header::{CONTENT_TYPE, HOST}, + header::HOST, uri::{Authority, Parts, PathAndQuery, Scheme}, Method, StatusCode, Uri, }, @@ -22,7 +22,7 @@ use tokio::sync::RwLock; use tracing::{trace, warn}; use crate::{ - container_orchestrator::{ContainerOrchestrator, PublishedContainer}, + container_orchestrator::{ContainerOrchestrator, PublishedContainer, RuntimeConfig}, registry::{ storage::ImageLocation, AuthProvider, ManifestReference, Reference, UnverifiedCredentials, }, @@ -176,6 +176,7 @@ enum AppError { AssertionFailed(&'static str), NonUtf8Header, AuthFailure(StatusCode), + InvalidPayload, Internal(anyhow::Error), } @@ -188,6 +189,7 @@ impl Display for AppError { AppError::AssertionFailed(msg) => f.write_str(msg), AppError::NonUtf8Header => f.write_str("a header contained non-utf8 data"), AppError::AuthFailure(_) => f.write_str("authentication missing or not present"), + AppError::InvalidPayload => f.write_str("invalid payload"), AppError::Internal(err) => Display::fmt(err, f), } } @@ -212,6 +214,7 @@ impl IntoResponse for AppError { AppError::AssertionFailed(msg) => (StatusCode::NOT_FOUND, msg).into_response(), AppError::NonUtf8Header => StatusCode::BAD_REQUEST.into_response(), AppError::AuthFailure(status) => status.into_response(), + AppError::InvalidPayload => StatusCode::BAD_REQUEST.into_response(), AppError::Internal(err) => { (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() } @@ -281,10 +284,10 @@ async fn route_request( let method = request.method().clone(); // Note: The auth functionality has been lifted from `registry`. It may need to be // refactored out because of that. - let creds = request - .extract::() + let (creds, opt_body) = request + .extract::<(UnverifiedCredentials, Option), _>() .await - .map_err(AppError::AuthFailure)?; + .map_err(|_| AppError::AuthFailure(StatusCode::UNAUTHORIZED))?; // Any internal URL is subject to requiring auth through the master key. if !rp.auth_provider.check_credentials(&creds).await { @@ -324,22 +327,19 @@ async fn route_request( .load_config(&manifest_reference) .await .map_err(AppError::Internal)?; - let config_toml = toml::to_string_pretty(&config) - .map_err(|err| AppError::Internal(err.into()))?; - - Response::builder() - .status(StatusCode::OK) - .header(CONTENT_TYPE, "application/toml") - .body(Body::from(config_toml)) - .map_err(|_| AppError::AssertionFailed("should not fail to build response")) + + Ok(config.into_response()) } Method::PUT => { - todo!("handle PUT"); - Response::builder() - .status(StatusCode::OK) - // .header(CONTENT_TYPE, "application/toml") - .body(Body::from("TODO: Replace")) - .map_err(|_| AppError::AssertionFailed("should not fail to build response")) + let raw = dbg!(opt_body.ok_or(AppError::InvalidPayload)?); + let new_config: RuntimeConfig = + toml::from_str(&raw).map_err(|_| AppError::InvalidPayload)?; + let stored = orchestrator + .save_config(&manifest_reference, &new_config) + .await + .map_err(AppError::Internal)?; + + Ok(stored.into_response()) } _ => Err(AppError::InternalUrlInvalid), }; From 4101850d1e25e543c3aa1bf23153d1eb2c631490 Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Sat, 6 Jan 2024 01:41:43 +0100 Subject: [PATCH 18/23] Move reverse proxy case into match block --- src/reverse_proxy.rs | 82 +++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/src/reverse_proxy.rs b/src/reverse_proxy.rs index 80e22d5..fff8ce5 100644 --- a/src/reverse_proxy.rs +++ b/src/reverse_proxy.rs @@ -278,8 +278,44 @@ async fn route_request( routing_table.get_destination_uri_from_request(&request) }; - let dest = match dest_uri { - Destination::ReverseProxied(dest) => dest, + match dest_uri { + Destination::ReverseProxied(dest) => { + trace!(%dest, "reverse proxying"); + + // Note: `reqwest` and `axum` currently use different versions of `http` + let method = request.method().to_string().parse().map_err(|_| { + AppError::AssertionFailed("method http version mismatch workaround failed") + })?; + let response = rp.client.request(method, dest.to_string()).send().await; + + match response { + Ok(response) => { + let mut bld = Response::builder().status(response.status().as_u16()); + for (key, value) in response.headers() { + if HOP_BY_HOP.contains(key) { + continue; + } + + let key_string = key.to_string(); + let value_str = value.to_str().map_err(|_| AppError::NonUtf8Header)?; + + bld = bld.header(key_string, value_str); + } + Ok(bld.body(Body::from(response.bytes().await?)).map_err(|_| { + AppError::AssertionFailed("should not fail to construct response") + })?) + } + Err(err) => { + warn!(%err, %dest, "failed request"); + Ok(Response::builder() + .status(500) + .body(Body::empty()) + .map_err(|_| { + AppError::AssertionFailed("should not fail to construct error response") + })?) + } + } + } Destination::Internal(uri) => { let method = request.method().clone(); // Note: The auth functionality has been lifted from `registry`. It may need to be @@ -321,7 +357,7 @@ async fn route_request( .get() .ok_or_else(|| AppError::AssertionFailed("no orchestrator configured"))?; - return match method { + match method { Method::GET => { let config = orchestrator .load_config(&manifest_reference) @@ -342,47 +378,9 @@ async fn route_request( Ok(stored.into_response()) } _ => Err(AppError::InternalUrlInvalid), - }; - } - Destination::NotFound => { - return Err(AppError::NoSuchContainer); - } - }; - trace!(%dest, "reverse proxying"); - - // Note: `reqwest` and `axum` currently use different versions of `http` - let method = - request.method().to_string().parse().map_err(|_| { - AppError::AssertionFailed("method http version mismatch workaround failed") - })?; - let response = rp.client.request(method, dest.to_string()).send().await; - - match response { - Ok(response) => { - let mut bld = Response::builder().status(response.status().as_u16()); - for (key, value) in response.headers() { - if HOP_BY_HOP.contains(key) { - continue; - } - - let key_string = key.to_string(); - let value_str = value.to_str().map_err(|_| AppError::NonUtf8Header)?; - - bld = bld.header(key_string, value_str); } - Ok(bld - .body(Body::from(response.bytes().await?)) - .map_err(|_| AppError::AssertionFailed("should not fail to construct response"))?) - } - Err(err) => { - warn!(%err, %dest, "failed request"); - Ok(Response::builder() - .status(500) - .body(Body::empty()) - .map_err(|_| { - AppError::AssertionFailed("should not fail to construct error response") - })?) } + Destination::NotFound => Err(AppError::NoSuchContainer), } } From e1e813536a049b3d6b7c155f7c463c3f7df6aa79 Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Sat, 6 Jan 2024 02:07:12 +0100 Subject: [PATCH 19/23] Apply actual password protection to protected endpoints --- Cargo.toml | 2 +- src/container_orchestrator.rs | 17 +++++++++++---- src/registry/auth.rs | 26 ++++++++++++++++++++++- src/reverse_proxy.rs | 40 ++++++++++++++++++++++++++++------- 4 files changed, 71 insertions(+), 14 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9357a67..5241c0c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ hex = "0.4.3" itertools = "0.12.0" nom = "7.1.3" reqwest = { version = "0.11.23", default-features = false } -sec = { version = "1.0.0", features = ["deserialize"] } +sec = { version = "1.0.0", features = [ "deserialize", "serialize" ] } serde = { version = "1.0.193", features = [ "derive" ] } serde_json = "1.0.108" sha2 = "0.10.8" diff --git a/src/container_orchestrator.rs b/src/container_orchestrator.rs index 336a3b8..fbbe177 100644 --- a/src/container_orchestrator.rs +++ b/src/container_orchestrator.rs @@ -46,7 +46,7 @@ pub(crate) struct ContainerOrchestrator { pub(crate) struct PublishedContainer { host_addr: SocketAddr, manifest_reference: ManifestReference, - config: RuntimeConfig, + config: Arc, } impl PublishedContainer { @@ -57,12 +57,16 @@ impl PublishedContainer { pub(crate) fn host_addr(&self) -> SocketAddr { self.host_addr } + + pub(crate) fn config(&self) -> &Arc { + &self.config + } } #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub(crate) struct RuntimeConfig { #[serde(default)] - http_access: HashMap, + pub(crate) http_access: Option>>, } impl IntoResponse for RuntimeConfig { @@ -115,7 +119,12 @@ impl ContainerOrchestrator { self.configs_dir .join(location.repository()) .join(location.image()) - .join(manifest_reference.reference().to_string()) + .join( + manifest_reference + .reference() + .to_string() + .trim_start_matches(':'), + ) } pub(crate) async fn load_config( @@ -196,7 +205,7 @@ impl ContainerOrchestrator { return Ok(None); }; - let config = self.load_config(&manifest_reference).await?; + let config = Arc::new(self.load_config(&manifest_reference).await?); Ok(Some(PublishedContainer { host_addr: port_mapping diff --git a/src/registry/auth.rs b/src/registry/auth.rs index 8f7887a..ff677af 100644 --- a/src/registry/auth.rs +++ b/src/registry/auth.rs @@ -1,4 +1,4 @@ -use std::{str, sync::Arc}; +use std::{collections::HashMap, str, sync::Arc}; use axum::{ async_trait, @@ -96,6 +96,30 @@ impl AuthProvider for bool { } } +#[async_trait] +impl AuthProvider for HashMap> { + async fn check_credentials( + &self, + UnverifiedCredentials { + username: unverified_username, + password: unverified_password, + }: &UnverifiedCredentials, + ) -> bool { + if let Some(correct_password) = self.get(unverified_username) { + // TODO: Use constant-time compare. Maybe add to `sec`? + if correct_password == unverified_password { + return true; + } + } + + false + } + + async fn has_access_to(&self, _username: &str, _namespace: &str, _image: &str) -> bool { + true + } +} + #[async_trait] impl AuthProvider for Box where diff --git a/src/reverse_proxy.rs b/src/reverse_proxy.rs index fff8ce5..b3a00ad 100644 --- a/src/reverse_proxy.rs +++ b/src/reverse_proxy.rs @@ -75,7 +75,10 @@ impl PartialEq for Domain { #[derive(Debug)] enum Destination { - ReverseProxied(Uri), + ReverseProxied { + uri: Uri, + config: Arc, + }, Internal(Uri), NotFound, } @@ -129,9 +132,10 @@ impl RoutingTable { Authority::from_str(&pc.host_addr().to_string()) .expect("SocketAddr should never fail to convert to Authority"), ); - return Destination::ReverseProxied( - Uri::from_parts(parts).expect("should not have invalidated Uri"), - ); + return Destination::ReverseProxied { + uri: Uri::from_parts(parts).expect("should not have invalidated Uri"), + config: pc.config().clone(), + }; } // Matching a domain did not succeed, let's try with a path. @@ -161,7 +165,10 @@ impl RoutingTable { parts.authority = Some(Authority::from_str(&container_addr.to_string()).unwrap()); parts.path_and_query = Some(PathAndQuery::from_str(&dest_path_and_query).unwrap()); - return Destination::ReverseProxied(Uri::from_parts(parts).unwrap()); + return Destination::ReverseProxied { + uri: Uri::from_parts(parts).unwrap(), + config: pc.config().clone(), + }; } } @@ -271,7 +278,7 @@ fn split_path_base_url(uri: &Uri) -> Option<(ImageLocation, String)> { async fn route_request( State(rp): State>, - request: Request, + mut request: Request, ) -> Result { let dest_uri = { let routing_table = rp.routing_table.read().await; @@ -279,9 +286,26 @@ async fn route_request( }; match dest_uri { - Destination::ReverseProxied(dest) => { + Destination::ReverseProxied { uri: dest, config } => { trace!(%dest, "reverse proxying"); + // First, check if http authentication is enabled. + if let Some(ref http_access) = config.http_access { + let creds = request + .extract_parts::() + .await + .map_err(AppError::AuthFailure)?; + + if !http_access.check_credentials(&creds).await { + return Response::builder() + .status(StatusCode::FORBIDDEN) + .body(Body::empty()) + .map_err(|_| { + AppError::AssertionFailed("should not fail to build response") + }); + } + } + // Note: `reqwest` and `axum` currently use different versions of `http` let method = request.method().to_string().parse().map_err(|_| { AppError::AssertionFailed("method http version mismatch workaround failed") @@ -367,7 +391,7 @@ async fn route_request( Ok(config.into_response()) } Method::PUT => { - let raw = dbg!(opt_body.ok_or(AppError::InvalidPayload)?); + let raw = opt_body.ok_or(AppError::InvalidPayload)?; let new_config: RuntimeConfig = toml::from_str(&raw).map_err(|_| AppError::InvalidPayload)?; let stored = orchestrator From f0f8cfaab469191207dad12612f5c39cb066c5fb Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Sat, 6 Jan 2024 02:08:46 +0100 Subject: [PATCH 20/23] Note down commands to save/load config --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ab19055..a512c45 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,13 @@ # rockslide +## Container runtime configuration + +``` +curl -u :devpw localhost:3000/_rockslide/config/foo/bar/prod > foobarprod.toml +curl -v -X PUT -u :devpw localhost:3000/_rockslide/config/foo/bar/prod --data-binar +y "@foobarprod.toml" +``` + ## macOS suppport macOS is supported as a tier 2 platform to develop rockslide itself, although currently completely untested for production use. [podman can run on Mac OS X](https://podman.io/docs/installation), where it will launch a Linux virtual machine to run containers. The `rockslide` application itself and its supporting nix-derivation all account for being built on macOS. @@ -21,4 +29,4 @@ podman run -it debian:latest /bin/sh -c 'echo everything is working fine' `rockslide` will check an envvar `PODMAN_IS_REMOTE`, if it is `true`, it will assume a remote instance and act accordingly. This envvar is set to `true` automatically when running `nix-shell` on a macOS machine. -With these prerequisites fulfilled, `rockslide` should operate normally as it does on Linux. \ No newline at end of file +With these prerequisites fulfilled, `rockslide` should operate normally as it does on Linux. From 6433325408f9de7dcc275bb73296a76d32abc468 Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Sat, 6 Jan 2024 02:11:24 +0100 Subject: [PATCH 21/23] Update containers when new configuration is uploaded to make changes take effect --- src/reverse_proxy.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/reverse_proxy.rs b/src/reverse_proxy.rs index b3a00ad..4a23ea2 100644 --- a/src/reverse_proxy.rs +++ b/src/reverse_proxy.rs @@ -399,6 +399,9 @@ async fn route_request( .await .map_err(AppError::Internal)?; + // Update containers. + orchestrator.updated_published_set().await; + Ok(stored.into_response()) } _ => Err(AppError::InternalUrlInvalid), From 1824634dc259cacdcf5ca32587a9461747938956 Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Sat, 6 Jan 2024 02:23:11 +0100 Subject: [PATCH 22/23] Add proper error handling to cause password prompts --- src/reverse_proxy.rs | 52 +++++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/src/reverse_proxy.rs b/src/reverse_proxy.rs index 4a23ea2..708ca64 100644 --- a/src/reverse_proxy.rs +++ b/src/reverse_proxy.rs @@ -182,7 +182,10 @@ enum AppError { InternalUrlInvalid, AssertionFailed(&'static str), NonUtf8Header, - AuthFailure(StatusCode), + AuthFailure { + realm: &'static str, + status: StatusCode, + }, InvalidPayload, Internal(anyhow::Error), } @@ -195,7 +198,7 @@ impl Display for AppError { AppError::InternalUrlInvalid => f.write_str("internal url invalid"), AppError::AssertionFailed(msg) => f.write_str(msg), AppError::NonUtf8Header => f.write_str("a header contained non-utf8 data"), - AppError::AuthFailure(_) => f.write_str("authentication missing or not present"), + AppError::AuthFailure { .. } => f.write_str("authentication missing or not present"), AppError::InvalidPayload => f.write_str("invalid payload"), AppError::Internal(err) => Display::fmt(err, f), } @@ -220,7 +223,11 @@ impl IntoResponse for AppError { AppError::InternalUrlInvalid => StatusCode::NOT_FOUND.into_response(), AppError::AssertionFailed(msg) => (StatusCode::NOT_FOUND, msg).into_response(), AppError::NonUtf8Header => StatusCode::BAD_REQUEST.into_response(), - AppError::AuthFailure(status) => status.into_response(), + AppError::AuthFailure { realm, status } => Response::builder() + .status(status) + .header("WWW-Authenticate", format!("basic realm={realm}")) + .body(Body::empty()) + .expect("should never fail to build auth failure response"), AppError::InvalidPayload => StatusCode::BAD_REQUEST.into_response(), AppError::Internal(err) => { (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()).into_response() @@ -294,15 +301,17 @@ async fn route_request( let creds = request .extract_parts::() .await - .map_err(AppError::AuthFailure)?; + .map_err(|status| AppError::AuthFailure { + // TODO: Output container name? + realm: "password protected container", + status, + })?; if !http_access.check_credentials(&creds).await { - return Response::builder() - .status(StatusCode::FORBIDDEN) - .body(Body::empty()) - .map_err(|_| { - AppError::AssertionFailed("should not fail to build response") - }); + return Err(AppError::AuthFailure { + realm: "password protected container", + status: StatusCode::FORBIDDEN, + }); } } @@ -344,17 +353,26 @@ async fn route_request( let method = request.method().clone(); // Note: The auth functionality has been lifted from `registry`. It may need to be // refactored out because of that. - let (creds, opt_body) = request - .extract::<(UnverifiedCredentials, Option), _>() + let creds: UnverifiedCredentials = + request + .extract_parts() + .await + .map_err(|status| AppError::AuthFailure { + realm: "internal", + status, + })?; + + let opt_body = request + .extract::, _>() .await - .map_err(|_| AppError::AuthFailure(StatusCode::UNAUTHORIZED))?; + .expect("infallible"); // Any internal URL is subject to requiring auth through the master key. if !rp.auth_provider.check_credentials(&creds).await { - return Response::builder() - .status(StatusCode::FORBIDDEN) - .body(Body::empty()) - .map_err(|_| AppError::AssertionFailed("should not fail to build response")); + return Err(AppError::AuthFailure { + realm: "internal", + status: StatusCode::FORBIDDEN, + }); } let remainder = uri From b0ec6f609ceb706f3064e635f8b1f1059db8bcf6 Mon Sep 17 00:00:00 2001 From: Marc Brinkmann Date: Sat, 6 Jan 2024 02:36:42 +0100 Subject: [PATCH 23/23] Add test and new structure for runtime config --- src/container_orchestrator.rs | 42 +++++++++++++++++++++++++++++++++-- src/registry.rs | 2 +- src/reverse_proxy.rs | 6 ++--- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/container_orchestrator.rs b/src/container_orchestrator.rs index fbbe177..5eacd0d 100644 --- a/src/container_orchestrator.rs +++ b/src/container_orchestrator.rs @@ -63,10 +63,16 @@ impl PublishedContainer { } } -#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub(crate) struct RuntimeConfig { #[serde(default)] - pub(crate) http_access: Option>>, + pub(crate) http: Http, +} + +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +pub(crate) struct Http { + #[serde(default)] + pub(crate) access: Option>>, } impl IntoResponse for RuntimeConfig { @@ -380,3 +386,35 @@ impl PortMapping { Some((ip, self.host_port).into()) } } + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use sec::Secret; + + use crate::container_orchestrator::Http; + + use super::RuntimeConfig; + + #[test] + fn can_parse_sample_configs() { + let example = r#" + [http] + access = { someuser = "somepw" } + "#; + + let parsed: RuntimeConfig = toml::from_str(example).expect("should parse"); + + let mut pw_map = HashMap::new(); + pw_map.insert("someuser".to_owned(), Secret::new("somepw".to_owned())); + assert_eq!( + parsed, + RuntimeConfig { + http: Http { + access: Some(pw_map) + } + } + ) + } +} diff --git a/src/registry.rs b/src/registry.rs index d2af01e..8227228 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -512,7 +512,7 @@ mod tests { let tmp = TempDir::new("rockslide-test").expect("could not create temporary directory"); let password = "random-test-password".to_owned(); - let master_key = MasterKey::new_key(password.clone()); + let master_key = Arc::new(MasterKey::new_key(password.clone())); let registry = ContainerRegistry::new(tmp.as_ref(), (), master_key) .expect("should not fail to create app"); diff --git a/src/reverse_proxy.rs b/src/reverse_proxy.rs index 708ca64..a23e78c 100644 --- a/src/reverse_proxy.rs +++ b/src/reverse_proxy.rs @@ -297,7 +297,7 @@ async fn route_request( trace!(%dest, "reverse proxying"); // First, check if http authentication is enabled. - if let Some(ref http_access) = config.http_access { + if let Some(ref http_access) = config.http.access { let creds = request .extract_parts::() .await @@ -310,7 +310,7 @@ async fn route_request( if !http_access.check_credentials(&creds).await { return Err(AppError::AuthFailure { realm: "password protected container", - status: StatusCode::FORBIDDEN, + status: StatusCode::UNAUTHORIZED, }); } } @@ -371,7 +371,7 @@ async fn route_request( if !rp.auth_provider.check_credentials(&creds).await { return Err(AppError::AuthFailure { realm: "internal", - status: StatusCode::FORBIDDEN, + status: StatusCode::UNAUTHORIZED, }); }