Skip to content

Commit

Permalink
adding new tests for lease
Browse files Browse the repository at this point in the history
  • Loading branch information
@drmorr0
drmorr0 committed Apr 9, 2024
1 parent 508f42d commit 4ebc0d4
Show file tree
Hide file tree
Showing 9 changed files with 357 additions and 94 deletions.
5 changes: 2 additions & 3 deletions ctrl/controller.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ use kube::api::{
};
use kube::error::ErrorResponse;
use kube::runtime::controller::Action;
use kube::Error::Api;
use kube::ResourceExt;
use serde_json::json;
use simkube::api::v1::build_simulation_root;
Expand Down Expand Up @@ -95,7 +94,7 @@ pub(super) async fn setup_driver(
bail!(SkControllerError::namespace_not_found(&sim.metrics_ns()));
};

match try_claim_lease(ctx.client.clone(), sim, metaroot, ctrl_ns, &UtcClock).await? {
match try_claim_lease(ctx.client.clone(), sim, metaroot, ctrl_ns, Box::new(UtcClock)).await? {
LeaseState::Claimed => (),
LeaseState::WaitingForClaim(t) => {
info!("sleeping for {t} seconds");
Expand Down Expand Up @@ -197,7 +196,7 @@ pub(super) async fn cleanup(ctx: &SimulationContext, sim: &Simulation) {

info!("cleaning up prometheus resources");
if let Err(e) = prom_api.delete(&ctx.prometheus_name, &Default::default()).await {
if matches!(e, Api(ErrorResponse { code: 404, .. })) {
if matches!(e, kube::Error::Api(ErrorResponse { code: 404, .. })) {
warn!("prometheus object not found; maybe already cleaned up?");
} else {
error!("Error cleaning up Prometheus: {e:?}");
Expand Down
151 changes: 84 additions & 67 deletions ctrl/tests/controller_test.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,36 +10,6 @@ use super::controller::*;
use super::*;
use crate::objects::*;

#[fixture]
fn sim() -> Simulation {
Simulation {
metadata: metav1::ObjectMeta {
name: Some(TEST_SIM_NAME.into()),
uid: Some("1234-asdf".into()),
..Default::default()
},
spec: SimulationSpec {
driver_namespace: TEST_NAMESPACE.into(),
trace_path: "file:///foo/bar".into(),
metrics_config: Some(Default::default()),
..Default::default()
},
status: Default::default(),
}
}

#[fixture]
fn root() -> SimulationRoot {
SimulationRoot {
metadata: metav1::ObjectMeta {
name: Some(format!("sk-{TEST_SIM_NAME}-root")),
uid: Some("qwerty-5678".into()),
..Default::default()
},
spec: SimulationRootSpec {},
}
}

#[fixture]
fn opts() -> Options {
Options {
Expand All @@ -53,9 +23,9 @@ fn opts() -> Options {

#[rstest]
#[tokio::test]
async fn test_fetch_driver_status_no_driver(sim: Simulation, opts: Options) {
async fn test_fetch_driver_status_no_driver(test_sim: Simulation, opts: Options) {
let (mut fake_apiserver, client) = make_fake_apiserver();
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&sim);
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&test_sim);

let driver_name = ctx.driver_name.clone();
fake_apiserver
Expand All @@ -67,9 +37,9 @@ async fn test_fetch_driver_status_no_driver(sim: Simulation, opts: Options) {

#[rstest]
#[tokio::test]
async fn test_fetch_driver_status_driver_no_status(sim: Simulation, opts: Options) {
async fn test_fetch_driver_status_driver_no_status(test_sim: Simulation, opts: Options) {
let (mut fake_apiserver, client) = make_fake_apiserver();
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&sim);
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&test_sim);

let driver_name = ctx.driver_name.clone();
fake_apiserver
Expand All @@ -84,9 +54,9 @@ async fn test_fetch_driver_status_driver_no_status(sim: Simulation, opts: Option

#[rstest]
#[tokio::test]
async fn test_fetch_driver_status_driver_running(sim: Simulation, opts: Options) {
async fn test_fetch_driver_status_driver_running(test_sim: Simulation, opts: Options) {
let (mut fake_apiserver, client) = make_fake_apiserver();
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&sim);
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&test_sim);

let driver_name = ctx.driver_name.clone();
fake_apiserver
Expand All @@ -107,15 +77,15 @@ async fn test_fetch_driver_status_driver_running(sim: Simulation, opts: Options)
#[case::complete(JOB_STATUS_CONDITION_COMPLETE)]
#[case::failed(JOB_STATUS_CONDITION_FAILED)]
#[tokio::test]
async fn test_fetch_driver_status_driver_finished(sim: Simulation, opts: Options, #[case] status: &'static str) {
async fn test_fetch_driver_status_driver_finished(test_sim: Simulation, opts: Options, #[case] status: &'static str) {
let expected_state = if status == JOB_STATUS_CONDITION_COMPLETE {
SimulationState::Finished
} else {
SimulationState::Failed
};

let (mut fake_apiserver, client) = make_fake_apiserver();
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&sim);
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&test_sim);

let driver_name = ctx.driver_name.clone();
fake_apiserver
Expand All @@ -133,36 +103,78 @@ async fn test_fetch_driver_status_driver_finished(sim: Simulation, opts: Options
}

#[rstest]
#[traced_test]
#[tokio::test]
async fn test_setup_driver_no_ns(sim: Simulation, root: SimulationRoot, opts: Options) {
async fn test_setup_driver_no_ns(test_sim: Simulation, test_sim_root: SimulationRoot, opts: Options) {
let (mut fake_apiserver, client) = make_fake_apiserver();
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&sim);
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&test_sim);
fake_apiserver
.handle_not_found(format!("/api/v1/namespaces/{DEFAULT_METRICS_NS}"))
.build();

assert!(matches!(
setup_driver(&ctx, &sim, &root, TEST_CTRL_NAMESPACE)
setup_driver(&ctx, &test_sim, &test_sim_root, TEST_CTRL_NAMESPACE)
.await
.unwrap_err()
.downcast::<SkControllerError>()
.unwrap(),
SkControllerError::NamespaceNotFound(_)
))
));
fake_apiserver.assert();
}

#[rstest]
#[traced_test]
#[tokio::test]
async fn test_setup_driver_lease_claim_fails(test_sim: Simulation, test_sim_root: SimulationRoot, opts: Options) {
let (mut fake_apiserver, client) = make_fake_apiserver();
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&test_sim);
fake_apiserver
.handle(|when, then| {
when.method(GET).path(format!("/api/v1/namespaces/{DEFAULT_METRICS_NS}"));
then.json_body(json!({
"kind": "Namespace",
}));
})
.handle_not_found(format!(
"/apis/coordination.k8s.io/v1/namespaces/{TEST_CTRL_NAMESPACE}/leases/{SK_LEASE_NAME}"
))
.handle(move |when, then| {
when.method(POST)
.path(format!("/apis/coordination.k8s.io/v1/namespaces/{TEST_CTRL_NAMESPACE}/leases"));
then.status(409).json_body(json!({
"kind": "Status",
"apiVersion": "v1",
"metadata": {},
"message": "the object has been modified; please apply your changes to the latest version and try again",
"status": "Failure",
"reason": "Conflict",
"code": 409
}));
})
.build();

let err = setup_driver(&ctx, &test_sim, &test_sim_root, TEST_CTRL_NAMESPACE)
.await
.unwrap_err()
.downcast::<kube::api::entry::CommitError>()
.unwrap();
assert!(matches!(err, kube::api::entry::CommitError::Save(..)));
fake_apiserver.assert();
}

#[rstest]
#[traced_test]
#[tokio::test]
async fn test_setup_driver_create_prom(sim: Simulation, root: SimulationRoot, opts: Options) {
async fn test_setup_driver_create_prom(test_sim: Simulation, test_sim_root: SimulationRoot, opts: Options) {
let (mut fake_apiserver, client) = make_fake_apiserver();
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&sim);
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&test_sim);

let lease_obj = build_lease(&sim, &root, TEST_CTRL_NAMESPACE, UtcClock.now());
let lease_obj = build_lease(&test_sim, &test_sim_root, TEST_CTRL_NAMESPACE, UtcClock.now());
let driver_ns = ctx.driver_ns.clone();
let prom_name = ctx.prometheus_name.clone();
let driver_ns_obj = build_driver_namespace(&ctx, &sim);
let prom_obj = build_prometheus(&ctx.prometheus_name, &sim, &sim.spec.metrics_config.clone().unwrap());
let driver_ns_obj = build_driver_namespace(&ctx, &test_sim);
let prom_obj = build_prometheus(&ctx.prometheus_name, &test_sim, &test_sim.spec.metrics_config.clone().unwrap());

fake_apiserver
.handle(|when, then| {
Expand All @@ -189,7 +201,9 @@ async fn test_setup_driver_create_prom(sim: Simulation, root: SimulationRoot, op
})
.build();
assert_eq!(
setup_driver(&ctx, &sim, &root, TEST_CTRL_NAMESPACE).await.unwrap(),
setup_driver(&ctx, &test_sim, &test_sim_root, TEST_CTRL_NAMESPACE)
.await
.unwrap(),
Action::requeue(REQUEUE_DURATION)
);
fake_apiserver.assert();
Expand All @@ -202,27 +216,27 @@ async fn test_setup_driver_create_prom(sim: Simulation, root: SimulationRoot, op
#[traced_test]
#[tokio::test]
async fn test_setup_driver_wait_prom(
mut sim: Simulation,
root: SimulationRoot,
mut test_sim: Simulation,
test_sim_root: SimulationRoot,
opts: Options,
#[case] ready: bool,
#[case] disabled: bool,
) {
env::set_var("POD_SVC_ACCOUNT", "asdf");
let (mut fake_apiserver, client) = make_fake_apiserver();
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&sim);
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&test_sim);

let driver_ns = ctx.driver_ns.clone();
let prom_name = ctx.prometheus_name.clone();
let driver_svc_name = ctx.driver_svc.clone();
let webhook_name = ctx.webhook_name.clone();
let driver_name = ctx.driver_name.clone();

let lease_obj = build_lease(&sim, &root, TEST_CTRL_NAMESPACE, UtcClock.now());
let driver_ns_obj = build_driver_namespace(&ctx, &sim);
let driver_svc_obj = build_driver_service(&ctx, &root);
let webhook_obj = build_mutating_webhook(&ctx, &root);
let driver_obj = build_driver_job(&ctx, &sim, "".into(), TEST_CTRL_NAMESPACE).unwrap();
let lease_obj = build_lease(&test_sim, &test_sim_root, TEST_CTRL_NAMESPACE, UtcClock.now());
let driver_ns_obj = build_driver_namespace(&ctx, &test_sim);
let driver_svc_obj = build_driver_service(&ctx, &test_sim_root);
let webhook_obj = build_mutating_webhook(&ctx, &test_sim_root);
let driver_obj = build_driver_job(&ctx, &test_sim, "".into(), TEST_CTRL_NAMESPACE).unwrap();

fake_apiserver
.handle(|when, then| {
Expand All @@ -242,9 +256,10 @@ async fn test_setup_driver_wait_prom(
});

if disabled {
sim.spec.metrics_config = None;
test_sim.spec.metrics_config = None;
} else {
let prom_obj = build_prometheus(&ctx.prometheus_name, &sim, &sim.spec.metrics_config.clone().unwrap());
let prom_obj =
build_prometheus(&ctx.prometheus_name, &test_sim, &test_sim.spec.metrics_config.clone().unwrap());
fake_apiserver.handle(move |when, then| {
when.method(GET)
.path(format!("/apis/monitoring.coreos.com/v1/namespaces/monitoring/prometheuses/{prom_name}"));
Expand Down Expand Up @@ -288,7 +303,9 @@ async fn test_setup_driver_wait_prom(
});
}
fake_apiserver.build();
let res = setup_driver(&ctx, &sim, &root, TEST_CTRL_NAMESPACE).await.unwrap();
let res = setup_driver(&ctx, &test_sim, &test_sim_root, TEST_CTRL_NAMESPACE)
.await
.unwrap();
if ready {
assert_eq!(res, Action::await_change());
} else {
Expand All @@ -301,9 +318,9 @@ async fn test_setup_driver_wait_prom(
#[rstest]
#[traced_test]
#[tokio::test]
async fn test_cleanup(sim: Simulation, opts: Options) {
async fn test_cleanup(test_sim: Simulation, opts: Options) {
let (mut fake_apiserver, client) = make_fake_apiserver();
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&sim);
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&test_sim);

let root = ctx.metaroot_name.clone();
let prom = ctx.prometheus_name.clone();
Expand All @@ -316,9 +333,9 @@ async fn test_cleanup(sim: Simulation, opts: Options) {
.handle(move |when, then| {
when.path(format!("/apis/monitoring.coreos.com/v1/namespaces/monitoring/prometheuses/{prom}"));
then.json_body(status_ok());
});
fake_apiserver.build();
cleanup(&ctx, &sim).await;
})
.build();
cleanup(&ctx, &test_sim).await;

assert!(!logs_contain("ERROR"));
fake_apiserver.assert();
Expand All @@ -329,9 +346,9 @@ async fn test_cleanup(sim: Simulation, opts: Options) {
#[rstest]
#[traced_test]
#[tokio::test]
async fn test_cleanup_not_found(sim: Simulation, opts: Options) {
async fn test_cleanup_not_found(test_sim: Simulation, opts: Options) {
let (mut fake_apiserver, client) = make_fake_apiserver();
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&sim);
let ctx = Arc::new(SimulationContext::new(client, opts)).with_sim(&test_sim);

let root = ctx.metaroot_name.clone();
let prom = ctx.prometheus_name.clone();
Expand All @@ -343,7 +360,7 @@ async fn test_cleanup_not_found(sim: Simulation, opts: Options) {
})
.handle_not_found(format!("/apis/monitoring.coreos.com/v1/namespaces/monitoring/prometheuses/{prom}"))
.build();
cleanup(&ctx, &sim).await;
cleanup(&ctx, &test_sim).await;

assert!(logs_contain("WARN"));
fake_apiserver.assert();
Expand Down
2 changes: 1 addition & 1 deletion driver/runner.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ pub async fn run_trace(ctx: DriverContext, client: kube::Client) -> EmptyResult
let sim_end_ts = ctx.store.end_ts().ok_or(anyhow!("no trace data"))?;
let sim_duration = sim_end_ts - sim_ts;

try_update_lease(client.clone(), &ctx.sim, &ctx.ctrl_ns, sim_duration, &UtcClock).await?;
try_update_lease(client.clone(), &ctx.sim, &ctx.ctrl_ns, sim_duration, Box::new(UtcClock)).await?;

for (evt, maybe_next_ts) in ctx.store.iter() {
// We're currently assuming that all tracked objects are namespace-scoped,
Expand Down
Loading

0 comments on commit 4ebc0d4

Please sign in to comment.