diff --git a/scheds/rust/scx_rusty/src/bpf/sdt_dom.h b/scheds/rust/scx_rusty/src/bpf/lb_domain.h similarity index 53% rename from scheds/rust/scx_rusty/src/bpf/sdt_dom.h rename to scheds/rust/scx_rusty/src/bpf/lb_domain.h index 6a366f645..a1cb21de1 100644 --- a/scheds/rust/scx_rusty/src/bpf/sdt_dom.h +++ b/scheds/rust/scx_rusty/src/bpf/lb_domain.h @@ -19,76 +19,77 @@ struct { __type(value, struct lb_domain); __uint(max_entries, MAX_DOMS); __uint(map_flags, 0); -} sdt_dom_map SEC(".maps"); +} lb_domain_map SEC(".maps"); -struct sdt_allocator sdt_dom_allocator; +volatile dom_ptr dom_ctxs[MAX_DOMS]; +struct sdt_allocator lb_domain_allocator; __hidden __noinline -int sdt_dom_init(void) +int lb_domain_init(void) { - return sdt_alloc_init(&sdt_dom_allocator, sizeof(struct dom_ctx)); + return sdt_alloc_init(&lb_domain_allocator, sizeof(struct dom_ctx)); } __hidden __noinline -dom_ptr sdt_dom_alloc(u32 dom_id) +dom_ptr lb_domain_alloc(u32 dom_id) { struct sdt_data __arena *data = NULL; - struct lb_domain mval; + struct lb_domain lb_domain; dom_ptr domc; int ret; - data = sdt_alloc(&sdt_dom_allocator); + data = sdt_alloc(&lb_domain_allocator); cast_kern(data); - mval.tid = data->tid; - mval.domc = (dom_ptr)data->payload; + lb_domain.tid = data->tid; + lb_domain.domc = (dom_ptr)data->payload; - ret = bpf_map_update_elem(&sdt_dom_map, &dom_id, &mval, + ret = bpf_map_update_elem(&lb_domain_map, &dom_id, &lb_domain, BPF_EXIST); if (ret) { - sdt_free_idx(&sdt_dom_allocator, data->tid.idx); + sdt_free_idx(&lb_domain_allocator, data->tid.idx); return NULL; } - domc = mval.domc; + domc = lb_domain.domc; cast_kern(domc); return domc; } __hidden -void sdt_dom_free(dom_ptr domc) +void lb_domain_free(dom_ptr domc) { - struct lb_domain *mval; + struct lb_domain *lb_domain; u32 key = domc->id; sdt_subprog_init_arena(); - mval = bpf_map_lookup_elem(&sdt_dom_map, &key); - if (!mval) + lb_domain = bpf_map_lookup_elem(&lb_domain_map, &key); + if (!lb_domain) return; - sdt_free_idx(&sdt_dom_allocator, mval->tid.idx); - mval->domc = NULL; + sdt_free_idx(&lb_domain_allocator, lb_domain->tid.idx); + lb_domain->domc = NULL; - bpf_map_delete_elem(&sdt_dom_map, &key); + bpf_map_delete_elem(&lb_domain_map, &key); } static __always_inline -struct lb_domain *sdt_dom_val(u32 dom_id) +struct lb_domain *lb_domain_get(u32 dom_id) { - return bpf_map_lookup_elem(&sdt_dom_map, &dom_id); + return bpf_map_lookup_elem(&lb_domain_map, &dom_id); } static dom_ptr try_lookup_dom_ctx_arena(u32 dom_id) { - struct lb_domain *mval; + struct lb_domain *lb_domain; - mval = sdt_dom_val(dom_id); - if (!mval) + lb_domain = lb_domain_get(dom_id); + if (!lb_domain) return NULL; - return mval->domc; + return lb_domain->domc; } static dom_ptr try_lookup_dom_ctx(u32 dom_id) @@ -115,13 +116,13 @@ static dom_ptr lookup_dom_ctx(u32 dom_id) static struct bpf_spin_lock *lookup_dom_vtime_lock(dom_ptr domc) { - struct lb_domain *mval; + struct lb_domain *lb_domain; - mval = sdt_dom_val(domc->id); - if (!mval) { + lb_domain = lb_domain_get(domc->id); + if (!lb_domain) { scx_bpf_error("Failed to lookup dom map value"); return NULL; } - return &mval->vtime_lock; + return &lb_domain->vtime_lock; } diff --git a/scheds/rust/scx_rusty/src/bpf/main.bpf.c b/scheds/rust/scx_rusty/src/bpf/main.bpf.c index b778b52e3..327a1dd63 100644 --- a/scheds/rust/scx_rusty/src/bpf/main.bpf.c +++ b/scheds/rust/scx_rusty/src/bpf/main.bpf.c @@ -47,7 +47,7 @@ #endif #include "intf.h" -#include "sdt_dom.h" +#include "lb_domain.h" #include #include @@ -76,7 +76,6 @@ const volatile u32 dom_numa_id_map[MAX_DOMS]; const volatile u64 dom_cpumasks[MAX_DOMS][MAX_CPUS / 64]; const volatile u64 numa_cpumasks[MAX_NUMA_NODES][MAX_CPUS / 64]; const volatile u32 load_half_life = 1000000000 /* 1s */; -volatile dom_ptr doms[MAX_DOMS]; const volatile bool kthreads_local; const volatile bool fifo_sched = false; @@ -196,17 +195,6 @@ struct { __uint(map_flags, 0); } node_data SEC(".maps"); -/* - * Domain context - */ -struct { - __uint(type, BPF_MAP_TYPE_ARRAY); - __type(key, u32); - __type(value, struct dom_ctx); - __uint(max_entries, MAX_DOMS); - __uint(map_flags, 0); -} dom_data SEC(".maps"); - struct lock_wrapper { struct bpf_spin_lock lock; }; @@ -551,24 +539,24 @@ static void refresh_tune_params(void) bpf_for(cpu, 0, nr_cpu_ids) { u32 dom_id = cpu_to_dom_id(cpu); - struct lb_domain *dval; + struct lb_domain *lb_domain; if (is_offline_cpu(cpu)) continue; - if (!(dval = sdt_dom_val(dom_id))) + if (!(lb_domain = lb_domain_get(dom_id))) return; if (tune_input.direct_greedy_cpumask[cpu / 64] & (1LLU << (cpu % 64))) { if (direct_greedy_cpumask) bpf_cpumask_set_cpu(cpu, direct_greedy_cpumask); - if (dval->direct_greedy_cpumask) - bpf_cpumask_set_cpu(cpu, dval->direct_greedy_cpumask); + if (lb_domain->direct_greedy_cpumask) + bpf_cpumask_set_cpu(cpu, lb_domain->direct_greedy_cpumask); } else { if (direct_greedy_cpumask) bpf_cpumask_clear_cpu(cpu, direct_greedy_cpumask); - if (dval->direct_greedy_cpumask) - bpf_cpumask_clear_cpu(cpu, dval->direct_greedy_cpumask); + if (lb_domain->direct_greedy_cpumask) + bpf_cpumask_clear_cpu(cpu, lb_domain->direct_greedy_cpumask); } if (tune_input.kick_greedy_cpumask[cpu / 64] & (1LLU << (cpu % 64))) { @@ -802,7 +790,7 @@ static bool task_set_domain(struct task_struct *p __arg_trusted, { dom_ptr old_domc, new_domc; struct bpf_cpumask *d_cpumask, *t_cpumask; - struct lb_domain *new_dval; + struct lb_domain *new_lb_domain; struct task_ctx *taskc; u32 old_dom_id; @@ -826,13 +814,13 @@ static bool task_set_domain(struct task_struct *p __arg_trusted, if (!new_domc) return false; - new_dval = sdt_dom_val(new_dom_id); - if (!new_dval) { - scx_bpf_error("no dval for dom%d\n", new_dom_id); + new_lb_domain = lb_domain_get(new_dom_id); + if (!new_lb_domain) { + scx_bpf_error("no lb_domain for dom%d\n", new_dom_id); return false; } - d_cpumask = new_dval->cpumask; + d_cpumask = new_lb_domain->cpumask; if (!d_cpumask) { scx_bpf_error("Failed to get dom%u cpumask kptr", new_dom_id); @@ -871,7 +859,7 @@ static s32 try_sync_wakeup(struct task_struct *p, struct task_ctx *taskc, s32 cpu; const struct cpumask *idle_cpumask; bool share_llc, has_idle; - struct lb_domain *dval; + struct lb_domain *lb_domain; struct bpf_cpumask *d_cpumask; struct pcpu_ctx *pcpuc; @@ -880,11 +868,11 @@ static s32 try_sync_wakeup(struct task_struct *p, struct task_ctx *taskc, if (!pcpuc) return -ENOENT; - dval = sdt_dom_val(pcpuc->dom_id); - if (!dval) + lb_domain = lb_domain_get(pcpuc->dom_id); + if (!lb_domain) return -ENOENT; - d_cpumask = dval->cpumask; + d_cpumask = lb_domain->cpumask; if (!d_cpumask) { scx_bpf_error("Failed to acquire dom%u cpumask kptr", taskc->target_dom); @@ -1025,7 +1013,7 @@ s32 BPF_STRUCT_OPS(rusty_select_cpu, struct task_struct *p, s32 prev_cpu, !bpf_cpumask_empty(cast_mask(direct_greedy_cpumask))) { u32 dom_id = cpu_to_dom_id(prev_cpu); dom_ptr domc; - struct lb_domain *dval; + struct lb_domain *lb_domain; struct bpf_cpumask *tmp_direct_greedy, *node_mask; /* @@ -1043,7 +1031,7 @@ s32 BPF_STRUCT_OPS(rusty_select_cpu, struct task_struct *p, s32 prev_cpu, else if (!(domc = lookup_dom_ctx(dom_id))) goto enoent; - if (!(dval = sdt_dom_val(domc->id))) { + if (!(lb_domain = lb_domain_get(domc->id))) { scx_bpf_error("Failed to lookup domain map value"); goto enoent; } @@ -1062,7 +1050,7 @@ s32 BPF_STRUCT_OPS(rusty_select_cpu, struct task_struct *p, s32 prev_cpu, * working set may end up spanning multiple NUMA nodes. */ if (!direct_greedy_numa && domc) { - node_mask = dval->node_cpumask; + node_mask = lb_domain->node_cpumask; if (!node_mask) { scx_bpf_error("Failed to lookup node mask"); goto enoent; @@ -1081,8 +1069,8 @@ s32 BPF_STRUCT_OPS(rusty_select_cpu, struct task_struct *p, s32 prev_cpu, /* Try to find an idle core in the previous and then any domain */ if (has_idle_cores) { - if (domc && dval->direct_greedy_cpumask) { - cpu = scx_bpf_pick_idle_cpu(cast_mask(dval->direct_greedy_cpumask), + if (domc && lb_domain->direct_greedy_cpumask) { + cpu = scx_bpf_pick_idle_cpu(cast_mask(lb_domain->direct_greedy_cpumask), SCX_PICK_IDLE_CORE); if (cpu >= 0) { stat_add(RUSTY_STAT_DIRECT_GREEDY, 1); @@ -1103,8 +1091,8 @@ s32 BPF_STRUCT_OPS(rusty_select_cpu, struct task_struct *p, s32 prev_cpu, /* * No idle core. Is there any idle CPU? */ - if (domc && dval->direct_greedy_cpumask) { - cpu = scx_bpf_pick_idle_cpu(cast_mask(dval->direct_greedy_cpumask), 0); + if (domc && lb_domain->direct_greedy_cpumask) { + cpu = scx_bpf_pick_idle_cpu(cast_mask(lb_domain->direct_greedy_cpumask), 0); if (cpu >= 0) { stat_add(RUSTY_STAT_DIRECT_GREEDY, 1); goto direct; @@ -1244,14 +1232,14 @@ void BPF_STRUCT_OPS(rusty_enqueue, struct task_struct *p __arg_trusted, u64 enq_ static bool cpumask_intersects_domain(const struct cpumask *cpumask, u32 dom_id) { - struct lb_domain *dval; + struct lb_domain *lb_domain; struct bpf_cpumask *dmask; - dval = sdt_dom_val(dom_id); - if (!dval) + lb_domain = lb_domain_get(dom_id); + if (!lb_domain) return false; - dmask = dval->cpumask; + dmask = lb_domain->cpumask; if (!dmask) return false; @@ -1769,7 +1757,7 @@ static s32 create_dom(u32 dom_id) dom_ptr domc; struct node_ctx *nodec; struct bpf_cpumask *dom_mask, *node_mask, *all_mask; - struct lb_domain *dval; + struct lb_domain *lb_domain; u32 cpu, node_id; s32 ret; @@ -1780,17 +1768,17 @@ static s32 create_dom(u32 dom_id) node_id = dom_node_id(dom_id); - domc = sdt_dom_alloc(dom_id); + domc = lb_domain_alloc(dom_id); if (!domc) return -ENOMEM; - doms[dom_id] = domc; - cast_user(doms[dom_id]); + dom_ctxs[dom_id] = domc; + cast_user(dom_ctxs[dom_id]); - dval = sdt_dom_val(dom_id); - if (!dval) { + lb_domain = lb_domain_get(dom_id); + if (!lb_domain) { scx_bpf_error("could not retrieve dom%d data\n", dom_id); - sdt_dom_free(domc); + lb_domain_free(domc); return -EINVAL; } @@ -1802,16 +1790,16 @@ static s32 create_dom(u32 dom_id) domc->id = dom_id; - ret = create_save_cpumask(&dval->cpumask); + ret = create_save_cpumask(&lb_domain->cpumask); if (ret) return ret; - bpf_printk("Created domain %d (%p)", dom_id, &dval->cpumask); - if (!dval->cpumask) + bpf_printk("Created domain %d (%p)", dom_id, &lb_domain->cpumask); + if (!lb_domain->cpumask) scx_bpf_error("NULL"); bpf_rcu_read_lock(); - dom_mask = dval->cpumask; + dom_mask = lb_domain->cpumask; all_mask = all_cpumask; if (!dom_mask || !all_mask) { bpf_rcu_read_unlock(); @@ -1838,7 +1826,7 @@ static s32 create_dom(u32 dom_id) if (ret) return ret; - ret = create_save_cpumask(&dval->direct_greedy_cpumask); + ret = create_save_cpumask(&lb_domain->direct_greedy_cpumask); if (ret) return ret; @@ -1848,13 +1836,13 @@ static s32 create_dom(u32 dom_id) scx_bpf_error("No node%u", node_id); return -ENOENT; } - ret = create_save_cpumask(&dval->node_cpumask); + ret = create_save_cpumask(&lb_domain->node_cpumask); if (ret) return ret; bpf_rcu_read_lock(); node_mask = nodec->cpumask; - dom_mask = dval->node_cpumask; + dom_mask = lb_domain->node_cpumask; if (!node_mask || !dom_mask) { bpf_rcu_read_unlock(); scx_bpf_error("cpumask lookup failed"); @@ -1877,20 +1865,21 @@ static s32 initialize_cpu(s32 cpu) pcpuc->dom_rr_cur = cpu; bpf_for(i, 0, nr_doms) { bool in_dom; - struct lb_domain *dval; + struct lb_domain *lb_domain; - dval = sdt_dom_val(i); - if (!dval) + lb_domain = lb_domain_get(i); + if (!lb_domain) return -ENOENT; bpf_rcu_read_lock(); - if (!dval->cpumask) { + if (!lb_domain->cpumask) { bpf_rcu_read_unlock(); - scx_bpf_error("Failed to lookup dom node %d cpumask %p", i, &dval->cpumask); + scx_bpf_error("Failed to lookup dom node %d cpumask %p", + i, &lb_domain->cpumask); return -ENOENT; } - in_dom = bpf_cpumask_test_cpu(cpu, cast_mask(dval->cpumask)); + in_dom = bpf_cpumask_test_cpu(cpu, cast_mask(lb_domain->cpumask)); bpf_rcu_read_unlock(); if (in_dom) { pcpuc->dom_id = i; @@ -1913,7 +1902,7 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(rusty_init) if (ret) return ret; - ret = sdt_dom_init(); + ret = lb_domain_init(); if (ret) return ret; diff --git a/scheds/rust/scx_rusty/src/load_balance.rs b/scheds/rust/scx_rusty/src/load_balance.rs index 6d0766d2a..cc47e234b 100644 --- a/scheds/rust/scx_rusty/src/load_balance.rs +++ b/scheds/rust/scx_rusty/src/load_balance.rs @@ -627,7 +627,7 @@ impl<'a, 'b> LoadBalancer<'a, 'b> { // Read active_tasks and update read_idx and gen. const MAX_TPTRS: u64 = bpf_intf::consts_MAX_DOM_ACTIVE_TPTRS as u64; let dom_ctx = - unsafe { &mut *(self.skel.maps.bss_data.doms[dom.id] as *mut bpf_intf::dom_ctx) }; + unsafe { &mut *(self.skel.maps.bss_data.dom_ctxs[dom.id] as *mut bpf_intf::dom_ctx) }; let active_tasks = &mut dom_ctx.active_tasks; let (mut ridx, widx) = (active_tasks.read_idx, active_tasks.write_idx); diff --git a/scheds/rust/scx_rusty/src/main.rs b/scheds/rust/scx_rusty/src/main.rs index 4de8de21c..616be225e 100644 --- a/scheds/rust/scx_rusty/src/main.rs +++ b/scheds/rust/scx_rusty/src/main.rs @@ -455,7 +455,7 @@ impl<'a> Scheduler<'a> { let mut ctx = dom.ctx.lock().unwrap(); - *ctx = Some(skel.maps.bss_data.doms[id] as *mut bpf_intf::dom_ctx); + *ctx = Some(skel.maps.bss_data.dom_ctxs[id] as *mut bpf_intf::dom_ctx); } info!("Rusty scheduler started! Run `scx_rusty --monitor` for metrics.");