diff --git a/profiles/workload/tpch_decima/100g/task_duration_1.npy b/profiles/workload/tpch_decima/100g/task_duration_1.npy new file mode 100644 index 00000000..13c827d3 Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_1.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_10.npy b/profiles/workload/tpch_decima/100g/task_duration_10.npy new file mode 100644 index 00000000..67ae23c1 Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_10.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_11.npy b/profiles/workload/tpch_decima/100g/task_duration_11.npy new file mode 100644 index 00000000..7e7071f5 Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_11.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_12.npy b/profiles/workload/tpch_decima/100g/task_duration_12.npy new file mode 100644 index 00000000..8b4f6a63 Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_12.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_13.npy b/profiles/workload/tpch_decima/100g/task_duration_13.npy new file mode 100644 index 00000000..a297fbc2 Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_13.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_14.npy b/profiles/workload/tpch_decima/100g/task_duration_14.npy new file mode 100644 index 00000000..282aab8c Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_14.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_15.npy b/profiles/workload/tpch_decima/100g/task_duration_15.npy new file mode 100644 index 00000000..9dd30f4e Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_15.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_16.npy b/profiles/workload/tpch_decima/100g/task_duration_16.npy new file mode 100644 index 00000000..0b83c535 Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_16.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_17.npy b/profiles/workload/tpch_decima/100g/task_duration_17.npy new file mode 100644 index 00000000..b6ebb609 Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_17.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_18.npy b/profiles/workload/tpch_decima/100g/task_duration_18.npy new file mode 100644 index 00000000..1217fd21 Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_18.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_19.npy b/profiles/workload/tpch_decima/100g/task_duration_19.npy new file mode 100644 index 00000000..d857d3d2 Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_19.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_2.npy b/profiles/workload/tpch_decima/100g/task_duration_2.npy new file mode 100644 index 00000000..ee579a86 Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_2.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_20.npy b/profiles/workload/tpch_decima/100g/task_duration_20.npy new file mode 100644 index 00000000..85c197f3 Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_20.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_21.npy b/profiles/workload/tpch_decima/100g/task_duration_21.npy new file mode 100644 index 00000000..819011a1 Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_21.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_22.npy b/profiles/workload/tpch_decima/100g/task_duration_22.npy new file mode 100644 index 00000000..2eb214e9 Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_22.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_3.npy b/profiles/workload/tpch_decima/100g/task_duration_3.npy new file mode 100644 index 00000000..55a0456b Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_3.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_4.npy b/profiles/workload/tpch_decima/100g/task_duration_4.npy new file mode 100644 index 00000000..e6d65f00 Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_4.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_5.npy b/profiles/workload/tpch_decima/100g/task_duration_5.npy new file mode 100644 index 00000000..ebf9bc26 Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_5.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_6.npy b/profiles/workload/tpch_decima/100g/task_duration_6.npy new file mode 100644 index 00000000..cdae9221 Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_6.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_7.npy b/profiles/workload/tpch_decima/100g/task_duration_7.npy new file mode 100644 index 00000000..b5a8e830 Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_7.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_8.npy b/profiles/workload/tpch_decima/100g/task_duration_8.npy new file mode 100644 index 00000000..715b7a8e Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_8.npy differ diff --git a/profiles/workload/tpch_decima/100g/task_duration_9.npy b/profiles/workload/tpch_decima/100g/task_duration_9.npy new file mode 100644 index 00000000..967368ce Binary files /dev/null and b/profiles/workload/tpch_decima/100g/task_duration_9.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_1.npy b/profiles/workload/tpch_decima/10g/task_duration_1.npy new file mode 100644 index 00000000..9c2fe6e5 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_1.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_10.npy b/profiles/workload/tpch_decima/10g/task_duration_10.npy new file mode 100644 index 00000000..635270de Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_10.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_11.npy b/profiles/workload/tpch_decima/10g/task_duration_11.npy new file mode 100644 index 00000000..23188129 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_11.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_12.npy b/profiles/workload/tpch_decima/10g/task_duration_12.npy new file mode 100644 index 00000000..04904c12 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_12.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_13.npy b/profiles/workload/tpch_decima/10g/task_duration_13.npy new file mode 100644 index 00000000..1adfdd29 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_13.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_14.npy b/profiles/workload/tpch_decima/10g/task_duration_14.npy new file mode 100644 index 00000000..b4c01c45 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_14.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_15.npy b/profiles/workload/tpch_decima/10g/task_duration_15.npy new file mode 100644 index 00000000..f4e63b66 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_15.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_16.npy b/profiles/workload/tpch_decima/10g/task_duration_16.npy new file mode 100644 index 00000000..1edabf09 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_16.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_17.npy b/profiles/workload/tpch_decima/10g/task_duration_17.npy new file mode 100644 index 00000000..4701d2e8 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_17.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_18.npy b/profiles/workload/tpch_decima/10g/task_duration_18.npy new file mode 100644 index 00000000..c28a71a9 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_18.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_19.npy b/profiles/workload/tpch_decima/10g/task_duration_19.npy new file mode 100644 index 00000000..687db73a Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_19.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_2.npy b/profiles/workload/tpch_decima/10g/task_duration_2.npy new file mode 100644 index 00000000..4c6846c1 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_2.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_20.npy b/profiles/workload/tpch_decima/10g/task_duration_20.npy new file mode 100644 index 00000000..1d197a31 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_20.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_21.npy b/profiles/workload/tpch_decima/10g/task_duration_21.npy new file mode 100644 index 00000000..be33a874 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_21.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_22.npy b/profiles/workload/tpch_decima/10g/task_duration_22.npy new file mode 100644 index 00000000..4145d282 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_22.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_3.npy b/profiles/workload/tpch_decima/10g/task_duration_3.npy new file mode 100644 index 00000000..b6bb7f94 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_3.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_4.npy b/profiles/workload/tpch_decima/10g/task_duration_4.npy new file mode 100644 index 00000000..3d56b6f7 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_4.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_5.npy b/profiles/workload/tpch_decima/10g/task_duration_5.npy new file mode 100644 index 00000000..96acfde4 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_5.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_6.npy b/profiles/workload/tpch_decima/10g/task_duration_6.npy new file mode 100644 index 00000000..9140ee96 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_6.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_7.npy b/profiles/workload/tpch_decima/10g/task_duration_7.npy new file mode 100644 index 00000000..91f7d146 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_7.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_8.npy b/profiles/workload/tpch_decima/10g/task_duration_8.npy new file mode 100644 index 00000000..d0790ad4 Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_8.npy differ diff --git a/profiles/workload/tpch_decima/10g/task_duration_9.npy b/profiles/workload/tpch_decima/10g/task_duration_9.npy new file mode 100644 index 00000000..c19f842e Binary files /dev/null and b/profiles/workload/tpch_decima/10g/task_duration_9.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_1.npy b/profiles/workload/tpch_decima/20g/task_duration_1.npy new file mode 100644 index 00000000..8b850813 Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_1.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_10.npy b/profiles/workload/tpch_decima/20g/task_duration_10.npy new file mode 100644 index 00000000..0ee97650 Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_10.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_11.npy b/profiles/workload/tpch_decima/20g/task_duration_11.npy new file mode 100644 index 00000000..caea0e48 Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_11.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_12.npy b/profiles/workload/tpch_decima/20g/task_duration_12.npy new file mode 100644 index 00000000..63027a14 Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_12.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_13.npy b/profiles/workload/tpch_decima/20g/task_duration_13.npy new file mode 100644 index 00000000..2b819427 Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_13.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_14.npy b/profiles/workload/tpch_decima/20g/task_duration_14.npy new file mode 100644 index 00000000..331843e3 Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_14.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_15.npy b/profiles/workload/tpch_decima/20g/task_duration_15.npy new file mode 100644 index 00000000..c9cc5ad6 Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_15.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_16.npy b/profiles/workload/tpch_decima/20g/task_duration_16.npy new file mode 100644 index 00000000..806953b1 Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_16.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_17.npy b/profiles/workload/tpch_decima/20g/task_duration_17.npy new file mode 100644 index 00000000..b9807336 Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_17.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_18.npy b/profiles/workload/tpch_decima/20g/task_duration_18.npy new file mode 100644 index 00000000..407a3b3b Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_18.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_19.npy b/profiles/workload/tpch_decima/20g/task_duration_19.npy new file mode 100644 index 00000000..08027102 Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_19.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_2.npy b/profiles/workload/tpch_decima/20g/task_duration_2.npy new file mode 100644 index 00000000..47862236 Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_2.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_20.npy b/profiles/workload/tpch_decima/20g/task_duration_20.npy new file mode 100644 index 00000000..0a592f34 Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_20.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_21.npy b/profiles/workload/tpch_decima/20g/task_duration_21.npy new file mode 100644 index 00000000..76371932 Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_21.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_22.npy b/profiles/workload/tpch_decima/20g/task_duration_22.npy new file mode 100644 index 00000000..60e2f92e Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_22.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_3.npy b/profiles/workload/tpch_decima/20g/task_duration_3.npy new file mode 100644 index 00000000..255a0a26 Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_3.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_4.npy b/profiles/workload/tpch_decima/20g/task_duration_4.npy new file mode 100644 index 00000000..f8c2ac6c Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_4.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_5.npy b/profiles/workload/tpch_decima/20g/task_duration_5.npy new file mode 100644 index 00000000..15610838 Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_5.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_6.npy b/profiles/workload/tpch_decima/20g/task_duration_6.npy new file mode 100644 index 00000000..825b1547 Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_6.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_7.npy b/profiles/workload/tpch_decima/20g/task_duration_7.npy new file mode 100644 index 00000000..9c60f636 Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_7.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_8.npy b/profiles/workload/tpch_decima/20g/task_duration_8.npy new file mode 100644 index 00000000..1f7158f2 Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_8.npy differ diff --git a/profiles/workload/tpch_decima/20g/task_duration_9.npy b/profiles/workload/tpch_decima/20g/task_duration_9.npy new file mode 100644 index 00000000..1c9b6e2e Binary files /dev/null and b/profiles/workload/tpch_decima/20g/task_duration_9.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_1.npy b/profiles/workload/tpch_decima/2g/task_duration_1.npy new file mode 100644 index 00000000..ed5df29f Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_1.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_10.npy b/profiles/workload/tpch_decima/2g/task_duration_10.npy new file mode 100644 index 00000000..4c1f827f Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_10.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_11.npy b/profiles/workload/tpch_decima/2g/task_duration_11.npy new file mode 100644 index 00000000..83e141b5 Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_11.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_12.npy b/profiles/workload/tpch_decima/2g/task_duration_12.npy new file mode 100644 index 00000000..0839a7ca Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_12.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_13.npy b/profiles/workload/tpch_decima/2g/task_duration_13.npy new file mode 100644 index 00000000..dc7f48fe Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_13.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_14.npy b/profiles/workload/tpch_decima/2g/task_duration_14.npy new file mode 100644 index 00000000..a2ce73f8 Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_14.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_15.npy b/profiles/workload/tpch_decima/2g/task_duration_15.npy new file mode 100644 index 00000000..cefcc1b6 Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_15.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_16.npy b/profiles/workload/tpch_decima/2g/task_duration_16.npy new file mode 100644 index 00000000..0a640be7 Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_16.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_17.npy b/profiles/workload/tpch_decima/2g/task_duration_17.npy new file mode 100644 index 00000000..4e0df4bd Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_17.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_18.npy b/profiles/workload/tpch_decima/2g/task_duration_18.npy new file mode 100644 index 00000000..101ea2f5 Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_18.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_19.npy b/profiles/workload/tpch_decima/2g/task_duration_19.npy new file mode 100644 index 00000000..631dc504 Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_19.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_2.npy b/profiles/workload/tpch_decima/2g/task_duration_2.npy new file mode 100644 index 00000000..24d5cf3e Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_2.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_20.npy b/profiles/workload/tpch_decima/2g/task_duration_20.npy new file mode 100644 index 00000000..71ffdc47 Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_20.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_21.npy b/profiles/workload/tpch_decima/2g/task_duration_21.npy new file mode 100644 index 00000000..60316304 Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_21.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_22.npy b/profiles/workload/tpch_decima/2g/task_duration_22.npy new file mode 100644 index 00000000..be91b5b5 Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_22.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_3.npy b/profiles/workload/tpch_decima/2g/task_duration_3.npy new file mode 100644 index 00000000..880eadb2 Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_3.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_4.npy b/profiles/workload/tpch_decima/2g/task_duration_4.npy new file mode 100644 index 00000000..386f7264 Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_4.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_5.npy b/profiles/workload/tpch_decima/2g/task_duration_5.npy new file mode 100644 index 00000000..4c0c3a24 Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_5.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_6.npy b/profiles/workload/tpch_decima/2g/task_duration_6.npy new file mode 100644 index 00000000..6b05d924 Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_6.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_7.npy b/profiles/workload/tpch_decima/2g/task_duration_7.npy new file mode 100644 index 00000000..40e5a6d9 Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_7.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_8.npy b/profiles/workload/tpch_decima/2g/task_duration_8.npy new file mode 100644 index 00000000..c456818a Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_8.npy differ diff --git a/profiles/workload/tpch_decima/2g/task_duration_9.npy b/profiles/workload/tpch_decima/2g/task_duration_9.npy new file mode 100644 index 00000000..9b8eb947 Binary files /dev/null and b/profiles/workload/tpch_decima/2g/task_duration_9.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_1.npy b/profiles/workload/tpch_decima/50g/task_duration_1.npy new file mode 100644 index 00000000..8c85a5c6 Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_1.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_10.npy b/profiles/workload/tpch_decima/50g/task_duration_10.npy new file mode 100644 index 00000000..e7b64577 Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_10.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_11.npy b/profiles/workload/tpch_decima/50g/task_duration_11.npy new file mode 100644 index 00000000..dd554924 Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_11.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_12.npy b/profiles/workload/tpch_decima/50g/task_duration_12.npy new file mode 100644 index 00000000..7cbd3a95 Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_12.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_13.npy b/profiles/workload/tpch_decima/50g/task_duration_13.npy new file mode 100644 index 00000000..dcde0f46 Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_13.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_14.npy b/profiles/workload/tpch_decima/50g/task_duration_14.npy new file mode 100644 index 00000000..87408ebf Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_14.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_15.npy b/profiles/workload/tpch_decima/50g/task_duration_15.npy new file mode 100644 index 00000000..2ad5d285 Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_15.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_16.npy b/profiles/workload/tpch_decima/50g/task_duration_16.npy new file mode 100644 index 00000000..72ded6bc Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_16.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_17.npy b/profiles/workload/tpch_decima/50g/task_duration_17.npy new file mode 100644 index 00000000..573768cc Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_17.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_18.npy b/profiles/workload/tpch_decima/50g/task_duration_18.npy new file mode 100644 index 00000000..f569940e Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_18.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_19.npy b/profiles/workload/tpch_decima/50g/task_duration_19.npy new file mode 100644 index 00000000..e7bf71fb Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_19.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_2.npy b/profiles/workload/tpch_decima/50g/task_duration_2.npy new file mode 100644 index 00000000..ced8fbae Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_2.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_20.npy b/profiles/workload/tpch_decima/50g/task_duration_20.npy new file mode 100644 index 00000000..eb033e80 Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_20.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_21.npy b/profiles/workload/tpch_decima/50g/task_duration_21.npy new file mode 100644 index 00000000..d5756fb8 Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_21.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_22.npy b/profiles/workload/tpch_decima/50g/task_duration_22.npy new file mode 100644 index 00000000..9f051af1 Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_22.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_3.npy b/profiles/workload/tpch_decima/50g/task_duration_3.npy new file mode 100644 index 00000000..e6525f30 Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_3.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_4.npy b/profiles/workload/tpch_decima/50g/task_duration_4.npy new file mode 100644 index 00000000..8f942a9b Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_4.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_5.npy b/profiles/workload/tpch_decima/50g/task_duration_5.npy new file mode 100644 index 00000000..5d09108a Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_5.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_6.npy b/profiles/workload/tpch_decima/50g/task_duration_6.npy new file mode 100644 index 00000000..f513ffa1 Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_6.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_7.npy b/profiles/workload/tpch_decima/50g/task_duration_7.npy new file mode 100644 index 00000000..2ca71cd2 Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_7.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_8.npy b/profiles/workload/tpch_decima/50g/task_duration_8.npy new file mode 100644 index 00000000..829de28b Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_8.npy differ diff --git a/profiles/workload/tpch_decima/50g/task_duration_9.npy b/profiles/workload/tpch_decima/50g/task_duration_9.npy new file mode 100644 index 00000000..98f3ab67 Binary files /dev/null and b/profiles/workload/tpch_decima/50g/task_duration_9.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_1.npy b/profiles/workload/tpch_decima/5g/task_duration_1.npy new file mode 100644 index 00000000..32667cbd Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_1.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_10.npy b/profiles/workload/tpch_decima/5g/task_duration_10.npy new file mode 100644 index 00000000..cee25165 Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_10.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_11.npy b/profiles/workload/tpch_decima/5g/task_duration_11.npy new file mode 100644 index 00000000..8c8e13fb Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_11.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_12.npy b/profiles/workload/tpch_decima/5g/task_duration_12.npy new file mode 100644 index 00000000..18552979 Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_12.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_13.npy b/profiles/workload/tpch_decima/5g/task_duration_13.npy new file mode 100644 index 00000000..f3e6e681 Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_13.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_14.npy b/profiles/workload/tpch_decima/5g/task_duration_14.npy new file mode 100644 index 00000000..bc6f1ec4 Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_14.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_15.npy b/profiles/workload/tpch_decima/5g/task_duration_15.npy new file mode 100644 index 00000000..84c47a1e Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_15.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_16.npy b/profiles/workload/tpch_decima/5g/task_duration_16.npy new file mode 100644 index 00000000..7f3501a1 Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_16.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_17.npy b/profiles/workload/tpch_decima/5g/task_duration_17.npy new file mode 100644 index 00000000..5e31f4a7 Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_17.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_18.npy b/profiles/workload/tpch_decima/5g/task_duration_18.npy new file mode 100644 index 00000000..c1b11923 Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_18.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_19.npy b/profiles/workload/tpch_decima/5g/task_duration_19.npy new file mode 100644 index 00000000..2df6e0e2 Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_19.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_2.npy b/profiles/workload/tpch_decima/5g/task_duration_2.npy new file mode 100644 index 00000000..d06771a8 Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_2.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_20.npy b/profiles/workload/tpch_decima/5g/task_duration_20.npy new file mode 100644 index 00000000..1fc91cb4 Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_20.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_21.npy b/profiles/workload/tpch_decima/5g/task_duration_21.npy new file mode 100644 index 00000000..aa6006a8 Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_21.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_22.npy b/profiles/workload/tpch_decima/5g/task_duration_22.npy new file mode 100644 index 00000000..d0326833 Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_22.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_3.npy b/profiles/workload/tpch_decima/5g/task_duration_3.npy new file mode 100644 index 00000000..f4f4d287 Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_3.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_4.npy b/profiles/workload/tpch_decima/5g/task_duration_4.npy new file mode 100644 index 00000000..1a85e60f Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_4.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_5.npy b/profiles/workload/tpch_decima/5g/task_duration_5.npy new file mode 100644 index 00000000..078ef483 Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_5.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_6.npy b/profiles/workload/tpch_decima/5g/task_duration_6.npy new file mode 100644 index 00000000..764030c0 Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_6.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_7.npy b/profiles/workload/tpch_decima/5g/task_duration_7.npy new file mode 100644 index 00000000..24da1289 Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_7.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_8.npy b/profiles/workload/tpch_decima/5g/task_duration_8.npy new file mode 100644 index 00000000..632d569d Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_8.npy differ diff --git a/profiles/workload/tpch_decima/5g/task_duration_9.npy b/profiles/workload/tpch_decima/5g/task_duration_9.npy new file mode 100644 index 00000000..d1b7638e Binary files /dev/null and b/profiles/workload/tpch_decima/5g/task_duration_9.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_1.npy b/profiles/workload/tpch_decima/80g/task_duration_1.npy new file mode 100644 index 00000000..a0d13c74 Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_1.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_10.npy b/profiles/workload/tpch_decima/80g/task_duration_10.npy new file mode 100644 index 00000000..8086667d Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_10.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_11.npy b/profiles/workload/tpch_decima/80g/task_duration_11.npy new file mode 100644 index 00000000..19ee4e16 Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_11.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_12.npy b/profiles/workload/tpch_decima/80g/task_duration_12.npy new file mode 100644 index 00000000..b814501f Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_12.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_13.npy b/profiles/workload/tpch_decima/80g/task_duration_13.npy new file mode 100644 index 00000000..99decb9b Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_13.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_14.npy b/profiles/workload/tpch_decima/80g/task_duration_14.npy new file mode 100644 index 00000000..04bf9478 Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_14.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_15.npy b/profiles/workload/tpch_decima/80g/task_duration_15.npy new file mode 100644 index 00000000..2044f095 Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_15.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_16.npy b/profiles/workload/tpch_decima/80g/task_duration_16.npy new file mode 100644 index 00000000..54d227ad Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_16.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_17.npy b/profiles/workload/tpch_decima/80g/task_duration_17.npy new file mode 100644 index 00000000..c66ca468 Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_17.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_18.npy b/profiles/workload/tpch_decima/80g/task_duration_18.npy new file mode 100644 index 00000000..2d4611a8 Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_18.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_19.npy b/profiles/workload/tpch_decima/80g/task_duration_19.npy new file mode 100644 index 00000000..ba1095f8 Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_19.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_2.npy b/profiles/workload/tpch_decima/80g/task_duration_2.npy new file mode 100644 index 00000000..13edf680 Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_2.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_20.npy b/profiles/workload/tpch_decima/80g/task_duration_20.npy new file mode 100644 index 00000000..00de4cdb Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_20.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_21.npy b/profiles/workload/tpch_decima/80g/task_duration_21.npy new file mode 100644 index 00000000..775765c9 Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_21.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_22.npy b/profiles/workload/tpch_decima/80g/task_duration_22.npy new file mode 100644 index 00000000..3b82fa09 Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_22.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_3.npy b/profiles/workload/tpch_decima/80g/task_duration_3.npy new file mode 100644 index 00000000..d8b1477c Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_3.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_4.npy b/profiles/workload/tpch_decima/80g/task_duration_4.npy new file mode 100644 index 00000000..bbd5bcb3 Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_4.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_5.npy b/profiles/workload/tpch_decima/80g/task_duration_5.npy new file mode 100644 index 00000000..c42c5a16 Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_5.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_6.npy b/profiles/workload/tpch_decima/80g/task_duration_6.npy new file mode 100644 index 00000000..e6eeb0d1 Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_6.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_7.npy b/profiles/workload/tpch_decima/80g/task_duration_7.npy new file mode 100644 index 00000000..aca38f9b Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_7.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_8.npy b/profiles/workload/tpch_decima/80g/task_duration_8.npy new file mode 100644 index 00000000..084089ae Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_8.npy differ diff --git a/profiles/workload/tpch_decima/80g/task_duration_9.npy b/profiles/workload/tpch_decima/80g/task_duration_9.npy new file mode 100644 index 00000000..4365eb7d Binary files /dev/null and b/profiles/workload/tpch_decima/80g/task_duration_9.npy differ diff --git a/profiles/workload/tpch_decima/query_dag.json b/profiles/workload/tpch_decima/query_dag.json new file mode 100644 index 00000000..ff20fbde --- /dev/null +++ b/profiles/workload/tpch_decima/query_dag.json @@ -0,0 +1,27 @@ +{ + "workload_type": "tpch", + "query_number": { + "1": "[(0, 1), (1, 2)]", + "2": "[(0, 2), (0, 9), (1, 2), (2, 4), (3, 4), (4, 7), (5, 7), (5, 13), (6, 15), (7, 15), (8, 9), (9, 11), (10, 11), (11, 13), (12, 14), (13, 14), (14, 16), (15, 16)]", + "3": "[(0, 2), (1, 2), (2, 4), (3, 4)]", + "4": "[(0, 2), (1, 2), (2, 3), (3, 4)]", + "5": "[(0, 2), (1, 2), (2, 4), (3, 4), (4, 6), (5, 6), (6, 8), (7, 8), (8, 10), (9, 10), (10, 11), (11, 12)]", + "6": "[(0, 1)]", + "7": "[(0, 2), (0, 5), (1, 2), (2, 8), (3, 8), (4, 5), (5, 7), (6, 7), (7, 9), (8, 9), (9, 10), (10, 11)]", + "8": "[(0, 5), (1, 5), (2, 4), (3, 4), (4, 13), (5, 13), (6, 8), (7, 8), (8, 10), (9, 10), (10, 12), (11, 12), (12, 14), (13, 14), (14, 15), (15, 16)]", + "9": "[(0, 5), (1, 5), (2, 4), (3, 4), (4, 6), (5, 6), (6, 8), (7, 8), (8, 10), (9, 10), (10, 11), (11, 12)]", + "10": "[(0, 2), (1, 2), (2, 4), (3, 4), (4, 6), (5, 6), (6, 7)]", + "11": "", + "12": "[(0, 2), (1, 2), (2, 3), (3, 4)]", + "13": "[(0, 2), (1, 2), (2, 3), (3, 4), (4, 5)]", + "14": "[(0, 2), (1, 2)]", + "15": "", + "16": "[(0, 2), (1, 2), (2, 4), (3, 4), (4, 5), (5, 6), (6, 7)]", + "17": "", + "18": "", + "19": "[(0, 2), (1, 2)]", + "20": "", + "21": "", + "22": "" + } +} \ No newline at end of file diff --git a/rpc/service.py b/rpc/service.py index 2d01b99f..8934d80c 100644 --- a/rpc/service.py +++ b/rpc/service.py @@ -16,6 +16,7 @@ import erdos_scheduler_pb2_grpc import grpc from absl import app, flags +from tpch_utils import get_all_stage_info_for_query, verify_and_relable_tpch_app_graph from schedulers import EDFScheduler, FIFOScheduler from utils import EventTime, setup_logging @@ -367,10 +368,62 @@ async def RegisterTaskGraph(self, request, context): num_executors=0, ) + self._logger.info( + "Attempting to register application ID %s with name %s", + request.id, + request.name, + ) + # Check if query is from TPC-H workload. + # If yes, retrieve profiled slots and runtime info. If no, use default values + is_tpch_query = False + tpch_query_all_stage_info = None + if request.name.startswith("TPCH_"): + is_tpch_query = True + # retrieve tasks-per-stage and runtime info based on query number + tpch_query_num = request.name.split("TPCH_Q", 1)[1] + tpch_query_all_stage_info = get_all_stage_info_for_query(tpch_query_num) + same_structure, stage_id_mapping = verify_and_relable_tpch_app_graph( + query_num=tpch_query_num, dependencies=request.dependencies + ) + + # return failure message if not tpch app isnt of same DAG structure + if not same_structure: + self._logger.warning( + "TPCH application with ID %s and name %s couldn't be registered." + "DAG structure mismatch!", + request.id, + request.name, + ) + return erdos_scheduler_pb2.RegisterTaskGraphResponse( + success=False, + message=f"TPCH application ID {request.id} with name {request.name}" + f" couldn't be registered. DAG structure mismatch!", + num_executors=0, + ) + # Construct all the Tasks for the TaskGraph. task_ids_to_task: Mapping[int, Task] = {} + default_resource = Resources( + resource_vector={Resource(name="Slot_CPU", _id="any"): 30} + ) + default_runtime = EventTime(20, EventTime.Unit.US) + for task_dependency in request.dependencies: framework_task = task_dependency.key + if is_tpch_query: + mapped_stage_id = stage_id_mapping[framework_task.id] + task_slots = tpch_query_all_stage_info[mapped_stage_id]["num_tasks"] + task_runtime = tpch_query_all_stage_info[mapped_stage_id][ + "avg_task_duration" + ] + self._logger.info( + "Creating Task for given app TPCH stage: %s, mapped to " + "original stage id %s, with tasks: %s and avg runtime: %s", + framework_task.id, + mapped_stage_id, + task_slots, + task_runtime, + ) task_ids_to_task[framework_task.id] = Task( name=framework_task.name, task_graph=request.id, @@ -380,17 +433,24 @@ async def RegisterTaskGraph(self, request, context): name=f"WorkProfile_{framework_task.name}", execution_strategies=ExecutionStrategies( [ - # TODO (Sukrit): Find the actual resource requirements - # for the particular TaskGraph, along with the expected - # runtime and set it here. ExecutionStrategy( - resources=Resources( - resource_vector={ - Resource(name="Slot_CPU", _id="any"): 30 - } + resources=( + default_resource + if not is_tpch_query + else Resources( + resource_vector={ + Resource( + name="Slot_CPU", _id="any" + ): task_slots + } + ) ), batch_size=1, - runtime=EventTime(20, EventTime.Unit.US), + runtime=( + default_runtime + if not is_tpch_query + else EventTime(task_runtime, EventTime.Unit.US) + ), ) ] ), diff --git a/rpc/tpch_utils.py b/rpc/tpch_utils.py new file mode 100644 index 00000000..ebc4e3cd --- /dev/null +++ b/rpc/tpch_utils.py @@ -0,0 +1,170 @@ +# Code adopted from decima-sim to use their profiles of TPC-H queries + +import ast +import json +import os +from typing import Mapping, Sequence + +import networkx as nx +import numpy as np + +HOME_TPCH_DIR = "../profiles/workload/tpch_decima/" +TPCH_SUBDIR = "2g/" + + +class SetWithCount(object): + """ + allow duplication in set + """ + + def __init__(self): + self.set = {} + + def __contains__(self, item): + return item in self.set + + def add(self, item): + if item in self.set: + self.set[item] += 1 + else: + self.set[item] = 1 + + def clear(self): + self.set.clear() + + def remove(self, item): + self.set[item] -= 1 + if self.set[item] == 0: + del self.set[item] + + +def pre_process_task_duration(task_duration): + # remove fresh durations from first wave + clean_first_wave = {} + for e in task_duration["first_wave"]: + clean_first_wave[e] = [] + fresh_durations = SetWithCount() + # O(1) access + for d in task_duration["fresh_durations"][e]: + fresh_durations.add(d) + for d in task_duration["first_wave"][e]: + if d not in fresh_durations: + clean_first_wave[e].append(d) + else: + # prevent duplicated fresh duration blocking first wave + fresh_durations.remove(d) + + +def get_all_stage_info_for_query(query_num): + task_durations = np.load( + os.path.join( + HOME_TPCH_DIR, TPCH_SUBDIR, "task_duration_" + str(query_num) + ".npy" + ), + allow_pickle=True, + ).item() + + num_nodes = len(task_durations) + + stage_info = {} + + for n in range(num_nodes): + task_duration = task_durations[n] + e = next(iter(task_duration["first_wave"])) + # NOTE: somehow only picks the first element {2: [n_tasks_in_ms]} + + num_tasks = len(task_duration["first_wave"][e]) + len( + task_duration["rest_wave"][e] + ) + + # remove fresh duration from first wave duration + # drag nearest neighbor first wave duration to empty spots + pre_process_task_duration(task_duration) + rough_duration = np.mean( + [i for t in task_duration["first_wave"].values() for i in t] + + [i for t in task_duration["rest_wave"].values() for i in t] + + [i for t in task_duration["fresh_durations"].values() for i in t] + ) + + curr_stage = { + "stage_id": n, + "num_tasks": num_tasks, + "avg_task_duration": round(rough_duration), + } + stage_info[n] = curr_stage + + return stage_info + + +def get_base_tpch_graph_structure(query_num): + # use query_num to read string from file + with open(os.path.join(HOME_TPCH_DIR, "query_dag.json")) as f: + tpch_query_json = json.load(f) + + # get query dependency from file + query_dependency = ast.literal_eval(tpch_query_json["query_number"][str(query_num)]) + + # convert job structure into a nx graph + base_tpch_graph = nx.DiGraph(query_dependency) + + # return the job nx graph for query + return base_tpch_graph + + +def get_graph_from_deps(dependencies): + # parse dependencies to get it in list of tuples + # construct the TaskGraph from the dependencies. + task_graph_structure: Mapping[int, Sequence[int]] = {} + for task_dependency in dependencies: + task_graph_structure[task_dependency.key.id] = [ + task_id for task_id in task_dependency.children_ids + ] + + # convert our TaskGraph into a nx friendly notation + all_edges_in_app = [] + for parent in task_graph_structure.keys(): + for child in task_graph_structure[parent]: + all_edges_in_app.append((parent, child)) + + # construct nx graph + given_app_tpch_graph = nx.DiGraph(all_edges_in_app) + + # return the graph + return given_app_tpch_graph + + +def are_structurally_same(graph1, graph2): + # Step 1: Check if both graphs have the same number of vertices + if len(graph1.nodes) != len(graph2.nodes): + print( + f"DAG structure mismatch! Graph1 has " + f"{graph1.nodes} while Graph2 has {graph2.nodes}" + ) + return False, None + + # Step 2: Check if there exists a bijection between the vertices + # of the two graphs such that their adjacency relationships match + for mapping in nx.isomorphism.GraphMatcher(graph1, graph2).isomorphisms_iter(): + # Check if the adjacency relationships match + if all(v in mapping for u, v in graph1.edges): + # graph structures match + # mapping is a dict {key=original-stage-id, val=app-stage-id} + # we reverse reversed mapping from app-stage-id to orig-stage-id + reversed_mapping = {v: k for k, v in mapping.items()} + + return True, reversed_mapping + + print("DAG structure mismatch! No mapping could be found") + return False, None + + +def verify_and_relable_tpch_app_graph(query_num, dependencies): + # get nx graphs from base tpch file and application dependencies + base_tpch_graph = get_base_tpch_graph_structure(query_num) + app_graph = get_graph_from_deps(dependencies) + + # verify that the graphs are isomorphic + # returns true if same_structure, along with stage_id_mapping wrt base tpch file + same_structure, stage_id_mapping = are_structurally_same(base_tpch_graph, app_graph) + + # return with stage_id_mapping back to assign correct runtime and num_executors + return same_structure, stage_id_mapping