timeout graph

dag_in_context/src/greedy_dag_extractor.rs

@@ -9,6 +9,7 @@ use std::{
     collections::{HashSet, VecDeque},
     f64::INFINITY,
     rc::Rc,
+    sync::Arc,
     time::{Duration, Instant},
 };
 use strum::IntoEnumIterator;
@@ -1904,7 +1905,7 @@ fn prune_egraph(
             // if the op is a ctx, replace it with a fresh DumC
             if is_ctx_operator(&node.op) {
                 let new_node = Node {
-                    op: format!("DumC{}", node.eclass),
+                    op: format!("DumC{}", nodeid),
                     children: vec![],
                     eclass: node.eclass.clone(),
                     cost: NotNan::new(0.).unwrap(),

dag_in_context/src/lib.rs

@@ -358,6 +358,7 @@ pub struct EggccConfig {
 pub struct EggccTimeStatistics {
     pub eggcc_extraction_time: Duration,
     pub eggcc_serialization_time: Duration,
+    // if ilp didn't time out, what portion of the time was spent in the extraction gym code
     pub ilp_test_time: Option<Duration>,
 }
 

infra/graphs.py

@@ -69,6 +69,9 @@ def get_eggcc_compile_time(data, benchmark_name):
 def get_eggcc_extraction_time(data, benchmark_name):
   return get_row(data, benchmark_name, 'llvm-eggcc-O0-O0').get('eggccExtractionTimeSecs')
 
+def get_ilp_test_time_seconds(data, benchmark_name):
+  return get_row(data, benchmark_name, 'eggcc-ILP-O0-O0').get('ilpTestTimeSeconds')
+
 def group_by_benchmark(profile):
   grouped_by_benchmark = {}
   for benchmark in profile:
@@ -78,6 +81,52 @@ def group_by_benchmark(profile):
     grouped_by_benchmark[benchmark_name].append(benchmark)
   return [grouped_by_benchmark[benchmark] for benchmark in grouped_by_benchmark]
 
+# a graph of how the ilp solver time changes
+# compared to the number of lines in the bril file
+# when the ilp solve time is null it timed out
+def make_ilp(json, output, benchmark_suite_folder):
+  ilp_timeout = profile.ilp_extraction_test_timeout()
+
+  num_lines = []
+  ilp_times  = []
+  is_ilp_timeout = []
+  eggcc_extract_time = []
+
+  benchmarks = dedup([b.get('benchmark') for b in json])
+
+  for benchmark in benchmarks:
+    extraction_time = get_eggcc_extraction_time(json, benchmark)
+    ilp_time = get_ilp_test_time_seconds(json, benchmark)
+
+    if ilp_time == None:
+      is_ilp_timeout.append(True)
+      ilp_time = ilp_timeout
+    else:
+      is_ilp_timeout.append(False)
+
+    num_lines.append(get_code_size(benchmark, benchmark_suite_folder))
+    eggcc_extract_time.append(extraction_time)
+    ilp_times.append(ilp_time)
+
+  # graph data
+  plt.figure(figsize=(10, 6))
+
+  # graph eggcc extraction times
+  plt.scatter(num_lines, eggcc_extract_time, label='EggCC Extraction Time', color='blue')
+
+  # graph ilp times, putting red x marks for timeouts
+  plt.scatter(num_lines, ilp_times, label='ILP Solver Time', color='orange')
+  plt.scatter([num_lines[i] for i in range(len(num_lines)) if is_ilp_timeout[i]], [ilp_times[i] for i in range(len(num_lines)) if is_ilp_timeout[i]], color='red', marker='x', label='ILP Solver Timeout')
+
+  plt.xlabel('Bril Number of Instructions')
+  plt.ylabel('Extraction Time')
+  plt.title('ILP Solver Time vs Code Size')
+  plt.savefig(output)
+
+
+
+
+
 def make_jitter(profile, upper_x_bound, output):
   # Prepare the data for the jitter plot
   # first y label is empty, underneath the first benchmark
@@ -352,18 +401,7 @@ def make_code_size_vs_compile_and_extraction_time(profile, compile_time_output,
 
 
 
-
-
-if __name__ == '__main__':
-  # parse two arguments: the output folder and the profile.json file
-  if len(sys.argv) != 4:
-      print("Usage: python graphs.py <output_folder> <profile.json> <benchmark_suite_folder>")
-      sys.exit(1)
-  output_folder = sys.argv[1]
-  graphs_folder = output_folder + '/graphs'
-  profile_file = sys.argv[2]
-  benchmark_suite_folder = sys.argv[3]
-
+def make_graphs(output_folder, graphs_folder, profile_file, benchmark_suite_folder):
   # Read profile.json from nightly/output/data/profile.json
   profile = []
   with open(profile_file) as f:
@@ -375,6 +413,8 @@ def make_code_size_vs_compile_and_extraction_time(profile, compile_time_output,
 
   make_jitter(profile, 4, f'{graphs_folder}/jitter_plot_max_4.png')
 
+  make_ilp(profile, f'{graphs_folder}/ilp_vs_lines.png', benchmark_suite_folder)
+
   for suite_path in benchmark_suites:
     suite = os.path.basename(suite_path)
     suite_benchmarks = benchmarks_in_folder(suite_path)
@@ -399,4 +439,11 @@ def make_code_size_vs_compile_and_extraction_time(profile, compile_time_output,
   with open(f'{output_folder}/graphs.json', 'w') as f:
     json.dump(graph_names, f)
 
-
+if __name__ == '__main__':
+  # parse two arguments: the output folder and the profile.json file
+  if len(sys.argv) != 4:
+      print("Usage: python graphs.py <output_folder> <profile.json> <benchmark_suite_folder>")
+      sys.exit(1)
+  make_graphs(sys.argv[1], sys.argv[1] + '/graphs', sys.argv[2], sys.argv[3])
+
+

infra/nightly-resources/chart.js

@@ -12,6 +12,7 @@ const COLORS = {
   "llvm-eggcc-ablation-O0-O0": "blue",
   "llvm-eggcc-ablation-O3-O0": "green",
   "llvm-eggcc-ablation-O3-O3": "orange",
+  "eggcc-ILP-O0-O0": "red",
 };
 
 const BASELINE_MODE = "llvm-O0-O0";

infra/profile.py

@@ -26,10 +26,10 @@ def num_samples():
 
 # timeout in seconds, per function, that we give ILP to find
 # a solution ignoring linearity constraints
-def ilp_test_timeout():
+def ilp_extraction_test_timeout():
   if IS_TESTING_MODE:
-    return 10
-  return 100
+    return 30 # 30 second timeout
+  return 300 # 5 minute timeout
 
 def average(lst):
   return sum(lst) / len(lst)
@@ -48,6 +48,7 @@ def average(lst):
   "llvm-O3-O3",
   "llvm-eggcc-O3-O0",
   "llvm-eggcc-O3-O3",
+  "eggcc-ILP-O0-O0"
 ]
 
 if TO_ABLATE != "":
@@ -97,6 +98,9 @@ def get_eggcc_options(benchmark):
       return (f'optimize', f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm O3_O0 --ablate {TO_ABLATE}')
     case "llvm-eggcc-ablation-O3-O3":
       return (f'optimize', f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm O3_O3 --ablate {TO_ABLATE}')
+    case "eggcc-ILP-O0-O0":
+      # run with the ilp-extraction-timeout flag
+      return (f'optimize --ilp-extraction-test-timeout {ilp_extraction_test_timeout()}', f'--run-mode llvm --optimize-egglog false --optimize-bril-llvm O0_O0')
     case _:
       raise Exception("Unexpected run mode: " + benchmark.treatment)
 
@@ -158,6 +162,7 @@ def optimize(benchmark):
   eggcc_compile_time = 0
   eggcc_extraction_time = 0
   eggcc_serialization_time = 0
+  ilp_test_time = None
   # parse json from eggcc run data
   with open(eggcc_run_data) as f:
     eggcc_data = json.load(f)
@@ -172,6 +177,12 @@ def optimize(benchmark):
     secs = eggcc_data["eggcc_extraction_time"]["secs"]
     nanos = eggcc_data["eggcc_extraction_time"]["nanos"]
     eggcc_extraction_time = secs + nanos / 1e9
+
+    if eggcc_data["ilp_test_time"] is not None:
+      secs = eggcc_data["ilp_test_time"]["secs"]
+      nanos = eggcc_data["ilp_test_time"]["nanos"]
+      ilp_test_time = secs + nanos / 1e9
+
 
   llvm_compile_time = 0
   with open(llvm_run_data) as f:
@@ -187,6 +198,7 @@ def optimize(benchmark):
         "eggccSerializationTimeSecs": eggcc_serialization_time,
         "eggccExtractionTimeSecs": eggcc_extraction_time,
         "llvmCompileTimeSecs": llvm_compile_time,
+        "ilpTestTimeSecs": ilp_test_time,
     }
   return res
 

src/main.rs

@@ -132,9 +132,7 @@ fn main() {
             linearity: !args.no_linearity,
             optimize_functions: args.optimize_function.map(|s| once(s.clone()).collect()),
             ablate: args.ablate,
-            ilp_extraction_test_timeout: args
-                .ilp_extraction_test_timeout
-                .map(Duration::from_secs),
+            ilp_extraction_test_timeout: args.ilp_extraction_test_timeout.map(Duration::from_secs),
         },
     };