diff --git a/devops/actions/benchmarking/aggregate/action.yml b/devops/actions/benchmarking/aggregate/action.yml index 589221f4ede41..a33a89bab55e0 100644 --- a/devops/actions/benchmarking/aggregate/action.yml +++ b/devops/actions/benchmarking/aggregate/action.yml @@ -2,9 +2,9 @@ name: 'Aggregate compute-benchmark results and produce historical averages' # The benchmarking workflow in sycl-linux-run-tests.yml passes or fails based on # how the benchmark results compare to a historical average: This historical -# average is calculated in this workflow, which aggregates historical data and -# produces measures of central tendency (median in this case) used for this -# purpose. +# average is calculated in this composite workflow, which aggregates historical +# data and produces measures of central tendency (median in this case) used for +# this purpose. # # This action assumes that /devops has been checked out in ./devops. This action # also assumes that GITHUB_TOKEN was properly set in env, because according to @@ -74,4 +74,4 @@ runs: uses: actions/upload-artifact@v4 with: name: llvm-ci-perf-results new medians - path: ./llvm-ci-perf-results/**/*-median.csv \ No newline at end of file + path: ./llvm-ci-perf-results/**/*-median.csv diff --git a/devops/actions/run-tests/benchmark/action.yml b/devops/actions/run-tests/benchmark/action.yml index 907f72eb6154b..db97c9f27ff03 100644 --- a/devops/actions/run-tests/benchmark/action.yml +++ b/devops/actions/run-tests/benchmark/action.yml @@ -54,8 +54,15 @@ runs: git add . git commit -m "[GHA] Upload compute-benchmarks results from https://github.com/intel/llvm/actions/runs/${{ github.run_id }}" git push "https://$GITHUB_TOKEN@github.com/$SANITIZED_PERF_RES_GIT_REPO.git" "$SANITIZED_PERF_RES_GIT_BRANCH" - - shell: bash - run: echo \n\#\n\# Artifact link for benchmark results here\n\#\n + - name: Find workflow artifact below + if: always() + shell: bash + run: | + cat << EOF + # + # Artifact link for benchmark results below: + # + EOF - name: Archive compute-benchmark results if: always() uses: actions/upload-artifact@v4 diff --git a/devops/benchmarking/config.ini b/devops/benchmarking/config.ini index ba8259d349c19..f562dc47a55fb 100644 --- a/devops/benchmarking/config.ini +++ b/devops/benchmarking/config.ini @@ -40,4 +40,4 @@ enabled_backends = level_zero,opencl,cuda,hip ; Devices to allow in device_selector enabled_devices = cpu,gpu -; fpga is disabled \ No newline at end of file +; fpga is disabled diff --git a/devops/benchmarking/constants.ini b/devops/benchmarking/constants.ini index db2d43832618c..1b584215a9ba2 100644 --- a/devops/benchmarking/constants.ini +++ b/devops/benchmarking/constants.ini @@ -43,4 +43,4 @@ git_branch = test-compute-bench ; ; Log file for test cases that perform over the allowed variance ; slow = ./artifact/benchmarks_failed.log ; ; Log file for test cases that errored / failed to build -; error = ./artifact/benchmarks_errored.log \ No newline at end of file +; error = ./artifact/benchmarks_errored.log diff --git a/devops/benchmarking/enabled_tests.conf b/devops/benchmarking/enabled_tests.conf index 0f6e21f93f67b..20659cbea636d 100644 --- a/devops/benchmarking/enabled_tests.conf +++ b/devops/benchmarking/enabled_tests.conf @@ -5,4 +5,4 @@ miscellaneous_benchmark_sycl ulls_benchmark_sycl # As of January 2025, these are every compute-benchmark tests with a SYCL -# implementation. \ No newline at end of file +# implementation. diff --git a/devops/scripts/benchmarking/common.py b/devops/scripts/benchmarking/common.py index 1495dbb62a406..1daaf84a182d9 100644 --- a/devops/scripts/benchmarking/common.py +++ b/devops/scripts/benchmarking/common.py @@ -10,14 +10,18 @@ class Validate: @staticmethod def filepath(path: str) -> bool: + """ + Returns True if path is clean (no illegal characters), otherwise False. + """ filepath_re = re.compile(r"[a-zA-Z0-9\/\._\-]+") return filepath_re.match(path) is not None @staticmethod - # TODO use config def timestamp(t: str) -> bool: + """ + Returns True if t is in form YYYYMMDD_HHMMSS, otherwise False. + """ timestamp_re = re.compile( - # YYYYMMDD_HHMMSS r"^\d{4}(0[1-9]|1[0-2])([0-2][0-9]|3[01])_([01][0-9]|2[0-3])[0-5][0-9][0-5][0-9]$" ) return timestamp_re.match(t) is not None @@ -25,7 +29,8 @@ def timestamp(t: str) -> bool: @staticmethod def sanitize_stat(stat: str) -> float: """ - Sanitize statistics found in compute-benchmark output csv files. Returns float if sanitized, None if not sanitizable + Sanitize statistics found in compute-benchmark output csv files. Returns + float if sanitized, None if not sanitizable. """ # Get rid of % if stat[-1] == "%": @@ -172,8 +177,9 @@ def export_python_globals(self): # self.__sanitize(all_opts["benchmark_log"]["error"], # "benchmark_log.error") + # Fields that are supposed to be python objects need to be changed to - # python objects: + # python objects manually: # metrics.recorded m_rec_str = self.__sanitize(all_opts["metrics"]["recorded"], "metrics.recorded") diff --git a/devops/scripts/benchmarking/compare.py b/devops/scripts/benchmarking/compare.py index aa150894fc391..2162427e53180 100644 --- a/devops/scripts/benchmarking/compare.py +++ b/devops/scripts/benchmarking/compare.py @@ -48,6 +48,7 @@ def to_hist_avg(benchmark_name: str, hist_avg_path: str, test_csv_path: str): exit(1) if sample_value > max_tolerated: + # Log failure if fail, otherwise proceed as usual print(f"\n-- FAILED {benchmark_name}::{test}") print( f" {metric}: {sample_value} -- Historic avg. {test_hist_avg[metric]} (max tolerance {threshold*100}%: {max_tolerated})\n" diff --git a/devops/scripts/benchmarking/load_config.py b/devops/scripts/benchmarking/load_config.py index 04b7799e455cd..69c994e5c296c 100644 --- a/devops/scripts/benchmarking/load_config.py +++ b/devops/scripts/benchmarking/load_config.py @@ -27,4 +27,4 @@ def usage_and_exit(): elif sys.argv[2] == "constants": print(config.export_shell_constants()) else: - usage_and_exit() \ No newline at end of file + usage_and_exit()