measure

bencherdev · Dec 4, 2023 · 49b22da · 49b22da
1 parent e75739a
commit 49b22da
Show file tree

Hide file tree

Showing 114 changed files with 1,559 additions and 1,348 deletions.
diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@ Bencher could have prevented that from happening.
 Bencher allows you to detect and prevent performance regressions _before_ they hit production.
 
 - **Run**: Run your benchmarks locally or in CI using your favorite benchmarking tools. The `bencher` CLI simply wraps your existing benchmark harness and stores its results.
-- **Track**: Track the results of your benchmarks over time. Monitor, query, and graph the results using the Bencher web console based on the source branch, testbed, and metric kind.
+- **Track**: Track the results of your benchmarks over time. Monitor, query, and graph the results using the Bencher web console based on the source branch, testbed, and measure.
 - **Catch**: Catch performance regressions in CI. Bencher uses state of the art, customizable analytics to detect performance regressions before they make it to production.
 
 For the same reasons that unit tests are run in CI to prevent feature regressions, benchmarks should be run in CI with Bencher to prevent performance regressions. Performance bugs are bugs!
@@ -192,23 +192,23 @@ bencher run --project my-project-slug --github-actions "${{ secrets.GITHUB_TOKEN
     </tr>
     <tr>
         <td>JsonAdapter::Json</td>
-        <td>🚨 (<a href="https://bencher.dev/perf/bencher?metric_kinds=4358146b-b647-4869-9d24-bd22bb0c49b5&branches=a90d5b4f-047e-4dbe-8bce-1516a30df049&testbeds=0d991aac-b241-493a-8b0f-8d41419455d2&benchmarks=e93b3d71-8499-4fae-bb7c-4e540b775714&start_time=1688169600000&upper_boundary=true">view plot</a> | <a href="https://bencher.dev/perf/bencher/alerts/90f565dd-202c-4a82-b852-aaa8f3e98da4">view alert</a>)</td>
+        <td>🚨 (<a href="https://bencher.dev/perf/bencher?measures=4358146b-b647-4869-9d24-bd22bb0c49b5&branches=a90d5b4f-047e-4dbe-8bce-1516a30df049&testbeds=0d991aac-b241-493a-8b0f-8d41419455d2&benchmarks=e93b3d71-8499-4fae-bb7c-4e540b775714&start_time=1688169600000&upper_boundary=true">view plot</a> | <a href="https://bencher.dev/perf/bencher/alerts/90f565dd-202c-4a82-b852-aaa8f3e98da4">view alert</a>)</td>
     </tr>
     <tr>
         <td>JsonAdapter::Magic (JSON)</td>
-        <td>✅ (<a href="https://bencher.dev/perf/bencher?metric_kinds=4358146b-b647-4869-9d24-bd22bb0c49b5&branches=a90d5b4f-047e-4dbe-8bce-1516a30df049&testbeds=0d991aac-b241-493a-8b0f-8d41419455d2&benchmarks=3bfd5887-83ec-4e62-8690-02855a38fbc9&start_time=1688169600000&upper_boundary=true">view plot</a>)</td>
+        <td>✅ (<a href="https://bencher.dev/perf/bencher?measures=4358146b-b647-4869-9d24-bd22bb0c49b5&branches=a90d5b4f-047e-4dbe-8bce-1516a30df049&testbeds=0d991aac-b241-493a-8b0f-8d41419455d2&benchmarks=3bfd5887-83ec-4e62-8690-02855a38fbc9&start_time=1688169600000&upper_boundary=true">view plot</a>)</td>
     </tr>
     <tr>
         <td>JsonAdapter::Magic (Rust)</td>
-        <td>✅ (<a href="https://bencher.dev/perf/bencher?metric_kinds=4358146b-b647-4869-9d24-bd22bb0c49b5&branches=a90d5b4f-047e-4dbe-8bce-1516a30df049&testbeds=0d991aac-b241-493a-8b0f-8d41419455d2&benchmarks=3525f177-fc8f-4a92-bd2f-dda7c4e15699&start_time=1688169600000&upper_boundary=true">view plot</a>)</td>
+        <td>✅ (<a href="https://bencher.dev/perf/bencher?measures=4358146b-b647-4869-9d24-bd22bb0c49b5&branches=a90d5b4f-047e-4dbe-8bce-1516a30df049&testbeds=0d991aac-b241-493a-8b0f-8d41419455d2&benchmarks=3525f177-fc8f-4a92-bd2f-dda7c4e15699&start_time=1688169600000&upper_boundary=true">view plot</a>)</td>
     </tr>
     <tr>
         <td>JsonAdapter::Rust</td>
-        <td>✅ (<a href="https://bencher.dev/perf/bencher?metric_kinds=4358146b-b647-4869-9d24-bd22bb0c49b5&branches=a90d5b4f-047e-4dbe-8bce-1516a30df049&testbeds=0d991aac-b241-493a-8b0f-8d41419455d2&benchmarks=5655ed2a-3e45-4622-bdbd-39cdd9837af8&start_time=1688169600000&upper_boundary=true">view plot</a>)</td>
+        <td>✅ (<a href="https://bencher.dev/perf/bencher?measures=4358146b-b647-4869-9d24-bd22bb0c49b5&branches=a90d5b4f-047e-4dbe-8bce-1516a30df049&testbeds=0d991aac-b241-493a-8b0f-8d41419455d2&benchmarks=5655ed2a-3e45-4622-bdbd-39cdd9837af8&start_time=1688169600000&upper_boundary=true">view plot</a>)</td>
     </tr>
     <tr>
         <td>JsonAdapter::RustBench</td>
-        <td>🚨 (<a href="https://bencher.dev/perf/bencher?metric_kinds=4358146b-b647-4869-9d24-bd22bb0c49b5&branches=a90d5b4f-047e-4dbe-8bce-1516a30df049&testbeds=0d991aac-b241-493a-8b0f-8d41419455d2&benchmarks=1db23e93-f909-40aa-bf42-838cc7ae05f5&start_time=1688169600000&upper_boundary=true">view plot</a> | <a href="https://bencher.dev/perf/bencher/alerts/f84550f3-972d-4f84-b204-3a9c292d46a2">view alert</a>)</td>
+        <td>🚨 (<a href="https://bencher.dev/perf/bencher?measures=4358146b-b647-4869-9d24-bd22bb0c49b5&branches=a90d5b4f-047e-4dbe-8bce-1516a30df049&testbeds=0d991aac-b241-493a-8b0f-8d41419455d2&benchmarks=1db23e93-f909-40aa-bf42-838cc7ae05f5&start_time=1688169600000&upper_boundary=true">view plot</a> | <a href="https://bencher.dev/perf/bencher/alerts/f84550f3-972d-4f84-b204-3a9c292d46a2">view alert</a>)</td>
     </tr>
 </table>
 <br />
@@ -245,9 +245,9 @@ Do **not** specify an exact version if using Bencher _Cloud_ as there are still
 All public projects have their own [perf page](https://bencher.dev/perf). These results can easily be shared with an auto-updating perf image. Perfect for your README!
 
 <p align="center">
-  <a href="https://bencher.dev/perf/bencher?key=true&metric_kinds=4358146b-b647-4869-9d24-bd22bb0c49b5&tab=benchmarks&testbeds=0d991aac-b241-493a-8b0f-8d41419455d2&branches=619d15ed-0fbd-4ccb-86cb-fddf3124da29&benchmarks=3525f177-fc8f-4a92-bd2f-dda7c4e15699%2C1db23e93-f909-40aa-bf42-838cc7ae05f5&start_time=1674950400000">
+  <a href="https://bencher.dev/perf/bencher?key=true&measures=4358146b-b647-4869-9d24-bd22bb0c49b5&tab=benchmarks&testbeds=0d991aac-b241-493a-8b0f-8d41419455d2&branches=619d15ed-0fbd-4ccb-86cb-fddf3124da29&benchmarks=3525f177-fc8f-4a92-bd2f-dda7c4e15699%2C1db23e93-f909-40aa-bf42-838cc7ae05f5&start_time=1674950400000">
     <img
-      src="https://api.bencher.dev/v0/projects/bencher/perf/img?branches=619d15ed-0fbd-4ccb-86cb-fddf3124da29&testbeds=0d991aac-b241-493a-8b0f-8d41419455d2&benchmarks=3525f177-fc8f-4a92-bd2f-dda7c4e15699%2C1db23e93-f909-40aa-bf42-838cc7ae05f5&metric_kinds=4358146b-b647-4869-9d24-bd22bb0c49b5&start_time=1674950400000&title=Benchmark+Adapter+Comparison"
+      src="https://api.bencher.dev/v0/projects/bencher/perf/img?branches=619d15ed-0fbd-4ccb-86cb-fddf3124da29&testbeds=0d991aac-b241-493a-8b0f-8d41419455d2&benchmarks=3525f177-fc8f-4a92-bd2f-dda7c4e15699%2C1db23e93-f909-40aa-bf42-838cc7ae05f5&measures=4358146b-b647-4869-9d24-bd22bb0c49b5&start_time=1674950400000&title=Benchmark+Adapter+Comparison"
       title="Benchmark Adapter Comparison"
       alt="Benchmark Adapter Comparison for Bencher - Bencher"
     />

diff --git a/lib/bencher_adapter/src/adapters/java/jmh.rs b/lib/bencher_adapter/src/adapters/java/jmh.rs
@@ -5,7 +5,7 @@ use serde::Deserialize;
 
 use crate::{
     adapters::util::{latency_as_nanos, throughput_as_secs},
-    results::adapter_results::{AdapterMetricKind, AdapterResults},
+    results::adapter_results::{AdapterMeasure, AdapterResults},
     Adaptable, AdapterError, Settings,
 };
 
@@ -66,7 +66,7 @@ impl TryFrom<Jmh> for Option<AdapterResults> {
                 score_unit,
             } = primary_metric;
 
-            let metric_kind = if let Some((unit, slash_op)) = score_unit.split_once("/op") {
+            let measure = if let Some((unit, slash_op)) = score_unit.split_once("/op") {
                 if !slash_op.is_empty() {
                     return Err(AdapterError::BenchmarkUnits(slash_op.into()));
                 }
@@ -80,7 +80,7 @@ impl TryFrom<Jmh> for Option<AdapterResults> {
                     lower_value: Some(lower_value),
                     upper_value: Some(upper_value),
                 };
-                AdapterMetricKind::Latency(json_metric)
+                AdapterMeasure::Latency(json_metric)
             } else if let Some((ops_slash, unit)) = score_unit.split_once("ops/") {
                 if !ops_slash.is_empty() {
                     return Err(AdapterError::BenchmarkUnits(ops_slash.into()));
@@ -95,12 +95,12 @@ impl TryFrom<Jmh> for Option<AdapterResults> {
                     lower_value: Some(lower_value),
                     upper_value: Some(upper_value),
                 };
-                AdapterMetricKind::Throughput(json_metric)
+                AdapterMeasure::Throughput(json_metric)
             } else {
                 return Err(AdapterError::BenchmarkUnits(score_unit));
             };
 
-            benchmark_metrics.push((benchmark_name, metric_kind));
+            benchmark_metrics.push((benchmark_name, measure));
         }
 
         Ok(AdapterResults::new(benchmark_metrics))

diff --git a/lib/bencher_adapter/src/adapters/mod.rs b/lib/bencher_adapter/src/adapters/mod.rs
@@ -25,7 +25,7 @@ fn print_ln(input: &str) -> IResult<&str, ()> {
 #[allow(clippy::panic, clippy::unwrap_used)]
 pub(crate) mod test_util {
     use bencher_json::project::{
-        metric_kind::{LATENCY_SLUG_STR, THROUGHPUT_SLUG_STR},
+        measure::{LATENCY_SLUG_STR, THROUGHPUT_SLUG_STR},
         report::JsonAverage,
     };
     use ordered_float::OrderedFloat;

diff --git a/lib/bencher_adapter/src/adapters/rust/iai.rs b/lib/bencher_adapter/src/adapters/rust/iai.rs
@@ -1,6 +1,6 @@
 use bencher_json::{
     project::{
-        metric_kind::{
+        measure::{
             ESTIMATED_CYCLES_NAME_STR, INSTRUCTIONS_NAME_STR, L1_ACCESSES_NAME_STR,
             L2_ACCESSES_NAME_STR, RAM_ACCESSES_NAME_STR,
         },
@@ -19,7 +19,7 @@ use nom::{
 
 use crate::{
     adapters::util::{parse_f64, parse_u64},
-    results::adapter_results::{AdapterResults, IaiMetricKind},
+    results::adapter_results::{AdapterResults, IaiMeasure},
     Adaptable, Settings,
 };
 
@@ -55,7 +55,7 @@ impl Adaptable for AdapterRustIai {
 
 fn parse_iai_lines(
     lines: [&str; IAI_METRICS_LINE_COUNT],
-) -> Option<(BenchmarkName, Vec<IaiMetricKind>)> {
+) -> Option<(BenchmarkName, Vec<IaiMeasure>)> {
     let [benchmark_name_line, instructions_line, l1_accesses_line, l2_accesses_line, ram_accesses_line, estimated_cycles_line] =
         lines;
 
@@ -65,33 +65,32 @@ fn parse_iai_lines(
         (
             INSTRUCTIONS_NAME_STR,
             instructions_line,
-            IaiMetricKind::Instructions as fn(JsonMetric) -> IaiMetricKind,
+            IaiMeasure::Instructions as fn(JsonMetric) -> IaiMeasure,
         ),
         (
             L1_ACCESSES_NAME_STR,
             l1_accesses_line,
-            IaiMetricKind::L1Accesses,
+            IaiMeasure::L1Accesses,
         ),
         (
             L2_ACCESSES_NAME_STR,
             l2_accesses_line,
-            IaiMetricKind::L2Accesses,
+            IaiMeasure::L2Accesses,
         ),
         (
             RAM_ACCESSES_NAME_STR,
             ram_accesses_line,
-            IaiMetricKind::RamAccesses,
+            IaiMeasure::RamAccesses,
         ),
         (
             ESTIMATED_CYCLES_NAME_STR,
             estimated_cycles_line,
-            IaiMetricKind::EstimatedCycles,
+            IaiMeasure::EstimatedCycles,
         ),
     ]
     .into_iter()
-    .map(|(metric_kind, input, into_variant)| {
-        parse_iai_metric(input, metric_kind)
-            .map(|(_remainder, json_metric)| into_variant(json_metric))
+    .map(|(measure, input, into_variant)| {
+        parse_iai_metric(input, measure).map(|(_remainder, json_metric)| into_variant(json_metric))
     })
     .collect::<Result<Vec<_>, _>>()
     .ok()?;
@@ -100,11 +99,11 @@ fn parse_iai_lines(
 }
 
 #[allow(clippy::cast_precision_loss)]
-fn parse_iai_metric<'a>(input: &'a str, metric_kind: &'static str) -> IResult<&'a str, JsonMetric> {
+fn parse_iai_metric<'a>(input: &'a str, measure: &'static str) -> IResult<&'a str, JsonMetric> {
     map(
         tuple((
             space0,
-            tag(metric_kind),
+            tag(measure),
             tag(":"),
             space1,
             parse_u64,
@@ -147,7 +146,7 @@ pub(crate) mod test_rust_iai {
         Adaptable, AdapterResults,
     };
     use bencher_json::{
-        project::metric_kind::{
+        project::measure::{
             ESTIMATED_CYCLES_SLUG_STR, INSTRUCTIONS_NAME_STR, INSTRUCTIONS_SLUG_STR,
             L1_ACCESSES_SLUG_STR, L2_ACCESSES_SLUG_STR, RAM_ACCESSES_SLUG_STR,
         },

diff --git a/lib/bencher_adapter/src/results/adapter_metrics.rs b/lib/bencher_adapter/src/results/adapter_metrics.rs
@@ -3,15 +3,15 @@ use std::{collections::HashMap, str::FromStr};
 use bencher_json::JsonMetric;
 use serde::{Deserialize, Serialize};
 
-use super::{CombinedKind, MetricKind, OrdKind};
+use super::{CombinedKind, Measure, OrdKind};
 
 #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
 pub struct AdapterMetrics {
     #[serde(flatten)]
     pub inner: MetricsMap,
 }
 
-pub type MetricsMap = HashMap<MetricKind, JsonMetric>;
+pub type MetricsMap = HashMap<Measure, JsonMetric>;
 
 impl From<MetricsMap> for AdapterMetrics {
     fn from(inner: MetricsMap) -> Self {
@@ -22,8 +22,8 @@ impl From<MetricsMap> for AdapterMetrics {
 impl AdapterMetrics {
     pub(crate) fn combined(self, mut other: Self, kind: CombinedKind) -> Self {
         let mut metric_map = HashMap::new();
-        for (metric_kind, metric) in self.inner {
-            let other_metric = other.inner.remove(&metric_kind);
+        for (measure, metric) in self.inner {
+            let other_metric = other.inner.remove(&measure);
             let combined_metric = if let Some(other_metric) = other_metric {
                 match kind {
                     CombinedKind::Ord(ord_kind) => match ord_kind {
@@ -35,14 +35,14 @@ impl AdapterMetrics {
             } else {
                 metric
             };
-            metric_map.insert(metric_kind, combined_metric);
+            metric_map.insert(measure, combined_metric);
         }
         metric_map.extend(other.inner);
         metric_map.into()
     }
 
     pub fn get(&self, key: &str) -> Option<&JsonMetric> {
-        self.inner.get(&MetricKind::from_str(key).ok()?)
+        self.inner.get(&Measure::from_str(key).ok()?)
     }
 }
 
@@ -51,8 +51,8 @@ impl std::ops::Div<usize> for AdapterMetrics {
 
     fn div(self, rhs: usize) -> Self::Output {
         let mut metric_map = HashMap::new();
-        for (metric_kind, metric) in self.inner {
-            metric_map.insert(metric_kind, metric / rhs);
+        for (measure, metric) in self.inner {
+            metric_map.insert(measure, metric / rhs);
         }
         metric_map.into()
     }