boundary_average

bencherdev · Dec 3, 2023 · 235a79a · 235a79a
1 parent 31fcabd
commit 235a79a
Show file tree

Hide file tree

Showing 16 changed files with 139 additions and 7 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/lib/bencher_boundary/Cargo.toml b/lib/bencher_boundary/Cargo.toml
@@ -10,6 +10,7 @@ plus = ["bencher_json/plus"]
 [dependencies]
 bencher_json = { workspace = true, features = ["full"] }
 bencher_logger.workspace = true
+ordered-float.workspace = true
 slog.workspace = true
 thiserror.workspace = true
 # Crate

diff --git a/lib/bencher_boundary/src/limits.rs b/lib/bencher_boundary/src/limits.rs
@@ -6,6 +6,7 @@ use crate::BoundaryError;
 
 #[derive(Debug, Default)]
 pub struct MetricsLimits {
+    pub average: f64,
     pub lower: Option<MetricsLimit>,
     pub upper: Option<MetricsLimit>,
 }
@@ -51,7 +52,11 @@ impl MetricsLimits {
                     let abs_limit = normal.inverse_cdf(limit.into());
                     MetricsLimit::upper(abs_limit)
                 });
-                Self { lower, upper }
+                Self {
+                    average: mean,
+                    lower,
+                    upper,
+                }
             },
             // Create a Student's t distribution and calculate the boundary limits for the threshold based on the boundary percentiles.
             TestKind::T { freedom } => {
@@ -75,7 +80,11 @@ impl MetricsLimits {
                     let abs_limit = students_t.inverse_cdf(limit.into());
                     MetricsLimit::upper(abs_limit)
                 });
-                Self { lower, upper }
+                Self {
+                    average: mean,
+                    lower,
+                    upper,
+                }
             },
         })
     }
@@ -124,6 +133,7 @@ mod test {
     use bencher_json::{project::boundary::BoundaryLimit, Boundary};
     use bencher_logger::bootstrap_logger;
     use once_cell::sync::Lazy;
+    use ordered_float::OrderedFloat;
     use pretty_assertions::assert_eq;
 
     use super::{MetricsLimit, MetricsLimits, TestKind};
@@ -146,6 +156,7 @@ mod test {
     fn test_limits_z_none() {
         let log = bootstrap_logger();
         let limits = MetricsLimits::new(&log, MEAN, STD_DEV, TestKind::Z, None, None).unwrap();
+        assert_eq!(OrderedFloat::from(limits.average), OrderedFloat::from(MEAN));
         assert_eq!(limits.lower, None);
         assert_eq!(limits.upper, None);
 
@@ -170,6 +181,7 @@ mod test {
         let log = bootstrap_logger();
         let limits =
             MetricsLimits::new(&log, MEAN, STD_DEV, TestKind::Z, Some(*PERCENTILE), None).unwrap();
+        assert_eq!(OrderedFloat::from(limits.average), OrderedFloat::from(MEAN));
         assert_eq!(limits.lower, Some(MetricsLimit { value: -Z_LIMIT }));
         assert_eq!(limits.upper, None);
 
@@ -194,6 +206,7 @@ mod test {
         let log = bootstrap_logger();
         let limits =
             MetricsLimits::new(&log, MEAN, STD_DEV, TestKind::Z, None, Some(*PERCENTILE)).unwrap();
+        assert_eq!(OrderedFloat::from(limits.average), OrderedFloat::from(MEAN));
         assert_eq!(limits.lower, None);
         assert_eq!(limits.upper, Some(MetricsLimit { value: Z_LIMIT }));
 
@@ -225,6 +238,7 @@ mod test {
             Some(*PERCENTILE),
         )
         .unwrap();
+        assert_eq!(OrderedFloat::from(limits.average), OrderedFloat::from(MEAN));
         assert_eq!(limits.lower, Some(MetricsLimit { value: -Z_LIMIT }));
         assert_eq!(limits.upper, Some(MetricsLimit { value: Z_LIMIT }));
 
@@ -246,17 +260,22 @@ mod test {
 
     #[test]
     fn test_limits_z_docs() {
+        const MEAN_100: f64 = 100.0;
         let log = bootstrap_logger();
         let boundary = 0.977.try_into().expect("Failed to create boundary.");
         let limits = MetricsLimits::new(
             &log,
-            100.0,
+            MEAN_100,
             10.0,
             TestKind::Z,
             Some(boundary),
             Some(boundary),
         )
         .unwrap();
+        assert_eq!(
+            OrderedFloat::from(limits.average),
+            OrderedFloat::from(MEAN_100)
+        );
         assert_eq!(
             limits.lower,
             Some(MetricsLimit {
@@ -298,6 +317,7 @@ mod test {
             None,
         )
         .unwrap();
+        assert_eq!(OrderedFloat::from(limits.average), OrderedFloat::from(MEAN));
         assert_eq!(limits.lower, None);
         assert_eq!(limits.upper, None);
 
@@ -329,6 +349,7 @@ mod test {
             None,
         )
         .unwrap();
+        assert_eq!(OrderedFloat::from(limits.average), OrderedFloat::from(MEAN));
         assert_eq!(limits.lower, Some(MetricsLimit { value: -T_LIMIT }));
         assert_eq!(limits.upper, None);
 
@@ -360,6 +381,7 @@ mod test {
             Some(*PERCENTILE),
         )
         .unwrap();
+        assert_eq!(OrderedFloat::from(limits.average), OrderedFloat::from(MEAN));
         assert_eq!(limits.lower, None);
         assert_eq!(limits.upper, Some(MetricsLimit { value: T_LIMIT }));
 
@@ -391,6 +413,7 @@ mod test {
             Some(*PERCENTILE),
         )
         .unwrap();
+        assert_eq!(OrderedFloat::from(limits.average), OrderedFloat::from(MEAN));
         assert_eq!(limits.lower, Some(MetricsLimit { value: -T_LIMIT }));
         assert_eq!(limits.upper, Some(MetricsLimit { value: T_LIMIT }));
 

diff --git a/lib/bencher_json/src/project/boundary.rs b/lib/bencher_json/src/project/boundary.rs
@@ -15,6 +15,7 @@ crate::from_vec!(JsonBoundaries[JsonBoundary]);
 #[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
 #[cfg_attr(feature = "schema", derive(JsonSchema))]
 pub struct JsonBoundary {
+    pub average: OrderedFloat<f64>,
     pub lower_limit: Option<OrderedFloat<f64>>,
     pub upper_limit: Option<OrderedFloat<f64>>,
 }

diff --git a/lib/bencher_valid/swagger.json b/lib/bencher_valid/swagger.json
@@ -6335,6 +6335,10 @@
       "JsonBoundary": {
         "type": "object",
         "properties": {
+          "average": {
+            "type": "number",
+            "format": "double"
+          },
           "lower_limit": {
             "nullable": true,
             "type": "number",
@@ -6345,7 +6349,10 @@
             "type": "number",
             "format": "double"
           }
-        }
+        },
+        "required": [
+          "average"
+        ]
       },
       "JsonBranch": {
         "type": "object",

diff --git a/services/api/migrations/2023-12-02-231731_boundary_average/down.sql b/services/api/migrations/2023-12-02-231731_boundary_average/down.sql
@@ -0,0 +1,34 @@
+PRAGMA foreign_keys = off;
+CREATE TABLE down_boundary (
+    id INTEGER PRIMARY KEY NOT NULL,
+    uuid TEXT NOT NULL UNIQUE,
+    threshold_id INTEGER NOT NULL,
+    statistic_id INTEGER NOT NULL,
+    metric_id INTEGER NOT NULL UNIQUE,
+    lower_limit DOUBLE,
+    upper_limit DOUBLE,
+    FOREIGN KEY (threshold_id) REFERENCES threshold (id),
+    FOREIGN KEY (statistic_id) REFERENCES statistic (id),
+    FOREIGN KEY (metric_id) REFERENCES metric (id) ON DELETE CASCADE
+);
+INSERT INTO down_boundary(
+        id,
+        uuid,
+        threshold_id,
+        statistic_id,
+        metric_id,
+        lower_limit,
+        upper_limit
+    )
+SELECT id,
+    uuid,
+    threshold_id,
+    statistic_id,
+    metric_id,
+    lower_limit,
+    upper_limit
+FROM boundary;
+DROP TABLE boundary;
+ALTER TABLE down_boundary
+    RENAME TO boundary;
+PRAGMA foreign_keys = on;
diff --git a/services/api/migrations/2023-12-02-231731_boundary_average/up.sql b/services/api/migrations/2023-12-02-231731_boundary_average/up.sql
@@ -0,0 +1,40 @@
+PRAGMA foreign_keys = off;
+CREATE TABLE up_boundary (
+    id INTEGER PRIMARY KEY NOT NULL,
+    uuid TEXT NOT NULL UNIQUE,
+    threshold_id INTEGER NOT NULL,
+    statistic_id INTEGER NOT NULL,
+    metric_id INTEGER NOT NULL UNIQUE,
+    average DOUBLE NOT NULL,
+    lower_limit DOUBLE,
+    upper_limit DOUBLE,
+    FOREIGN KEY (threshold_id) REFERENCES threshold (id),
+    FOREIGN KEY (statistic_id) REFERENCES statistic (id),
+    FOREIGN KEY (metric_id) REFERENCES metric (id) ON DELETE CASCADE
+);
+INSERT INTO up_boundary(
+        id,
+        uuid,
+        threshold_id,
+        statistic_id,
+        metric_id,
+        average,
+        lower_limit,
+        upper_limit
+    )
+SELECT id,
+    uuid,
+    threshold_id,
+    statistic_id,
+    metric_id,
+    (
+        SELECT value
+        FROM metric
+        WHERE metric.id = boundary.metric_id
+        LIMIT 1
+    ), lower_limit, upper_limit
+FROM boundary;
+DROP TABLE boundary;
+ALTER TABLE up_boundary
+    RENAME TO boundary;
+PRAGMA foreign_keys = on;
diff --git a/services/api/src/endpoints/project/perf/mod.rs b/services/api/src/endpoints/project/perf/mod.rs
@@ -276,6 +276,7 @@ fn perf_query(
                     schema::boundary::threshold_id,
                     schema::boundary::statistic_id,
                     schema::boundary::metric_id,
+                    schema::boundary::average,
                     schema::boundary::lower_limit,
                     schema::boundary::upper_limit,
                 ),

diff --git a/services/api/src/model/project/report/mod.rs b/services/api/src/model/project/report/mod.rs
@@ -158,6 +158,7 @@ fn get_report_results(
             schema::boundary::threshold_id,
             schema::boundary::statistic_id,
             schema::boundary::metric_id,
+            schema::boundary::average,
             schema::boundary::lower_limit,
             schema::boundary::upper_limit,
         ).nullable(),

diff --git a/services/api/src/model/project/report/results/detector/mod.rs b/services/api/src/model/project/report/results/detector/mod.rs
@@ -85,6 +85,7 @@ impl Detector {
             threshold_id: self.threshold.id,
             statistic_id: self.threshold.statistic.id,
             metric_id: query_metric.id,
+            average: boundary.limits.average,
             lower_limit: boundary.limits.lower.map(Into::into),
             upper_limit: boundary.limits.upper.map(Into::into),
         };

diff --git a/services/api/src/model/project/threshold/boundary.rs b/services/api/src/model/project/threshold/boundary.rs
@@ -23,6 +23,7 @@ pub struct QueryBoundary {
     pub threshold_id: ThresholdId,
     pub statistic_id: StatisticId,
     pub metric_id: MetricId,
+    pub average: f64,
     pub lower_limit: Option<f64>,
     pub upper_limit: Option<f64>,
 }
@@ -48,6 +49,7 @@ impl QueryBoundary {
 
     pub fn into_json(self) -> JsonBoundary {
         JsonBoundary {
+            average: self.average.into(),
             lower_limit: self.lower_limit.map(Into::into),
             upper_limit: self.upper_limit.map(Into::into),
         }
@@ -61,6 +63,7 @@ pub struct InsertBoundary {
     pub threshold_id: ThresholdId,
     pub statistic_id: StatisticId,
     pub metric_id: MetricId,
+    pub average: f64,
     pub lower_limit: Option<f64>,
     pub upper_limit: Option<f64>,
 }
diff --git a/services/api/src/schema.rs b/services/api/src/schema.rs
@@ -30,6 +30,7 @@ diesel::table! {
         threshold_id -> Integer,
         statistic_id -> Integer,
         metric_id -> Integer,
+        average -> Double,
         lower_limit -> Nullable<Double>,
         upper_limit -> Nullable<Double>,
     }

diff --git a/services/cli/src/bencher/sub/project/run/mod.rs b/services/cli/src/bencher/sub/project/run/mod.rs
@@ -263,7 +263,8 @@ impl Run {
 
         // TODO disable when quiet
         if self.html {
-            cli_println!("{}", report_urls.html(false, false, false, None));
+            cli_println!("{}", report_urls.html(true, false, false, None));
+            // cli_println!("{}", report_urls.html(false, false, false, None));
         } else {
             cli_println!("{report_urls}");
         }

diff --git a/services/cli/src/bencher/sub/project/run/urls.rs b/services/cli/src/bencher/sub/project/run/urls.rs
@@ -142,7 +142,9 @@ impl ReportUrls {
 
             if with_metrics {
                 let units = &metric_kind.units;
-                html.push_str(&format!("<th>{metric_kind_name} Results<br/>{units}</th>",));
+                html.push_str(&format!(
+                    "<th>{metric_kind_name} Results<br/>{units} | (Δ%)</th>",
+                ));
                 if boundary.lower_boundary.is_some() {
                     html.push_str(&format!(
                         "<th>{metric_kind_name} Lower Boundary<br/>{units} | (%)</th>"
@@ -220,7 +222,16 @@ impl ReportUrls {
 
                 if with_metrics {
                     let value = *value;
-                    html.push_str(&format!("<td>{value:.3}</td>"));
+                    let value_percent = if value.is_normal() && boundary.average.is_normal() {
+                        ((value - boundary.average) / boundary.average) * 100.0
+                    } else {
+                        0.0
+                    };
+                    let value_plus = if value_percent > 0.0 { "+" } else { "" };
+
+                    html.push_str(&format!(
+                        "<td>{value:.3} ({value_plus}{value_percent:.2}%)</td>"
+                    ));
                     if let Some(lower_boundary) = boundary.lower_boundary {
                         let limit_percent = if value.is_normal() && lower_boundary.is_normal() {
                             (lower_boundary / value) * 100.0
@@ -500,13 +511,15 @@ impl BenchmarkUrl {
 
 #[derive(Clone, Copy)]
 pub struct BoundaryParam {
+    average: f64,
     lower_boundary: Option<f64>,
     upper_boundary: Option<f64>,
 }
 
 impl From<JsonBoundary> for BoundaryParam {
     fn from(json_boundary: JsonBoundary) -> Self {
         Self {
+            average: json_boundary.average.into(),
             lower_boundary: json_boundary.lower_limit.map(Into::into),
             upper_boundary: json_boundary.upper_limit.map(Into::into),
         }

diff --git a/services/console/src/chunks/reference/en/changelog.mdx b/services/console/src/chunks/reference/en/changelog.mdx
@@ -1,3 +1,6 @@
+## Pending `v0.3.19`
+- Include the `average` used for the Boundary Limits
+
 ## `v0.3.18`
 - Add public Alert pages
 - Add a `--ci-public-links` option to `bencher run` to only post public links to PRs

diff --git a/services/console/src/types/bencher.ts b/services/console/src/types/bencher.ts
@@ -69,6 +69,7 @@ export interface JsonThresholdStatistic {
 }
 
 export interface JsonBoundary {
+	average: number;
 	lower_limit?: number;
 	upper_limit?: number;
 }