add evaluation index

lmnr-ai · Feb 1, 2025 · 3f4f858 · 3f4f858
1 parent 4f3f7bd
commit 3f4f858
Show file tree

Hide file tree

Showing 9 changed files with 3,013 additions and 8 deletions.
diff --git a/app-server/src/db/evaluations.rs b/app-server/src/db/evaluations.rs
@@ -79,7 +79,9 @@ pub async fn set_evaluation_results(
     targets: &Vec<Value>,
     executor_outputs: &Vec<Option<Value>>,
     trace_ids: &Vec<Uuid>,
+    indices: &Vec<i32>,
 ) -> Result<()> {
+    dbg!(&indices);
     let results = sqlx::query_as::<_, EvaluationDatapointPreview>(
         r"INSERT INTO evaluation_results (
             id,
@@ -88,7 +90,7 @@ pub async fn set_evaluation_results(
             target,
             executor_output,
             trace_id,
-            index_in_batch
+            index
         )
         SELECT
             id,
@@ -97,10 +99,10 @@ pub async fn set_evaluation_results(
             target,
             executor_output,
             trace_id,
-            index_in_batch
+            index
         FROM
         UNNEST ($1::uuid[], $2::jsonb[], $3::jsonb[], $4::jsonb[], $5::uuid[], $6::int8[])
-        AS tmp_table(id, data, target, executor_output, trace_id, index_in_batch)
+        AS tmp_table(id, data, target, executor_output, trace_id, index)
         RETURNING id, created_at, evaluation_id, trace_id
         ",
     )
@@ -109,7 +111,7 @@ pub async fn set_evaluation_results(
     .bind(targets)
     .bind(executor_outputs)
     .bind(trace_ids)
-    .bind(&Vec::from_iter(0..ids.len() as i64))
+    .bind(indices)
     .bind(evaluation_id)
     .fetch_all(pool)
     .await?;

diff --git a/app-server/src/evaluations/mod.rs b/app-server/src/evaluations/mod.rs
@@ -41,6 +41,7 @@ pub async fn save_evaluation_scores(
             &columns.targets,
             &columns.executor_outputs,
             &columns.trace_ids,
+            &columns.indices,
         )
         .await
     });

diff --git a/app-server/src/evaluations/utils.rs b/app-server/src/evaluations/utils.rs
@@ -27,6 +27,8 @@ pub struct EvaluationDatapointResult {
     pub human_evaluators: Vec<HumanEvaluator>,
     #[serde(default)]
     pub executor_span_id: Uuid,
+    #[serde(default)]
+    pub index: i32,
 }
 
 pub struct DatapointColumns {
@@ -35,6 +37,7 @@ pub struct DatapointColumns {
     pub executor_outputs: Vec<Option<Value>>,
     pub trace_ids: Vec<Uuid>,
     pub scores: Vec<HashMap<String, f64>>,
+    pub indices: Vec<i32>,
 }
 
 pub fn get_columns_from_points(points: &Vec<EvaluationDatapointResult>) -> DatapointColumns {
@@ -63,12 +66,15 @@ pub fn get_columns_from_points(points: &Vec<EvaluationDatapointResult>) -> Datap
         .map(|point| point.trace_id)
         .collect::<Vec<_>>();
 
+    let indices = points.iter().map(|point| point.index).collect::<Vec<_>>();
+
     DatapointColumns {
         datas,
         targets,
         executor_outputs,
         trace_ids,
         scores,
+        indices,
     }
 }
 

diff --git a/frontend/app/api/projects/[projectId]/evaluations/[evaluationId]/route.ts b/frontend/app/api/projects/[projectId]/evaluations/[evaluationId]/route.ts
@@ -52,8 +52,8 @@ export async function GET(
     )
     .where(eq(evaluationResults.evaluationId, evaluationId))
     .orderBy(
-      asc(evaluationResults.createdAt),
-      asc(evaluationResults.indexInBatch)
+      asc(evaluationResults.index),
+      asc(evaluationResults.createdAt)
     );
 
   const [evaluation, results] = await Promise.all([

diff --git a/frontend/app/project/[projectId]/evaluations/[evaluationId]/page.tsx b/frontend/app/project/[projectId]/evaluations/[evaluationId]/page.tsx
@@ -84,8 +84,8 @@ async function getEvaluationInfo(
     )
     .where(eq(evaluationResults.evaluationId, evaluationId))
     .orderBy(
-      asc(evaluationResults.createdAt),
-      asc(evaluationResults.indexInBatch)
+      asc(evaluationResults.index),
+      asc(evaluationResults.createdAt)
     );
 
   const [evaluation, results] = await Promise.all([

diff --git a/frontend/lib/db/migrations/0018_vengeful_violations.sql b/frontend/lib/db/migrations/0018_vengeful_violations.sql
@@ -0,0 +1 @@
+ALTER TABLE "evaluation_results" ADD COLUMN "index" integer DEFAULT 0 NOT NULL;
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		ALTER TABLE "evaluation_results" ADD COLUMN "index" integer DEFAULT 0 NOT NULL;