expose env var + wip: realtime

app-server/src/db/evaluations.rs

@@ -81,7 +81,6 @@ pub async fn set_evaluation_results(
     trace_ids: &Vec<Uuid>,
     indices: &Vec<i32>,
 ) -> Result<()> {
-    dbg!(&indices);
     let results = sqlx::query_as::<_, EvaluationDatapointPreview>(
         r"INSERT INTO evaluation_results (
             id,

app-server/src/traces/spans.rs

@@ -1,4 +1,4 @@
-use std::{collections::HashMap, sync::Arc};
+use std::{collections::HashMap, env, sync::Arc};
 
 use anyhow::Result;
 use chrono::{TimeZone, Utc};
@@ -46,7 +46,7 @@ const HAS_BROWSER_SESSION_ATTRIBUTE_NAME: &str = "lmnr.internal.has_browser_sess
 //
 // We use 7/2 as an estimate of the number of characters per token.
 // And 128K is a common input size for LLM calls.
-const PAYLOAD_SIZE_THRESHOLD: usize = (7 / 2) * 128_000; // approx 448KB
+const DEFAULT_PAYLOAD_SIZE_THRESHOLD: usize = (7 / 2) * 128_000; // approx 448KB
 
 #[derive(Debug, Clone, PartialEq, Deserialize)]
 #[serde(rename_all = "snake_case")]
@@ -585,6 +585,10 @@ impl Span {
         project_id: &Uuid,
         storage: Arc<S>,
     ) -> Result<()> {
+        let payload_size_threshold = env::var("MAX_DB_SPAN_PAYLOAD_BYTES")
+            .ok()
+            .and_then(|s| s.parse::<usize>().ok())
+            .unwrap_or(DEFAULT_PAYLOAD_SIZE_THRESHOLD);
         if let Some(input) = self.input.clone() {
             let span_input = serde_json::from_value::<Vec<ChatMessage>>(input);
             if let Ok(span_input) = span_input {
@@ -605,7 +609,7 @@ impl Span {
             // but we don't need to be exact here.
             } else {
                 let input_str = serde_json::to_string(&self.input).unwrap_or_default();
-                if input_str.len() > PAYLOAD_SIZE_THRESHOLD {
+                if input_str.len() > payload_size_threshold {
                     let key = crate::storage::create_key(project_id, &None);
                     let mut data = Vec::new();
                     serde_json::to_writer(&mut data, &self.input)?;
@@ -619,7 +623,7 @@ impl Span {
         }
         if let Some(output) = self.output.clone() {
             let output_str = serde_json::to_string(&output).unwrap_or_default();
-            if output_str.len() > PAYLOAD_SIZE_THRESHOLD {
+            if output_str.len() > payload_size_threshold {
                 let key = crate::storage::create_key(project_id, &None);
                 let mut data = Vec::new();
                 serde_json::to_writer(&mut data, &output)?;

frontend/app/api/projects/[projectId]/evaluations/[evaluationId]/route.ts

@@ -44,6 +44,7 @@ export async function GET(
       target: sql<string>`SUBSTRING(${evaluationResults.target}::text, 0, 100)`.as('target'),
       executorOutput: evaluationResults.executorOutput,
       scores: subQueryScoreCte.cteScores,
+      index: evaluationResults.index
     })
     .from(evaluationResults)
     .leftJoin(

frontend/app/project/[projectId]/evaluations/[evaluationId]/page.tsx

@@ -9,6 +9,7 @@ import {
   evaluations, evaluationScores
 } from '@/lib/db/migrations/schema';
 import { EvaluationResultsInfo } from '@/lib/evaluation/types';
+import { isFeatureEnabled, Feature } from '@/lib/features/features';
 
 
 export const metadata: Metadata = {
@@ -37,6 +38,7 @@ export default async function EvaluationPage(
     <Evaluation
       evaluationInfo={evaluationInfo}
       evaluations={evaluationsByGroupId}
+      isSupabaseEnabled={isFeatureEnabled(Feature.SUPABASE)}
     />
   );
 }
@@ -75,7 +77,8 @@ async function getEvaluationInfo(
       target: sql<string>`SUBSTRING(${evaluationResults.target}::text, 0, 100)`.as('target'),
       executorOutput: evaluationResults.executorOutput,
       scores: subQueryScoreCte.cteScores,
-      traceId: evaluationResults.traceId
+      traceId: evaluationResults.traceId,
+      index: evaluationResults.index
     })
     .from(evaluationResults)
     .leftJoin(
@@ -94,7 +97,7 @@ async function getEvaluationInfo(
   ]);
 
   if (!evaluation) {
-    redirect('/404');
+    redirect('/not-found');
   }
 
   const result = {

frontend/app/project/[projectId]/layout.tsx

@@ -17,9 +17,9 @@ import { GetProjectResponse } from '@/lib/workspaces/types';
 
 export default async function ProjectIdLayout(
   props: {
-      children: React.ReactNode;
-      params: Promise<{ projectId: string }>;
-    }
+    children: React.ReactNode;
+    params: Promise<{ projectId: string }>;
+  }
 ) {
   const params = await props.params;
 
@@ -45,9 +45,9 @@ export default async function ProjectIdLayout(
   const project = projectResponse as GetProjectResponse;
 
   const showBanner =
-      isFeatureEnabled(Feature.WORKSPACE) &&
-      project.isFreeTier &&
-      project.spansThisMonth >= 0.8 * project.spansLimit;
+    isFeatureEnabled(Feature.WORKSPACE) &&
+    project.isFreeTier &&
+    project.spansThisMonth >= 0.8 * project.spansLimit;
 
   const posthog = PostHogClient();
   posthog.identify({

frontend/components/evaluation/evaluation.tsx

@@ -3,7 +3,7 @@ import { ColumnDef } from '@tanstack/react-table';
 import { ArrowRight } from 'lucide-react';
 import { usePathname, useRouter, useSearchParams } from 'next/navigation';
 import { Resizable } from 're-resizable';
-import { useEffect, useState } from 'react';
+import { useEffect, useMemo, useState } from 'react';
 
 import { useProjectContext } from '@/contexts/project-context';
 import {
@@ -27,6 +27,9 @@ import {
 import Chart from './chart';
 import CompareChart from './compare-chart';
 import ScoreCard from './score-card';
+import { useUserContext } from '@/contexts/user-context';
+import { createClient } from '@supabase/supabase-js';
+import { SUPABASE_ANON_KEY, SUPABASE_URL } from '@/lib/const';
 
 const URL_QUERY_PARAMS = {
   COMPARE_EVAL_ID: 'comparedEvaluationId'
@@ -35,41 +38,53 @@ const URL_QUERY_PARAMS = {
 interface EvaluationProps {
   evaluationInfo: EvaluationResultsInfo;
   evaluations: EvaluationType[];
+  isSupabaseEnabled: boolean;
 }
 
 export default function Evaluation({
   evaluationInfo,
-  evaluations
+  evaluations,
+  isSupabaseEnabled
 }: EvaluationProps) {
   const router = useRouter();
   const pathName = usePathname();
   const searchParams = new URLSearchParams(useSearchParams().toString());
   const { toast } = useToast();
-
   const { projectId } = useProjectContext();
-
   const evaluation = evaluationInfo.evaluation;
-
   const [comparedEvaluation, setComparedEvaluation] =
     useState<EvaluationType | null>(null);
+  const defaultResults =
+    evaluationInfo.results as EvaluationDatapointPreviewWithCompared[];
+  const [results, setResults] = useState(defaultResults);
+  // Selected score name must usually not be undefined, as we expect
+  // to have at least one score, it's done just to not throw error if there are no scores
+
+  const [scoreColumns, setScoreColumns] = useState<Set<string>>(new Set());
+  const [selectedScoreName, setSelectedScoreName] = useState<string | undefined>(
+    scoreColumns.size > 0 ? Array.from(scoreColumns)[0] : undefined
+  );
+
+  const updateScoreColumns = (rows: EvaluationDatapointPreviewWithCompared[]) => {
+    let newScoreColumns = new Set<string>(scoreColumns);
+    for (const row of rows) {
+      for (const key of Object.keys(row.scores ?? {})) {
+        newScoreColumns.add(key);
+      }
+    }
+    setScoreColumns(newScoreColumns);
+    setSelectedScoreName(newScoreColumns.size > 0 ? Array.from(newScoreColumns)[0] : undefined);
+  }
 
   useEffect(() => {
     const comparedEvaluationId = searchParams.get(
       URL_QUERY_PARAMS.COMPARE_EVAL_ID
     );
-    handleComparedEvaluationChange(comparedEvaluationId ?? null);
-  }, []);
-
-  let defaultResults =
-    evaluationInfo.results as EvaluationDatapointPreviewWithCompared[];
-  const [results, setResults] = useState(defaultResults);
-
-  let scoreColumns = new Set<string>();
-  for (const row of defaultResults) {
-    for (const key of Object.keys(row.scores ?? {})) {
-      scoreColumns.add(key);
+    if (comparedEvaluationId) {
+      handleComparedEvaluationChange(comparedEvaluationId);
     }
-  }
+    updateScoreColumns(defaultResults);
+  }, []);
 
   // TODO: get datapoints paginated.
   const [selectedDatapoint, setSelectedDatapoint] =
@@ -79,12 +94,6 @@ export default function Evaluation({
       ) ?? null
     );
 
-  // Selected score name must usually not be undefined, as we expect
-  // to have at least one score, it's done just to not throw error if there are no scores
-  const [selectedScoreName, setSelectedScoreName] = useState<string | undefined>(
-    scoreColumns.size > 0 ? Array.from(scoreColumns)[0] : undefined
-  );
-
   // Columns used when there is no compared evaluation
   let defaultColumns: ColumnDef<EvaluationDatapointPreviewWithCompared>[] = [
     {
@@ -109,6 +118,76 @@ export default function Evaluation({
     }))
   );
 
+  const { supabaseAccessToken } = useUserContext();
+
+  const supabase = useMemo(() => {
+    if (!isSupabaseEnabled || !supabaseAccessToken) {
+      return null;
+    }
+
+    return createClient(SUPABASE_URL, SUPABASE_ANON_KEY, {
+      global: {
+        headers: {
+          Authorization: `Bearer ${supabaseAccessToken}`
+        }
+      }
+    });
+  }, [isSupabaseEnabled, supabaseAccessToken]);
+
+  if (supabase) {
+    supabase.realtime.setAuth(supabaseAccessToken);
+  }
+
+  const insertResult = (newRow: { [key: string]: any }) => {
+    const newResult = {
+      id: newRow.id,
+      createdAt: newRow.created_at,
+      index: newRow.index,
+      evaluationId: newRow.evaluation_id,
+      data: newRow.data,
+      target: newRow.target,
+      executorOutput: newRow.executor_output,
+      scores: newRow.scores,
+      traceId: newRow.trace_id,
+    } as EvaluationDatapointPreviewWithCompared;
+    const insertBefore = results.findIndex((result) => result.index > newResult.index);
+    if (insertBefore === -1) {
+      const newResults = [...results, newResult];
+      setResults(newResults);
+    } else {
+      const newResults = [...results.slice(0, insertBefore), newResult, ...results.slice(insertBefore)];
+      setResults(newResults);
+    }
+    updateScoreColumns([newResult]);
+    setColumns(defaultColumns);
+  };
+
+
+  useEffect(() => {
+    if (!supabase) {
+      return;
+    }
+
+    supabase.channel('table-db-changes').unsubscribe();
+
+    supabase
+      .channel('table-db-changes')
+      .on(
+        'postgres_changes',
+        {
+          event: 'INSERT',
+          schema: 'public',
+          table: 'evaluation_results',
+          filter: `evaluation_id=eq.${evaluation.id}`
+        },
+        (payload) => {
+          // FIXME: updates on this break the existing order of the results
+          // insertResult(payload.new);
+        }
+      )
+      .subscribe();
+  }, [supabase]);
+
   const [columns, setColumns] = useState(defaultColumns);
 
   const handleRowClick = (row: EvaluationDatapointPreviewWithCompared) => {

frontend/lib/const.ts

@@ -1,3 +1,5 @@
+// In theory, the SUPABASE keys must be loaded from .env
+// with NEXT_PUBLIC_ prefix, but I couldn't get it to work
 export const SUPABASE_URL = 'https://kqwbtwycngtmqtxwudlx.supabase.co';
 export const SUPABASE_ANON_KEY =
   'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Imtxd2J0d3ljbmd0bXF0eHd1ZGx4Iiwicm9sZSI6ImFub24iLCJpYXQiOjE3MTY2MjcwNDksImV4cCI6MjAzMjIwMzA0OX0.HONeW9Fgy-_whUwGtPYbobmbcKijx650jTrBpuB_E9A';

frontend/lib/evaluation/types.ts

@@ -28,6 +28,7 @@ export type EvaluationDatapointPreview = {
   target: any;
   executorOutput: any;
   traceId: string;
+  index: number;
 };
 
 export type EvaluationDatapointPreviewWithCompared = {