Periodically capture resource metrics for every sandbox

packages/api/internal/handlers/sanbox_metrics.go

@@ -34,19 +34,21 @@ func (a *APIStore) GetSandboxesSandboxIDMetrics(
 		attribute.String("team.id", teamID.String()),
 	)
 
-	var start time.Time
-
 	end := time.Now()
+	start := end.Add(-oldestLogsLimit)
 
 	// Sanitize ID
 	// https://grafana.com/blog/2021/01/05/how-to-escape-special-characters-with-lokis-logql/
 	id := strings.ReplaceAll(sandboxID, "`", "")
+
+	// equivalent CLI query:
+	// logcli query '{source="logs-collector", service="envd", teamID="65d165ab-69f6-4b5c-9165-6b93cd341503", sandboxID="izuhqjlfabd8ataeixrtl", category="metrics"}' --from="2025-01-19T10:00:00Z"
 	query := fmt.Sprintf(
-		"{source=\"logs-collector\", category=\"metrics\", service=\"envd\", teamID=`%s`, sandboxID=`%s`}", teamID.String(), id)
+		"{source=\"logs-collector\", service=\"envd\", teamID=\"%s\", sandboxID=\"%s\", category=\"metrics\"}", teamID.String(), id)
 
 	res, err := a.lokiClient.QueryRange(query, 100, start, end, logproto.FORWARD, time.Duration(0), time.Duration(0), true)
 	if err != nil {
-		errMsg := fmt.Errorf("error when returning metrics for sandbox: %w", err)
+		errMsg := fmt.Errorf("error when returning metrics for sandbox: %w\tquery:%s", err, query)
 		telemetry.ReportCriticalError(ctx, errMsg)
 		a.sendAPIStoreError(c, http.StatusNotFound, fmt.Sprintf("Error returning metrics for sandbox '%s'", sandboxID))
 
@@ -63,9 +65,10 @@ func (a *APIStore) GetSandboxesSandboxIDMetrics(
 			for _, entry := range stream.Entries {
 
 				var metric struct {
-					CPUPct      *float32 `json:"cpuPct"`
-					MemTotalMiB int64    `json:"memTotalMiB"`
-					MemUsedMiB  int64    `json:"memUsedMiB"`
+					CPUPct      float32 `json:"cpuPct"`
+					CPUCount    int32   `json:"cpuCount"`
+					MemTotalMiB int64   `json:"memTotalMiB"`
+					MemUsedMiB  int64   `json:"memUsedMiB"`
 				}
 
 				err := json.Unmarshal([]byte(entry.Line), &metric)
@@ -76,6 +79,7 @@ func (a *APIStore) GetSandboxesSandboxIDMetrics(
 				metrics = append(metrics, api.SandboxMetric{
 					Timestamp:   entry.Timestamp,
 					CpuPct:      metric.CPUPct,
+					CpuCount:    metric.CPUCount,
 					MemMiBTotal: metric.MemTotalMiB,
 					MemMiBUsed:  metric.MemUsedMiB,
 				})
@@ -87,9 +91,7 @@ func (a *APIStore) GetSandboxesSandboxIDMetrics(
 			return a.Timestamp.Compare(b.Timestamp)
 		})
 
-		c.JSON(http.StatusOK, &api.SandboxMetrics{
-			Metrics: metrics,
-		})
+		c.JSON(http.StatusOK, metrics)
 
 	default:
 		errMsg := fmt.Errorf("unexpected value type %T", res.Data.Result.Type())

packages/nomad/logs-collector.hcl

@@ -152,6 +152,7 @@ service = "{{ service }}"
 teamID = "{{ teamID }}"
 envID = "{{ envID }}"
 sandboxID = "{{ sandboxID }}"
+category = "{{ category }}"
 
 %{ if var.grafana_logs_endpoint != " " }
 [sinks.grafana]

packages/orchestrator/internal/sandbox/checks.go

@@ -16,7 +16,7 @@ import (
 const (
 	healthCheckInterval      = 10 * time.Second
 	metricsCheckInterval     = 5 * time.Second
-	minEnvdVersionForMetrcis = "0.1.3"
+	minEnvdVersionForMetrcis = "0.1.5"
 )
 
 func (s *Sandbox) logHeathAndUsage(ctx *utils.LockableCancelableContext) {