From d55cc8446eb2c7e03eacaafeab705ad46c89d678 Mon Sep 17 00:00:00 2001 From: Mitch Ross Date: Sat, 14 Dec 2024 13:48:02 -0500 Subject: [PATCH 1/5] Add prometheus metrics --- docker/main/requirements-wheels.txt | 1 + frigate/api/app.py | 15 +- frigate/stats/emitter.py | 12 ++ frigate/stats/prometheus.py | 207 ++++++++++++++++++++++++++++ 4 files changed, 232 insertions(+), 3 deletions(-) create mode 100644 frigate/stats/prometheus.py diff --git a/docker/main/requirements-wheels.txt b/docker/main/requirements-wheels.txt index b163e8627f..5fb46a68f6 100644 --- a/docker/main/requirements-wheels.txt +++ b/docker/main/requirements-wheels.txt @@ -52,3 +52,4 @@ pywebpush == 2.0.* # alpr pyclipper == 1.3.* shapely == 2.0.* +prometheus-client == 0.21.* diff --git a/frigate/api/app.py b/frigate/api/app.py index a94c6415ca..6a89d43fb0 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -36,6 +36,9 @@ ) from frigate.version import VERSION +from prometheus_client import generate_latest, CONTENT_TYPE_LATEST +from fastapi.responses import Response + logger = logging.getLogger(__name__) @@ -105,6 +108,12 @@ def stats_history(request: Request, keys: str = None): return JSONResponse(content=request.app.stats_emitter.get_stats_history(keys)) +@router.get("/metrics") +def metrics(): + """Expose Prometheus metrics endpoint""" + return Response(content=generate_latest(), media_type=CONTENT_TYPE_LATEST) + + @router.get("/config") def config(request: Request): config_obj: FrigateConfig = request.app.frigate_config @@ -138,9 +147,9 @@ def config(request: Request): config["model"]["colormap"] = config_obj.model.colormap for detector_config in config["detectors"].values(): - detector_config["model"]["labelmap"] = ( - request.app.frigate_config.model.merged_labelmap - ) + detector_config["model"][ + "labelmap" + ] = request.app.frigate_config.model.merged_labelmap return JSONResponse(content=config) diff --git a/frigate/stats/emitter.py b/frigate/stats/emitter.py index 8a09ff51b9..2d288dcdf6 100644 --- a/frigate/stats/emitter.py +++ b/frigate/stats/emitter.py @@ -13,6 +13,8 @@ from frigate.const import FREQUENCY_STATS_POINTS from frigate.stats.util import stats_snapshot from frigate.types import StatsTrackingTypes +from frigate.stats.prometheus import update_metrics + logger = logging.getLogger(__name__) @@ -67,6 +69,16 @@ def get_stats_history( return selected_stats + def stats_init(config, camera_metrics, detectors, processes): + stats = { + "cameras": camera_metrics, + "detectors": detectors, + "processes": processes, + } + # Update Prometheus metrics with initial stats + update_metrics(stats) + return stats + def run(self) -> None: time.sleep(10) for counter in itertools.cycle( diff --git a/frigate/stats/prometheus.py b/frigate/stats/prometheus.py new file mode 100644 index 0000000000..59d9f10803 --- /dev/null +++ b/frigate/stats/prometheus.py @@ -0,0 +1,207 @@ +from prometheus_client import ( + Counter, + Gauge, + Histogram, + Info, + generate_latest, + CONTENT_TYPE_LATEST, +) +from typing import Dict + +# System metrics +SYSTEM_INFO = Info("frigate_system", "System information") +CPU_USAGE = Gauge( + "frigate_cpu_usage_percent", + "Process CPU usage %", + ["pid", "name", "process", "type", "cmdline"], +) +MEMORY_USAGE = Gauge( + "frigate_mem_usage_percent", + "Process memory usage %", + ["pid", "name", "process", "type", "cmdline"], +) + +# Camera metrics +CAMERA_FPS = Gauge( + "frigate_camera_fps", + "Frames per second being consumed from your camera", + ["camera_name"], +) +DETECTION_FPS = Gauge( + "frigate_detection_fps", + "Number of times detection is run per second", + ["camera_name"], +) +PROCESS_FPS = Gauge( + "frigate_process_fps", + "Frames per second being processed by frigate", + ["camera_name"], +) +SKIPPED_FPS = Gauge( + "frigate_skipped_fps", "Frames per second skipped for processing", ["camera_name"] +) +DETECTION_ENABLED = Gauge( + "frigate_detection_enabled", "Detection enabled for camera", ["camera_name"] +) +AUDIO_DBFS = Gauge("frigate_audio_dBFS", "Audio dBFS for camera", ["camera_name"]) +AUDIO_RMS = Gauge("frigate_audio_rms", "Audio RMS for camera", ["camera_name"]) + +# Detector metrics +DETECTOR_INFERENCE = Gauge( + "frigate_detector_inference_speed_seconds", + "Time spent running object detection in seconds", + ["name"], +) +DETECTOR_START = Gauge( + "frigate_detection_start", "Detector start time (unix timestamp)", ["name"] +) + +# GPU metrics +GPU_USAGE = Gauge("frigate_gpu_usage_percent", "GPU utilisation %", ["gpu_name"]) +GPU_MEMORY = Gauge("frigate_gpu_mem_usage_percent", "GPU memory usage %", ["gpu_name"]) + +# Storage metrics +STORAGE_FREE = Gauge("frigate_storage_free_bytes", "Storage free bytes", ["storage"]) +STORAGE_TOTAL = Gauge("frigate_storage_total_bytes", "Storage total bytes", ["storage"]) +STORAGE_USED = Gauge("frigate_storage_used_bytes", "Storage used bytes", ["storage"]) +STORAGE_MOUNT = Info( + "frigate_storage_mount_type", "Storage mount type", ["mount_type", "storage"] +) + +# Service metrics +UPTIME = Gauge("frigate_service_uptime_seconds", "Uptime seconds") +LAST_UPDATE = Gauge( + "frigate_service_last_updated_timestamp", "Stats recorded time (unix timestamp)" +) +TEMPERATURE = Gauge("frigate_device_temperature", "Device Temperature", ["device"]) + +# Event metrics +CAMERA_EVENTS = Counter( + "frigate_camera_events", + "Count of camera events since exporter started", + ["camera", "label"], +) + + +def update_metrics(stats: Dict) -> None: + """Update Prometheus metrics based on Frigate stats""" + try: + # Update process metrics + if "cpu_usages" in stats: + for pid, proc_stats in stats["cpu_usages"].items(): + cmdline = proc_stats.get("cmdline", "") + process_type = "Other" + process_name = cmdline + + CPU_USAGE.labels( + pid=pid, + name=process_name, + process=process_name, + type=process_type, + cmdline=cmdline, + ).set(float(proc_stats["cpu"])) + + MEMORY_USAGE.labels( + pid=pid, + name=process_name, + process=process_name, + type=process_type, + cmdline=cmdline, + ).set(float(proc_stats["mem"])) + + # Update camera metrics + if "cameras" in stats: + for camera_name, camera_stats in stats["cameras"].items(): + if "camera_fps" in camera_stats: + CAMERA_FPS.labels(camera_name=camera_name).set( + camera_stats["camera_fps"] + ) + if "detection_fps" in camera_stats: + DETECTION_FPS.labels(camera_name=camera_name).set( + camera_stats["detection_fps"] + ) + if "process_fps" in camera_stats: + PROCESS_FPS.labels(camera_name=camera_name).set( + camera_stats["process_fps"] + ) + if "skipped_fps" in camera_stats: + SKIPPED_FPS.labels(camera_name=camera_name).set( + camera_stats["skipped_fps"] + ) + if "detection_enabled" in camera_stats: + DETECTION_ENABLED.labels(camera_name=camera_name).set( + camera_stats["detection_enabled"] + ) + if "audio_dBFS" in camera_stats: + AUDIO_DBFS.labels(camera_name=camera_name).set( + camera_stats["audio_dBFS"] + ) + if "audio_rms" in camera_stats: + AUDIO_RMS.labels(camera_name=camera_name).set( + camera_stats["audio_rms"] + ) + + # Update detector metrics + if "detectors" in stats: + for name, detector in stats["detectors"].items(): + if "inference_speed" in detector: + DETECTOR_INFERENCE.labels(name=name).set( + detector["inference_speed"] * 0.001 + ) # ms to seconds + if "detection_start" in detector: + DETECTOR_START.labels(name=name).set(detector["detection_start"]) + + # Update GPU metrics + if "gpu_usages" in stats: + for gpu_name, gpu_stats in stats["gpu_usages"].items(): + if "gpu" in gpu_stats: + GPU_USAGE.labels(gpu_name=gpu_name).set(float(gpu_stats["gpu"])) + if "mem" in gpu_stats: + GPU_MEMORY.labels(gpu_name=gpu_name).set(float(gpu_stats["mem"])) + + # Update service metrics + if "service" in stats: + service = stats["service"] + + if "uptime" in service: + UPTIME.set(service["uptime"]) + if "last_updated" in service: + LAST_UPDATE.set(service["last_updated"]) + + # Storage metrics + if "storage" in service: + for path, storage in service["storage"].items(): + if "free" in storage: + STORAGE_FREE.labels(storage=path).set( + storage["free"] * 1e6 + ) # MB to bytes + if "total" in storage: + STORAGE_TOTAL.labels(storage=path).set(storage["total"] * 1e6) + if "used" in storage: + STORAGE_USED.labels(storage=path).set(storage["used"] * 1e6) + if "mount_type" in storage: + STORAGE_MOUNT.labels(storage=path).info( + {"mount_type": storage["mount_type"], "storage": path} + ) + + # Temperature metrics + if "temperatures" in service: + for device, temp in service["temperatures"].items(): + TEMPERATURE.labels(device=device).set(temp) + + # Version info + if "version" in service and "latest_version" in service: + SYSTEM_INFO.info( + { + "version": service["version"], + "latest_version": service["latest_version"], + } + ) + + except Exception as e: + print(f"Error updating Prometheus metrics: {str(e)}") + + +def get_metrics() -> tuple[str, str]: + """Get Prometheus metrics in text format""" + return generate_latest(), CONTENT_TYPE_LATEST From 64cf6595c51086801bd0cbc68e691ab1d55c0ae2 Mon Sep 17 00:00:00 2001 From: Mitch Ross Date: Sun, 15 Dec 2024 01:06:10 -0500 Subject: [PATCH 2/5] add docs for metrics --- docs/docs/configuration/metrics.md | 99 ++++++++++++++++++++++++++++++ docs/sidebars.ts | 1 + 2 files changed, 100 insertions(+) create mode 100644 docs/docs/configuration/metrics.md diff --git a/docs/docs/configuration/metrics.md b/docs/docs/configuration/metrics.md new file mode 100644 index 0000000000..a12238f0a0 --- /dev/null +++ b/docs/docs/configuration/metrics.md @@ -0,0 +1,99 @@ +--- +id: metrics +title: Metrics +--- + +# Metrics + +Frigate exposes Prometheus metrics at the `/metrics` endpoint that can be used to monitor the performance and health of your Frigate instance. + +## Available Metrics + +### System Metrics +- `frigate_cpu_usage_percent{pid="", name="", process="", type="", cmdline=""}` - Process CPU usage percentage +- `frigate_mem_usage_percent{pid="", name="", process="", type="", cmdline=""}` - Process memory usage percentage +- `frigate_gpu_usage_percent{gpu_name=""}` - GPU utilization percentage +- `frigate_gpu_mem_usage_percent{gpu_name=""}` - GPU memory usage percentage + +### Camera Metrics +- `frigate_camera_fps{camera_name=""}` - Frames per second being consumed from your camera +- `frigate_detection_fps{camera_name=""}` - Number of times detection is run per second +- `frigate_process_fps{camera_name=""}` - Frames per second being processed +- `frigate_skipped_fps{camera_name=""}` - Frames per second skipped for processing +- `frigate_detection_enabled{camera_name=""}` - Detection enabled status for camera +- `frigate_audio_dBFS{camera_name=""}` - Audio dBFS for camera +- `frigate_audio_rms{camera_name=""}` - Audio RMS for camera + +### Detector Metrics +- `frigate_detector_inference_speed_seconds{name=""}` - Time spent running object detection in seconds +- `frigate_detection_start{name=""}` - Detector start time (unix timestamp) + +### Storage Metrics +- `frigate_storage_free_bytes{storage=""}` - Storage free bytes +- `frigate_storage_total_bytes{storage=""}` - Storage total bytes +- `frigate_storage_used_bytes{storage=""}` - Storage used bytes +- `frigate_storage_mount_type{mount_type="", storage=""}` - Storage mount type info + +### Service Metrics +- `frigate_service_uptime_seconds` - Uptime in seconds +- `frigate_service_last_updated_timestamp` - Stats recorded time (unix timestamp) +- `frigate_device_temperature{device=""}` - Device Temperature + +### Event Metrics +- `frigate_camera_events{camera="", label=""}` - Count of camera events since exporter started + +## Configuring Prometheus + +To scrape metrics from Frigate, add the following to your Prometheus configuration: + +```yaml +scrape_configs: + - job_name: 'frigate' + metrics_path: '/metrics' + static_configs: + - targets: ['frigate:5000'] + scrape_interval: 15s +``` + +## Example Queries + +Here are some example PromQL queries that might be useful: + +```promql +# Average CPU usage across all processes +avg(frigate_cpu_usage_percent) + +# Total GPU memory usage +sum(frigate_gpu_mem_usage_percent) + +# Detection FPS by camera +rate(frigate_detection_fps{camera_name="front_door"}[5m]) + +# Storage usage percentage +(frigate_storage_used_bytes / frigate_storage_total_bytes) * 100 + +# Event count by camera in last hour +increase(frigate_camera_events[1h]) +``` + +## Grafana Dashboard + +You can use these metrics to create Grafana dashboards to monitor your Frigate instance. Here's an example of metrics you might want to track: + +- CPU, Memory and GPU usage over time +- Camera FPS and detection rates +- Storage usage and trends +- Event counts by camera +- System temperatures + +A sample Grafana dashboard JSON will be provided in a future update. + +## Metric Types + +The metrics exposed by Frigate use the following Prometheus metric types: + +- **Counter**: Cumulative values that only increase (e.g., `frigate_camera_events`) +- **Gauge**: Values that can go up and down (e.g., `frigate_cpu_usage_percent`) +- **Info**: Key-value pairs for metadata (e.g., `frigate_storage_mount_type`) + +For more information about Prometheus metric types, see the [Prometheus documentation](https://prometheus.io/docs/concepts/metric_types/). \ No newline at end of file diff --git a/docs/sidebars.ts b/docs/sidebars.ts index b0b8cdf486..90cbcc828c 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -22,6 +22,7 @@ const sidebars: SidebarsConfig = { Configuration: { 'Configuration Files': [ 'configuration/index', + 'configuration/metrics', 'configuration/reference', { type: 'link', From d05ad07621b1468b3fe3e8f4c326ccd71ec60d9f Mon Sep 17 00:00:00 2001 From: Mitch Ross Date: Sun, 15 Dec 2024 13:11:52 -0500 Subject: [PATCH 3/5] sidebar --- docs/sidebars.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sidebars.ts b/docs/sidebars.ts index 90cbcc828c..f8f47f592d 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -22,7 +22,6 @@ const sidebars: SidebarsConfig = { Configuration: { 'Configuration Files': [ 'configuration/index', - 'configuration/metrics', 'configuration/reference', { type: 'link', @@ -85,6 +84,7 @@ const sidebars: SidebarsConfig = { items: frigateHttpApiSidebar, }, 'integrations/mqtt', + 'configuration/metrics', 'integrations/third_party_extensions', ], 'Frigate+': [ From 5be6fe1cfbe381202998bac8be20f48beba1124e Mon Sep 17 00:00:00 2001 From: Mitch Ross Date: Sun, 15 Dec 2024 13:17:43 -0500 Subject: [PATCH 4/5] lint --- frigate/api/app.py | 6 +++--- frigate/stats/prometheus.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/frigate/api/app.py b/frigate/api/app.py index 6a89d43fb0..70053b8f2a 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -147,9 +147,9 @@ def config(request: Request): config["model"]["colormap"] = config_obj.model.colormap for detector_config in config["detectors"].values(): - detector_config["model"][ - "labelmap" - ] = request.app.frigate_config.model.merged_labelmap + detector_config["model"]["labelmap"] = ( + request.app.frigate_config.model.merged_labelmap + ) return JSONResponse(content=config) diff --git a/frigate/stats/prometheus.py b/frigate/stats/prometheus.py index 59d9f10803..8e6b42f1ed 100644 --- a/frigate/stats/prometheus.py +++ b/frigate/stats/prometheus.py @@ -1,7 +1,6 @@ from prometheus_client import ( Counter, Gauge, - Histogram, Info, generate_latest, CONTENT_TYPE_LATEST, From 2122a2a6aed148edfc22e13f58844c0ae264881b Mon Sep 17 00:00:00 2001 From: Mitch Ross Date: Sun, 15 Dec 2024 13:18:46 -0500 Subject: [PATCH 5/5] lint --- frigate/api/app.py | 4 +--- frigate/stats/emitter.py | 3 +-- frigate/stats/prometheus.py | 5 +++-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/frigate/api/app.py b/frigate/api/app.py index 70053b8f2a..6fc9ba9e41 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -16,6 +16,7 @@ from fastapi.responses import JSONResponse, PlainTextResponse from markupsafe import escape from peewee import operator +from prometheus_client import CONTENT_TYPE_LATEST, generate_latest from frigate.api.defs.query.app_query_parameters import AppTimelineHourlyQueryParameters from frigate.api.defs.request.app_body import AppConfigSetBody @@ -36,9 +37,6 @@ ) from frigate.version import VERSION -from prometheus_client import generate_latest, CONTENT_TYPE_LATEST -from fastapi.responses import Response - logger = logging.getLogger(__name__) diff --git a/frigate/stats/emitter.py b/frigate/stats/emitter.py index 2d288dcdf6..022e992138 100644 --- a/frigate/stats/emitter.py +++ b/frigate/stats/emitter.py @@ -11,10 +11,9 @@ from frigate.comms.inter_process import InterProcessRequestor from frigate.config import FrigateConfig from frigate.const import FREQUENCY_STATS_POINTS +from frigate.stats.prometheus import update_metrics from frigate.stats.util import stats_snapshot from frigate.types import StatsTrackingTypes -from frigate.stats.prometheus import update_metrics - logger = logging.getLogger(__name__) diff --git a/frigate/stats/prometheus.py b/frigate/stats/prometheus.py index 8e6b42f1ed..a43c091e27 100644 --- a/frigate/stats/prometheus.py +++ b/frigate/stats/prometheus.py @@ -1,11 +1,12 @@ +from typing import Dict + from prometheus_client import ( + CONTENT_TYPE_LATEST, Counter, Gauge, Info, generate_latest, - CONTENT_TYPE_LATEST, ) -from typing import Dict # System metrics SYSTEM_INFO = Info("frigate_system", "System information")