From 14e94154f75c2ba15cc9583cbd97531e742bf0ea Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Thu, 12 Dec 2024 14:46:00 -0600 Subject: [PATCH 01/15] Cap storage bandwidth (#15473) --- frigate/storage.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/frigate/storage.py b/frigate/storage.py index 2dbd07a510..1c4650271c 100644 --- a/frigate/storage.py +++ b/frigate/storage.py @@ -17,6 +17,8 @@ Recordings.end_time - Recordings.start_time ) +MAX_CALCULATED_BANDWIDTH = 10000 # 10Gb/hr + class StorageMaintainer(threading.Thread): """Maintain frigates recording storage.""" @@ -52,6 +54,12 @@ def calculate_camera_bandwidth(self) -> None: * 3600, 2, ) + + if bandwidth > MAX_CALCULATED_BANDWIDTH: + logger.warning( + f"{camera} has a bandwidth of {bandwidth} MB/hr which exceeds the expected maximum. This typically indicates an issue with the cameras recordings." + ) + bandwidth = MAX_CALCULATED_BANDWIDTH except TypeError: bandwidth = 0 From aa0348556a6d0214448d53a7984af125925fcbc7 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Thu, 12 Dec 2024 21:22:47 -0600 Subject: [PATCH 02/15] Cleanup handling of first object message (#15480) --- frigate/events/maintainer.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/frigate/events/maintainer.py b/frigate/events/maintainer.py index e2b9245d6e..68e7432abd 100644 --- a/frigate/events/maintainer.py +++ b/frigate/events/maintainer.py @@ -82,18 +82,23 @@ def run(self) -> None: ) if source_type == EventTypeEnum.tracked_object: + id = event_data["id"] self.timeline_queue.put( ( camera, source_type, event_type, - self.events_in_process.get(event_data["id"]), + self.events_in_process.get(id), event_data, ) ) - if event_type == EventStateEnum.start: - self.events_in_process[event_data["id"]] = event_data + # if this is the first message, just store it and continue, its not time to insert it in the db + if ( + event_type == EventStateEnum.start + or id not in self.events_in_process + ): + self.events_in_process[id] = event_data continue self.handle_object_detection(event_type, camera, event_data) @@ -123,10 +128,6 @@ def handle_object_detection( """handle tracked object event updates.""" updated_db = False - # if this is the first message, just store it and continue, its not time to insert it in the db - if event_type == EventStateEnum.start: - self.events_in_process[event_data["id"]] = event_data - if should_update_db(self.events_in_process[event_data["id"]], event_data): updated_db = True camera_config = self.config.cameras[camera] From 2bc160f9fb2374b8aab629e6a951dd886319075a Mon Sep 17 00:00:00 2001 From: Blake Blackshear Date: Fri, 13 Dec 2024 07:34:09 -0600 Subject: [PATCH 03/15] apply zizmor recommendations (#15490) --- .github/workflows/ci.yml | 16 ++++++++++++- .github/workflows/dependabot-auto-merge.yaml | 24 -------------------- .github/workflows/pull_request.yml | 12 +++++++++- .github/workflows/release.yml | 9 ++++++-- .github/workflows/stale.yml | 5 ++-- 5 files changed, 36 insertions(+), 30 deletions(-) delete mode 100644 .github/workflows/dependabot-auto-merge.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b06f0dc8f..996b1e8e75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: - dev - master paths-ignore: - - 'docs/**' + - "docs/**" # only run the latest commit to avoid cache overwrites concurrency: @@ -24,6 +24,8 @@ jobs: steps: - name: Check out code uses: actions/checkout@v4 + with: + persist-credentials: false - name: Set up QEMU and Buildx id: setup uses: ./.github/actions/setup @@ -45,6 +47,8 @@ jobs: steps: - name: Check out code uses: actions/checkout@v4 + with: + persist-credentials: false - name: Set up QEMU and Buildx id: setup uses: ./.github/actions/setup @@ -86,6 +90,8 @@ jobs: steps: - name: Check out code uses: actions/checkout@v4 + with: + persist-credentials: false - name: Set up QEMU and Buildx id: setup uses: ./.github/actions/setup @@ -112,6 +118,8 @@ jobs: steps: - name: Check out code uses: actions/checkout@v4 + with: + persist-credentials: false - name: Set up QEMU and Buildx id: setup uses: ./.github/actions/setup @@ -140,6 +148,8 @@ jobs: steps: - name: Check out code uses: actions/checkout@v4 + with: + persist-credentials: false - name: Set up QEMU and Buildx id: setup uses: ./.github/actions/setup @@ -165,6 +175,8 @@ jobs: steps: - name: Check out code uses: actions/checkout@v4 + with: + persist-credentials: false - name: Set up QEMU and Buildx id: setup uses: ./.github/actions/setup @@ -188,6 +200,8 @@ jobs: steps: - name: Check out code uses: actions/checkout@v4 + with: + persist-credentials: false - name: Set up QEMU and Buildx id: setup uses: ./.github/actions/setup diff --git a/.github/workflows/dependabot-auto-merge.yaml b/.github/workflows/dependabot-auto-merge.yaml deleted file mode 100644 index 1c047c346b..0000000000 --- a/.github/workflows/dependabot-auto-merge.yaml +++ /dev/null @@ -1,24 +0,0 @@ -name: dependabot-auto-merge -on: pull_request - -permissions: - contents: write - -jobs: - dependabot-auto-merge: - runs-on: ubuntu-latest - if: github.actor == 'dependabot[bot]' - steps: - - name: Get Dependabot metadata - id: metadata - uses: dependabot/fetch-metadata@v2 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Enable auto-merge for Dependabot PRs - if: steps.metadata.outputs.dependency-type == 'direct:development' && (steps.metadata.outputs.update-type == 'version-update:semver-minor' || steps.metadata.outputs.update-type == 'version-update:semver-patch') - run: | - gh pr review --approve "$PR_URL" - gh pr merge --auto --squash "$PR_URL" - env: - PR_URL: ${{ github.event.pull_request.html_url }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index bce97a07e9..39c76e3503 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -3,7 +3,7 @@ name: On pull request on: pull_request: paths-ignore: - - 'docs/**' + - "docs/**" env: DEFAULT_PYTHON: 3.9 @@ -19,6 +19,8 @@ jobs: DOCKER_BUILDKIT: "1" steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-node@master with: node-version: 16.x @@ -38,6 +40,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-node@master with: node-version: 16.x @@ -52,6 +56,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-node@master with: node-version: 20.x @@ -67,6 +73,8 @@ jobs: steps: - name: Check out the repository uses: actions/checkout@v4 + with: + persist-credentials: false - name: Set up Python ${{ env.DEFAULT_PYTHON }} uses: actions/setup-python@v5.1.0 with: @@ -88,6 +96,8 @@ jobs: steps: - name: Check out code uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-node@master with: node-version: 16.x diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6e90b9c784..ace4c3b3f4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,6 +11,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - id: lowercaseRepo uses: ASzc/change-string-case-action@v6 with: @@ -22,10 +24,13 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Create tag variables + env: + TAG: ${{ github.ref_name }} + LOWERCASE_REPO: ${{ steps.lowercaseRepo.outputs.lowercase }} run: | - BUILD_TYPE=$([[ "${{ github.ref_name }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && echo "stable" || echo "beta") + BUILD_TYPE=$([[ "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && echo "stable" || echo "beta") echo "BUILD_TYPE=${BUILD_TYPE}" >> $GITHUB_ENV - echo "BASE=ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}" >> $GITHUB_ENV + echo "BASE=ghcr.io/${LOWERCASE_REPO}" >> $GITHUB_ENV echo "BUILD_TAG=${GITHUB_SHA::7}" >> $GITHUB_ENV echo "CLEAN_VERSION=$(echo ${GITHUB_REF##*/} | tr '[:upper:]' '[:lower:]' | sed 's/^[v]//')" >> $GITHUB_ENV - name: Tag and push the main image diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 8e7e3223cb..011f70afd3 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -23,7 +23,9 @@ jobs: exempt-pr-labels: "pinned,security,dependencies" operations-per-run: 120 - name: Print outputs - run: echo ${{ join(steps.stale.outputs.*, ',') }} + env: + STALE_OUTPUT: ${{ join(steps.stale.outputs.*, ',') }} + run: echo "$STALE_OUTPUT" # clean_ghcr: # name: Delete outdated dev container images @@ -38,4 +40,3 @@ jobs: # account-type: personal # token: ${{ secrets.GITHUB_TOKEN }} # token-type: github-token - From 7e24f64007222e369b23875c44501be05467ebc4 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 13 Dec 2024 13:02:41 -0600 Subject: [PATCH 04/15] Improve the message for missing objects in review items (#15500) --- .../overlay/detail/ReviewDetailDialog.tsx | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/web/src/components/overlay/detail/ReviewDetailDialog.tsx b/web/src/components/overlay/detail/ReviewDetailDialog.tsx index c3e7ac91dd..d3c8864b77 100644 --- a/web/src/components/overlay/detail/ReviewDetailDialog.tsx +++ b/web/src/components/overlay/detail/ReviewDetailDialog.tsx @@ -74,6 +74,23 @@ export default function ReviewDetailDialog({ return events.length != review?.data.detections.length; }, [review, events]); + const missingObjects = useMemo(() => { + if (!review || !events) { + return []; + } + + const detectedIds = review.data.detections; + const missing = Array.from( + new Set( + events + .filter((event) => !detectedIds.includes(event.id)) + .map((event) => event.label), + ), + ); + + return missing; + }, [review, events]); + const formattedDate = useFormattedTimestamp( review?.start_time ?? 0, config?.ui.time_format == "24hour" @@ -263,8 +280,13 @@ export default function ReviewDetailDialog({ {hasMismatch && (
- Some objects that were detected are not included in this list - because the object does not have a snapshot + Some objects may have been detected in this review item that + did not qualify as an alert or detection. Adjust your + configuration if you want Frigate to save tracked objects for + any missing labels. + {missingObjects.length > 0 && ( +
{missingObjects.join(", ")}
+ )}
)}
From 5c371380f0a71927641af4bac941ed64a644965f Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 13 Dec 2024 13:52:56 -0600 Subject: [PATCH 05/15] Update iframe interval recommendation (#15501) * Update iframe interval recommendation * clarify * tweaks * wording --- docs/docs/configuration/live.md | 2 +- docs/docs/frigate/camera_setup.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/configuration/live.md b/docs/docs/configuration/live.md index 31e720031d..25bf7537c0 100644 --- a/docs/docs/configuration/live.md +++ b/docs/docs/configuration/live.md @@ -23,7 +23,7 @@ If you are using go2rtc, you should adjust the following settings in your camera - Video codec: **H.264** - provides the most compatible video codec with all Live view technologies and browsers. Avoid any kind of "smart codec" or "+" codec like _H.264+_ or _H.265+_. as these non-standard codecs remove keyframes (see below). - Audio codec: **AAC** - provides the most compatible audio codec with all Live view technologies and browsers that support audio. -- I-frame interval (sometimes called the keyframe interval, the interframe space, or the GOP length): match your camera's frame rate, or choose "1x" (for interframe space on Reolink cameras). For example, if your stream outputs 20fps, your i-frame interval should be 20 (or 1x on Reolink). Values higher than the frame rate will cause the stream to take longer to begin playback. See [this page](https://gardinal.net/understanding-the-keyframe-interval/) for more on keyframes. +- I-frame interval (sometimes called the keyframe interval, the interframe space, or the GOP length): match your camera's frame rate, or choose "1x" (for interframe space on Reolink cameras). For example, if your stream outputs 20fps, your i-frame interval should be 20 (or 1x on Reolink). Values higher than the frame rate will cause the stream to take longer to begin playback. See [this page](https://gardinal.net/understanding-the-keyframe-interval/) for more on keyframes. For many users this may not be an issue, but it should be noted that that a 1x i-frame interval will cause more storage utilization if you are using the stream for the `record` role as well. The default video and audio codec on your camera may not always be compatible with your browser, which is why setting them to H.264 and AAC is recommended. See the [go2rtc docs](https://github.com/AlexxIT/go2rtc?tab=readme-ov-file#codecs-madness) for codec support information. diff --git a/docs/docs/frigate/camera_setup.md b/docs/docs/frigate/camera_setup.md index 33ae24cabd..421046dd7c 100644 --- a/docs/docs/frigate/camera_setup.md +++ b/docs/docs/frigate/camera_setup.md @@ -28,7 +28,7 @@ For the Dahua/Loryta 5442 camera, I use the following settings: - Encode Mode: H.264 - Resolution: 2688\*1520 - Frame Rate(FPS): 15 -- I Frame Interval: 30 +- I Frame Interval: 30 (15 can also be used to prioritize streaming performance - see the [camera settings recommendations](../configuration/live) for more info) **Sub Stream (Detection)** From e78b857476f7a9dbecd69c499e1b0d33a78cc09d Mon Sep 17 00:00:00 2001 From: FL42 <46161216+fl42@users.noreply.github.com> Date: Sat, 14 Dec 2024 15:54:13 +0100 Subject: [PATCH 06/15] fix: use requests.Session() for DeepStack API (#15505) --- frigate/detectors/plugins/deepstack.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frigate/detectors/plugins/deepstack.py b/frigate/detectors/plugins/deepstack.py index 20d37fa8e9..e00a4e70d2 100644 --- a/frigate/detectors/plugins/deepstack.py +++ b/frigate/detectors/plugins/deepstack.py @@ -32,6 +32,7 @@ def __init__(self, detector_config: DeepstackDetectorConfig): self.api_timeout = detector_config.api_timeout self.api_key = detector_config.api_key self.labels = detector_config.model.merged_labelmap + self.session = requests.Session() def get_label_index(self, label_value): if label_value.lower() == "truck": @@ -51,7 +52,7 @@ def detect_raw(self, tensor_input): data = {"api_key": self.api_key} try: - response = requests.post( + response = self.session.post( self.api_url, data=data, files={"image": image_bytes}, From a91e4e7363c4020e68fbb70fa964c43102160f4c Mon Sep 17 00:00:00 2001 From: Mitch Ross Date: Sat, 14 Dec 2024 13:48:02 -0500 Subject: [PATCH 07/15] Add prometheus metrics --- docker/main/requirements-wheels.txt | 1 + frigate/api/app.py | 15 +- frigate/stats/emitter.py | 12 ++ frigate/stats/prometheus.py | 207 ++++++++++++++++++++++++++++ 4 files changed, 232 insertions(+), 3 deletions(-) create mode 100644 frigate/stats/prometheus.py diff --git a/docker/main/requirements-wheels.txt b/docker/main/requirements-wheels.txt index b163e8627f..5fb46a68f6 100644 --- a/docker/main/requirements-wheels.txt +++ b/docker/main/requirements-wheels.txt @@ -52,3 +52,4 @@ pywebpush == 2.0.* # alpr pyclipper == 1.3.* shapely == 2.0.* +prometheus-client == 0.21.* diff --git a/frigate/api/app.py b/frigate/api/app.py index a94c6415ca..6a89d43fb0 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -36,6 +36,9 @@ ) from frigate.version import VERSION +from prometheus_client import generate_latest, CONTENT_TYPE_LATEST +from fastapi.responses import Response + logger = logging.getLogger(__name__) @@ -105,6 +108,12 @@ def stats_history(request: Request, keys: str = None): return JSONResponse(content=request.app.stats_emitter.get_stats_history(keys)) +@router.get("/metrics") +def metrics(): + """Expose Prometheus metrics endpoint""" + return Response(content=generate_latest(), media_type=CONTENT_TYPE_LATEST) + + @router.get("/config") def config(request: Request): config_obj: FrigateConfig = request.app.frigate_config @@ -138,9 +147,9 @@ def config(request: Request): config["model"]["colormap"] = config_obj.model.colormap for detector_config in config["detectors"].values(): - detector_config["model"]["labelmap"] = ( - request.app.frigate_config.model.merged_labelmap - ) + detector_config["model"][ + "labelmap" + ] = request.app.frigate_config.model.merged_labelmap return JSONResponse(content=config) diff --git a/frigate/stats/emitter.py b/frigate/stats/emitter.py index 8a09ff51b9..2d288dcdf6 100644 --- a/frigate/stats/emitter.py +++ b/frigate/stats/emitter.py @@ -13,6 +13,8 @@ from frigate.const import FREQUENCY_STATS_POINTS from frigate.stats.util import stats_snapshot from frigate.types import StatsTrackingTypes +from frigate.stats.prometheus import update_metrics + logger = logging.getLogger(__name__) @@ -67,6 +69,16 @@ def get_stats_history( return selected_stats + def stats_init(config, camera_metrics, detectors, processes): + stats = { + "cameras": camera_metrics, + "detectors": detectors, + "processes": processes, + } + # Update Prometheus metrics with initial stats + update_metrics(stats) + return stats + def run(self) -> None: time.sleep(10) for counter in itertools.cycle( diff --git a/frigate/stats/prometheus.py b/frigate/stats/prometheus.py new file mode 100644 index 0000000000..59d9f10803 --- /dev/null +++ b/frigate/stats/prometheus.py @@ -0,0 +1,207 @@ +from prometheus_client import ( + Counter, + Gauge, + Histogram, + Info, + generate_latest, + CONTENT_TYPE_LATEST, +) +from typing import Dict + +# System metrics +SYSTEM_INFO = Info("frigate_system", "System information") +CPU_USAGE = Gauge( + "frigate_cpu_usage_percent", + "Process CPU usage %", + ["pid", "name", "process", "type", "cmdline"], +) +MEMORY_USAGE = Gauge( + "frigate_mem_usage_percent", + "Process memory usage %", + ["pid", "name", "process", "type", "cmdline"], +) + +# Camera metrics +CAMERA_FPS = Gauge( + "frigate_camera_fps", + "Frames per second being consumed from your camera", + ["camera_name"], +) +DETECTION_FPS = Gauge( + "frigate_detection_fps", + "Number of times detection is run per second", + ["camera_name"], +) +PROCESS_FPS = Gauge( + "frigate_process_fps", + "Frames per second being processed by frigate", + ["camera_name"], +) +SKIPPED_FPS = Gauge( + "frigate_skipped_fps", "Frames per second skipped for processing", ["camera_name"] +) +DETECTION_ENABLED = Gauge( + "frigate_detection_enabled", "Detection enabled for camera", ["camera_name"] +) +AUDIO_DBFS = Gauge("frigate_audio_dBFS", "Audio dBFS for camera", ["camera_name"]) +AUDIO_RMS = Gauge("frigate_audio_rms", "Audio RMS for camera", ["camera_name"]) + +# Detector metrics +DETECTOR_INFERENCE = Gauge( + "frigate_detector_inference_speed_seconds", + "Time spent running object detection in seconds", + ["name"], +) +DETECTOR_START = Gauge( + "frigate_detection_start", "Detector start time (unix timestamp)", ["name"] +) + +# GPU metrics +GPU_USAGE = Gauge("frigate_gpu_usage_percent", "GPU utilisation %", ["gpu_name"]) +GPU_MEMORY = Gauge("frigate_gpu_mem_usage_percent", "GPU memory usage %", ["gpu_name"]) + +# Storage metrics +STORAGE_FREE = Gauge("frigate_storage_free_bytes", "Storage free bytes", ["storage"]) +STORAGE_TOTAL = Gauge("frigate_storage_total_bytes", "Storage total bytes", ["storage"]) +STORAGE_USED = Gauge("frigate_storage_used_bytes", "Storage used bytes", ["storage"]) +STORAGE_MOUNT = Info( + "frigate_storage_mount_type", "Storage mount type", ["mount_type", "storage"] +) + +# Service metrics +UPTIME = Gauge("frigate_service_uptime_seconds", "Uptime seconds") +LAST_UPDATE = Gauge( + "frigate_service_last_updated_timestamp", "Stats recorded time (unix timestamp)" +) +TEMPERATURE = Gauge("frigate_device_temperature", "Device Temperature", ["device"]) + +# Event metrics +CAMERA_EVENTS = Counter( + "frigate_camera_events", + "Count of camera events since exporter started", + ["camera", "label"], +) + + +def update_metrics(stats: Dict) -> None: + """Update Prometheus metrics based on Frigate stats""" + try: + # Update process metrics + if "cpu_usages" in stats: + for pid, proc_stats in stats["cpu_usages"].items(): + cmdline = proc_stats.get("cmdline", "") + process_type = "Other" + process_name = cmdline + + CPU_USAGE.labels( + pid=pid, + name=process_name, + process=process_name, + type=process_type, + cmdline=cmdline, + ).set(float(proc_stats["cpu"])) + + MEMORY_USAGE.labels( + pid=pid, + name=process_name, + process=process_name, + type=process_type, + cmdline=cmdline, + ).set(float(proc_stats["mem"])) + + # Update camera metrics + if "cameras" in stats: + for camera_name, camera_stats in stats["cameras"].items(): + if "camera_fps" in camera_stats: + CAMERA_FPS.labels(camera_name=camera_name).set( + camera_stats["camera_fps"] + ) + if "detection_fps" in camera_stats: + DETECTION_FPS.labels(camera_name=camera_name).set( + camera_stats["detection_fps"] + ) + if "process_fps" in camera_stats: + PROCESS_FPS.labels(camera_name=camera_name).set( + camera_stats["process_fps"] + ) + if "skipped_fps" in camera_stats: + SKIPPED_FPS.labels(camera_name=camera_name).set( + camera_stats["skipped_fps"] + ) + if "detection_enabled" in camera_stats: + DETECTION_ENABLED.labels(camera_name=camera_name).set( + camera_stats["detection_enabled"] + ) + if "audio_dBFS" in camera_stats: + AUDIO_DBFS.labels(camera_name=camera_name).set( + camera_stats["audio_dBFS"] + ) + if "audio_rms" in camera_stats: + AUDIO_RMS.labels(camera_name=camera_name).set( + camera_stats["audio_rms"] + ) + + # Update detector metrics + if "detectors" in stats: + for name, detector in stats["detectors"].items(): + if "inference_speed" in detector: + DETECTOR_INFERENCE.labels(name=name).set( + detector["inference_speed"] * 0.001 + ) # ms to seconds + if "detection_start" in detector: + DETECTOR_START.labels(name=name).set(detector["detection_start"]) + + # Update GPU metrics + if "gpu_usages" in stats: + for gpu_name, gpu_stats in stats["gpu_usages"].items(): + if "gpu" in gpu_stats: + GPU_USAGE.labels(gpu_name=gpu_name).set(float(gpu_stats["gpu"])) + if "mem" in gpu_stats: + GPU_MEMORY.labels(gpu_name=gpu_name).set(float(gpu_stats["mem"])) + + # Update service metrics + if "service" in stats: + service = stats["service"] + + if "uptime" in service: + UPTIME.set(service["uptime"]) + if "last_updated" in service: + LAST_UPDATE.set(service["last_updated"]) + + # Storage metrics + if "storage" in service: + for path, storage in service["storage"].items(): + if "free" in storage: + STORAGE_FREE.labels(storage=path).set( + storage["free"] * 1e6 + ) # MB to bytes + if "total" in storage: + STORAGE_TOTAL.labels(storage=path).set(storage["total"] * 1e6) + if "used" in storage: + STORAGE_USED.labels(storage=path).set(storage["used"] * 1e6) + if "mount_type" in storage: + STORAGE_MOUNT.labels(storage=path).info( + {"mount_type": storage["mount_type"], "storage": path} + ) + + # Temperature metrics + if "temperatures" in service: + for device, temp in service["temperatures"].items(): + TEMPERATURE.labels(device=device).set(temp) + + # Version info + if "version" in service and "latest_version" in service: + SYSTEM_INFO.info( + { + "version": service["version"], + "latest_version": service["latest_version"], + } + ) + + except Exception as e: + print(f"Error updating Prometheus metrics: {str(e)}") + + +def get_metrics() -> tuple[str, str]: + """Get Prometheus metrics in text format""" + return generate_latest(), CONTENT_TYPE_LATEST From 362d7836dcc8fd76a45320254b9dac8af3748d84 Mon Sep 17 00:00:00 2001 From: Mitch Ross Date: Sat, 14 Dec 2024 13:50:13 -0500 Subject: [PATCH 08/15] lint --- frigate/api/app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frigate/api/app.py b/frigate/api/app.py index 6a89d43fb0..70053b8f2a 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -147,9 +147,9 @@ def config(request: Request): config["model"]["colormap"] = config_obj.model.colormap for detector_config in config["detectors"].values(): - detector_config["model"][ - "labelmap" - ] = request.app.frigate_config.model.merged_labelmap + detector_config["model"]["labelmap"] = ( + request.app.frigate_config.model.merged_labelmap + ) return JSONResponse(content=config) From 0b4085df71bb92e5ce0b29f5cb50c9408ec532e9 Mon Sep 17 00:00:00 2001 From: Mitch Ross Date: Sat, 14 Dec 2024 14:00:05 -0500 Subject: [PATCH 09/15] lint --- frigate/api/app.py | 4 +--- frigate/stats/emitter.py | 3 +-- frigate/stats/prometheus.py | 6 +++--- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/frigate/api/app.py b/frigate/api/app.py index 70053b8f2a..6fc9ba9e41 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -16,6 +16,7 @@ from fastapi.responses import JSONResponse, PlainTextResponse from markupsafe import escape from peewee import operator +from prometheus_client import CONTENT_TYPE_LATEST, generate_latest from frigate.api.defs.query.app_query_parameters import AppTimelineHourlyQueryParameters from frigate.api.defs.request.app_body import AppConfigSetBody @@ -36,9 +37,6 @@ ) from frigate.version import VERSION -from prometheus_client import generate_latest, CONTENT_TYPE_LATEST -from fastapi.responses import Response - logger = logging.getLogger(__name__) diff --git a/frigate/stats/emitter.py b/frigate/stats/emitter.py index 2d288dcdf6..022e992138 100644 --- a/frigate/stats/emitter.py +++ b/frigate/stats/emitter.py @@ -11,10 +11,9 @@ from frigate.comms.inter_process import InterProcessRequestor from frigate.config import FrigateConfig from frigate.const import FREQUENCY_STATS_POINTS +from frigate.stats.prometheus import update_metrics from frigate.stats.util import stats_snapshot from frigate.types import StatsTrackingTypes -from frigate.stats.prometheus import update_metrics - logger = logging.getLogger(__name__) diff --git a/frigate/stats/prometheus.py b/frigate/stats/prometheus.py index 59d9f10803..a43c091e27 100644 --- a/frigate/stats/prometheus.py +++ b/frigate/stats/prometheus.py @@ -1,12 +1,12 @@ +from typing import Dict + from prometheus_client import ( + CONTENT_TYPE_LATEST, Counter, Gauge, - Histogram, Info, generate_latest, - CONTENT_TYPE_LATEST, ) -from typing import Dict # System metrics SYSTEM_INFO = Info("frigate_system", "System information") From d70b82630cbf435963abd7343318d9bc23173dbd Mon Sep 17 00:00:00 2001 From: Mitch Ross Date: Sun, 15 Dec 2024 01:06:10 -0500 Subject: [PATCH 10/15] add docs for metrics --- docs/docs/configuration/metrics.md | 99 ++++++++++++++++++++++++++++++ docs/sidebars.ts | 1 + 2 files changed, 100 insertions(+) create mode 100644 docs/docs/configuration/metrics.md diff --git a/docs/docs/configuration/metrics.md b/docs/docs/configuration/metrics.md new file mode 100644 index 0000000000..a12238f0a0 --- /dev/null +++ b/docs/docs/configuration/metrics.md @@ -0,0 +1,99 @@ +--- +id: metrics +title: Metrics +--- + +# Metrics + +Frigate exposes Prometheus metrics at the `/metrics` endpoint that can be used to monitor the performance and health of your Frigate instance. + +## Available Metrics + +### System Metrics +- `frigate_cpu_usage_percent{pid="", name="", process="", type="", cmdline=""}` - Process CPU usage percentage +- `frigate_mem_usage_percent{pid="", name="", process="", type="", cmdline=""}` - Process memory usage percentage +- `frigate_gpu_usage_percent{gpu_name=""}` - GPU utilization percentage +- `frigate_gpu_mem_usage_percent{gpu_name=""}` - GPU memory usage percentage + +### Camera Metrics +- `frigate_camera_fps{camera_name=""}` - Frames per second being consumed from your camera +- `frigate_detection_fps{camera_name=""}` - Number of times detection is run per second +- `frigate_process_fps{camera_name=""}` - Frames per second being processed +- `frigate_skipped_fps{camera_name=""}` - Frames per second skipped for processing +- `frigate_detection_enabled{camera_name=""}` - Detection enabled status for camera +- `frigate_audio_dBFS{camera_name=""}` - Audio dBFS for camera +- `frigate_audio_rms{camera_name=""}` - Audio RMS for camera + +### Detector Metrics +- `frigate_detector_inference_speed_seconds{name=""}` - Time spent running object detection in seconds +- `frigate_detection_start{name=""}` - Detector start time (unix timestamp) + +### Storage Metrics +- `frigate_storage_free_bytes{storage=""}` - Storage free bytes +- `frigate_storage_total_bytes{storage=""}` - Storage total bytes +- `frigate_storage_used_bytes{storage=""}` - Storage used bytes +- `frigate_storage_mount_type{mount_type="", storage=""}` - Storage mount type info + +### Service Metrics +- `frigate_service_uptime_seconds` - Uptime in seconds +- `frigate_service_last_updated_timestamp` - Stats recorded time (unix timestamp) +- `frigate_device_temperature{device=""}` - Device Temperature + +### Event Metrics +- `frigate_camera_events{camera="", label=""}` - Count of camera events since exporter started + +## Configuring Prometheus + +To scrape metrics from Frigate, add the following to your Prometheus configuration: + +```yaml +scrape_configs: + - job_name: 'frigate' + metrics_path: '/metrics' + static_configs: + - targets: ['frigate:5000'] + scrape_interval: 15s +``` + +## Example Queries + +Here are some example PromQL queries that might be useful: + +```promql +# Average CPU usage across all processes +avg(frigate_cpu_usage_percent) + +# Total GPU memory usage +sum(frigate_gpu_mem_usage_percent) + +# Detection FPS by camera +rate(frigate_detection_fps{camera_name="front_door"}[5m]) + +# Storage usage percentage +(frigate_storage_used_bytes / frigate_storage_total_bytes) * 100 + +# Event count by camera in last hour +increase(frigate_camera_events[1h]) +``` + +## Grafana Dashboard + +You can use these metrics to create Grafana dashboards to monitor your Frigate instance. Here's an example of metrics you might want to track: + +- CPU, Memory and GPU usage over time +- Camera FPS and detection rates +- Storage usage and trends +- Event counts by camera +- System temperatures + +A sample Grafana dashboard JSON will be provided in a future update. + +## Metric Types + +The metrics exposed by Frigate use the following Prometheus metric types: + +- **Counter**: Cumulative values that only increase (e.g., `frigate_camera_events`) +- **Gauge**: Values that can go up and down (e.g., `frigate_cpu_usage_percent`) +- **Info**: Key-value pairs for metadata (e.g., `frigate_storage_mount_type`) + +For more information about Prometheus metric types, see the [Prometheus documentation](https://prometheus.io/docs/concepts/metric_types/). \ No newline at end of file diff --git a/docs/sidebars.ts b/docs/sidebars.ts index b0b8cdf486..90cbcc828c 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -22,6 +22,7 @@ const sidebars: SidebarsConfig = { Configuration: { 'Configuration Files': [ 'configuration/index', + 'configuration/metrics', 'configuration/reference', { type: 'link', From 238e3ab3fcd43800aae2b13da6b3051056b1688a Mon Sep 17 00:00:00 2001 From: Mitch Ross Date: Sun, 15 Dec 2024 12:58:03 -0500 Subject: [PATCH 11/15] move sidebars --- docs/sidebars.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sidebars.ts b/docs/sidebars.ts index 90cbcc828c..f8f47f592d 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -22,7 +22,6 @@ const sidebars: SidebarsConfig = { Configuration: { 'Configuration Files': [ 'configuration/index', - 'configuration/metrics', 'configuration/reference', { type: 'link', @@ -85,6 +84,7 @@ const sidebars: SidebarsConfig = { items: frigateHttpApiSidebar, }, 'integrations/mqtt', + 'configuration/metrics', 'integrations/third_party_extensions', ], 'Frigate+': [ From fe1d27381b1c97fea73fac0afd3cde015b66bef5 Mon Sep 17 00:00:00 2001 From: Mitch Ross Date: Sat, 14 Dec 2024 13:48:02 -0500 Subject: [PATCH 12/15] Add prometheus metrics --- docker/main/requirements-wheels.txt | 3 --- frigate/api/app.py | 9 ++++++--- frigate/stats/emitter.py | 2 ++ frigate/stats/prometheus.py | 6 +++--- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docker/main/requirements-wheels.txt b/docker/main/requirements-wheels.txt index 5fb46a68f6..7f67cb732c 100644 --- a/docker/main/requirements-wheels.txt +++ b/docker/main/requirements-wheels.txt @@ -49,7 +49,4 @@ openai == 1.51.* # push notifications py-vapid == 1.9.* pywebpush == 2.0.* -# alpr -pyclipper == 1.3.* -shapely == 2.0.* prometheus-client == 0.21.* diff --git a/frigate/api/app.py b/frigate/api/app.py index 6fc9ba9e41..a8e4ac9d48 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -37,6 +37,9 @@ ) from frigate.version import VERSION +from prometheus_client import generate_latest, CONTENT_TYPE_LATEST +from fastapi.responses import Response + logger = logging.getLogger(__name__) @@ -145,9 +148,9 @@ def config(request: Request): config["model"]["colormap"] = config_obj.model.colormap for detector_config in config["detectors"].values(): - detector_config["model"]["labelmap"] = ( - request.app.frigate_config.model.merged_labelmap - ) + detector_config["model"][ + "labelmap" + ] = request.app.frigate_config.model.merged_labelmap return JSONResponse(content=config) diff --git a/frigate/stats/emitter.py b/frigate/stats/emitter.py index 022e992138..d607e2a477 100644 --- a/frigate/stats/emitter.py +++ b/frigate/stats/emitter.py @@ -14,6 +14,8 @@ from frigate.stats.prometheus import update_metrics from frigate.stats.util import stats_snapshot from frigate.types import StatsTrackingTypes +from frigate.stats.prometheus import update_metrics + logger = logging.getLogger(__name__) diff --git a/frigate/stats/prometheus.py b/frigate/stats/prometheus.py index a43c091e27..59d9f10803 100644 --- a/frigate/stats/prometheus.py +++ b/frigate/stats/prometheus.py @@ -1,12 +1,12 @@ -from typing import Dict - from prometheus_client import ( - CONTENT_TYPE_LATEST, Counter, Gauge, + Histogram, Info, generate_latest, + CONTENT_TYPE_LATEST, ) +from typing import Dict # System metrics SYSTEM_INFO = Info("frigate_system", "System information") From 97049212c62f2615c04a9e4923dbd8843dec2aa9 Mon Sep 17 00:00:00 2001 From: Mitch Ross Date: Sat, 14 Dec 2024 13:50:13 -0500 Subject: [PATCH 13/15] lint --- frigate/api/app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frigate/api/app.py b/frigate/api/app.py index a8e4ac9d48..ec6a93de9c 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -148,9 +148,9 @@ def config(request: Request): config["model"]["colormap"] = config_obj.model.colormap for detector_config in config["detectors"].values(): - detector_config["model"][ - "labelmap" - ] = request.app.frigate_config.model.merged_labelmap + detector_config["model"]["labelmap"] = ( + request.app.frigate_config.model.merged_labelmap + ) return JSONResponse(content=config) From 66c718573c0e1e8dc44fad2dae2faf75d769e6eb Mon Sep 17 00:00:00 2001 From: Mitch Ross Date: Sat, 14 Dec 2024 14:00:05 -0500 Subject: [PATCH 14/15] lint --- frigate/api/app.py | 3 --- frigate/stats/emitter.py | 2 -- frigate/stats/prometheus.py | 6 +++--- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/frigate/api/app.py b/frigate/api/app.py index ec6a93de9c..6fc9ba9e41 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -37,9 +37,6 @@ ) from frigate.version import VERSION -from prometheus_client import generate_latest, CONTENT_TYPE_LATEST -from fastapi.responses import Response - logger = logging.getLogger(__name__) diff --git a/frigate/stats/emitter.py b/frigate/stats/emitter.py index d607e2a477..022e992138 100644 --- a/frigate/stats/emitter.py +++ b/frigate/stats/emitter.py @@ -14,8 +14,6 @@ from frigate.stats.prometheus import update_metrics from frigate.stats.util import stats_snapshot from frigate.types import StatsTrackingTypes -from frigate.stats.prometheus import update_metrics - logger = logging.getLogger(__name__) diff --git a/frigate/stats/prometheus.py b/frigate/stats/prometheus.py index 59d9f10803..a43c091e27 100644 --- a/frigate/stats/prometheus.py +++ b/frigate/stats/prometheus.py @@ -1,12 +1,12 @@ +from typing import Dict + from prometheus_client import ( + CONTENT_TYPE_LATEST, Counter, Gauge, - Histogram, Info, generate_latest, - CONTENT_TYPE_LATEST, ) -from typing import Dict # System metrics SYSTEM_INFO = Info("frigate_system", "System information") From d5a4cf335325cb7199cfd552f01c0e9281539e35 Mon Sep 17 00:00:00 2001 From: Mitch Ross Date: Sun, 15 Dec 2024 01:06:10 -0500 Subject: [PATCH 15/15] add docs for metrics --- docs/sidebars.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/sidebars.ts b/docs/sidebars.ts index f8f47f592d..5566619905 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -22,6 +22,7 @@ const sidebars: SidebarsConfig = { Configuration: { 'Configuration Files': [ 'configuration/index', + 'configuration/metrics', 'configuration/reference', { type: 'link',