diff --git a/Dockerfile.cpu b/Dockerfile.cpu new file mode 100644 index 00000000..e4e1ad43 --- /dev/null +++ b/Dockerfile.cpu @@ -0,0 +1,23 @@ +FROM python:3.9-slim + +WORKDIR /app + +# Install dependencies +RUN apt-get update && apt-get install --no-install-recommends -y \ + apt-transport-https \ + ca-certificates \ + software-properties-common \ + && rm -rf /var/lib/apt/lists/* + +# Install mosquitto-clients +RUN apt-get update && apt-get install --no-install-recommends -y mosquitto-clients \ + && rm -rf /var/lib/apt/lists/* + +# Install Python packages +COPY requirements.txt /app/ +RUN pip install --no-cache-dir -r /app/requirements.txt + +# Copy the application code +COPY . /app/ + +CMD ["bash", "-c", "python3 mqtt/publisher_intel.py"] diff --git a/Dockerfile.fps b/Dockerfile.fps new file mode 100644 index 00000000..917392a4 --- /dev/null +++ b/Dockerfile.fps @@ -0,0 +1,24 @@ +# Dockerfile.mqtt +FROM python:3.9-slim + +WORKDIR /app + +# Install dependencies +RUN apt-get update && apt-get install --no-install-recommends -y \ + apt-transport-https \ + ca-certificates \ + software-properties-common \ + && rm -rf /var/lib/apt/lists/* + +# Install mosquitto-clients +RUN apt-get update && apt-get install --no-install-recommends -y mosquitto-clients \ + && rm -rf /var/lib/apt/lists/* + +# Install Python packages +COPY requirements.txt /app/ +RUN pip install --no-cache-dir -r /app/requirements.txt + +# Copy the application code +COPY . /app/ + +CMD ["bash", "-c", "python3 mqtt/fps_extracter.py"] \ No newline at end of file diff --git a/Makefile b/Makefile index deffbfe4..9f79d9db 100644 --- a/Makefile +++ b/Makefile @@ -50,12 +50,39 @@ run-render-mode: down: docker compose -f src/$(DOCKER_COMPOSE) down + docker container stop grafana mqtt-broker + docker container rm grafana mqtt-broker run-demo: | download-models update-submodules download-sample-videos @echo "Building automated self checkout app" $(MAKE) build @echo Running automated self checkout pipeline $(MAKE) run-render-mode + + +run-mqtt: + # Check if Python 3 is installed + @python3 --version || (echo "Python 3 is not installed. Please install Python 3 and try again." && exit 1) + # Ensure python points to python3 + @sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 1 + + # Build and start the Docker Compose services + docker compose up -d + rm -f performance-tools/benchmark-scripts/results/* 2>/dev/null + $(MAKE) benchmark-cmd + + # Build and run the Python scripts container + docker build -t mqtt-scripts -f Dockerfile.fps . + docker build -t mqtt-scripts -f Dockerfile.cpu . + docker run -d --rm \ + -v $(PWD)/performance-tools/benchmark-scripts/results:/app/results \ + -v $(PWD)/mqtt:/app/mqtt \ + mqtt-scripts + @echo "To view the results, open the browser and navigate to http://localhost:3000" + @echo "wait" + +benchmark-cmd: + $(MAKE) PIPELINE_COUNT=$(PIPELINE_COUNT) DURATION=$(DURATION) DEVICE_ENV=$(DEVICE_ENV) RESULTS_DIR=$(RESULTS_DIR) benchmark run-headless: | download-models update-submodules download-sample-videos @echo "Building automated self checkout app" diff --git a/README.md b/README.md index e7ff3eec..d9ab96aa 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ - [Manage Docker as a non-root user](https://docs.docker.com/engine/install/linux-postinstall/) - [Docker Compose v2](https://docs.docker.com/compose/) (Optional) - Intel hardware (CPU, GPU, dGPU) +- Python 3 ## QuickStart @@ -34,6 +35,14 @@ stop containers: make down ``` +## Viewing Benchmark Results in Grafana Dashboard + +To run the MQTT scripts and ensure that the necessary Python packages are installed, use the following command: + +``` +make run-mqtt +``` + ## [Advanced Documentation](https://intel-retail.github.io/documentation/use-cases/automated-self-checkout/automated-self-checkout.html) ## Join the community diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..e59b4cad --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,45 @@ +version: '3.7' +services: + mqtt-broker_2: + image: eclipse-mosquitto:2.0.18 + container_name: mqtt-broker_2 + ports: + - "1885:1883" + volumes: + - ./mosquitto-custom/mosquitto.conf:/mosquitto/config/mosquitto.conf + restart: unless-stopped + + grafana: + image: grafana/grafana:latest + container_name: my_grafana_1 + environment: + - GF_INSTALL_PLUGINS=grafana-mqtt-datasource + volumes: + - ./grafana-custom/provisioning/dashboards/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml + - ./grafana-custom/provisioning/datasource/mqtt-datasource.yaml:/etc/grafana/provisioning/datasources/mqtt-datasource.yaml + - ./grafana-custom/dashboards/:/var/lib/grafana/dashboards + ports: + - "3001:3000" + depends_on: + - mqtt-broker_2 + + mqtt-runner-1: + build: + context: . + dockerfile: Dockerfile.fps + container_name: fps-mqtt-runner-1 + volumes: + - ./performance-tools/benchmark-scripts/results:/app/results + - ./mqtt:/app/mqtt # Mount your MQTT script directory if needed + restart: unless-stopped + + mqtt-runner-2: + build: + context: . + dockerfile: Dockerfile.cpu + container_name: cpu-mqtt-runner-1 + volumes: + - ./performance-tools/benchmark-scripts/results:/app/results + - ./mqtt:/app/mqtt # Mount your MQTT script directory if needed + restart: unless-stopped + diff --git a/grafana-custom/dashboards/dashboard.json b/grafana-custom/dashboards/dashboard.json new file mode 100644 index 00000000..518f136d --- /dev/null +++ b/grafana-custom/dashboards/dashboard.json @@ -0,0 +1,568 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-mqtt-datasource", + "uid": "feb8cw2js8tmoc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 31, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "refId": "A", + "topic": "grafana/log_data" + } + ], + "title": "FPS", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Cpu_idle": true, + "Cpu_iowait": true, + "Cpu_user": true, + "Memory_total": true, + "Memory_used": true, + "avg": true, + "current_read": true, + "current_write": true, + "frame_latency": true, + "frame_num": true, + "latency": true, + "max": true, + "min": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-mqtt-datasource", + "uid": "feb8cw2js8tmoc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "refId": "A", + "topic": "grafana/log_data" + } + ], + "title": "Latency", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Cpu_idle": true, + "Cpu_iowait": true, + "Cpu_user": true, + "Memory_total": true, + "Memory_used": true, + "avg": true, + "current_read": true, + "current_write": true, + "fps": true, + "frame_latency": true, + "frame_num": true, + "latency": false, + "max": true, + "min": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-mqtt-datasource", + "uid": "feb8cw2js8tmoc" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "timezone": [ + "America/Phoenix" + ], + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "refId": "A", + "topic": "grafana/log_data" + } + ], + "title": "Frame Latency", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Cpu_idle": true, + "Cpu_iowait": true, + "Cpu_user": true, + "Memory_total": true, + "Memory_used": true, + "avg": true, + "current_read": true, + "current_write": true, + "fps": true, + "frame_latency": false, + "frame_num": true, + "latency": true, + "max": true, + "min": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-mqtt-datasource", + "uid": "feb8cw2js8tmoc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "fieldMinMax": false, + "mappings": [], + "noValue": "disconnected", + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 50 + } + ] + }, + "unit": "deckbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": true, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "refId": "A", + "topic": "grafana/log_data" + } + ], + "title": "Memory ", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Cpu_idle": true, + "Cpu_iowait": true, + "Cpu_user": true, + "Memory_total": false, + "Memory_used": false, + "Time": false, + "avg": true, + "current_read": true, + "current_write": true, + "fps": true, + "frame_latency": true, + "frame_num": true, + "latency": true, + "max": true, + "min": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Memory_total": "Available", + "Memory_used": "Used" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "grafana-mqtt-datasource", + "uid": "feb8cw2js8tmoc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 4, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "refId": "A", + "topic": "grafana/log_data" + } + ], + "title": "CPU Stats", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "CPU": true, + "DISK": true, + "MEMORY": true, + "Memory_total": true, + "Memory_used": true, + "avg": true, + "current_read": true, + "current_write": true, + "fps": true, + "frame_latency": true, + "frame_num": true, + "latency": true, + "max": true, + "min": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "gauge" + } + ], + "preload": false, + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Frame_latency", + "uid": "febagokt7o64gc", + "version": 13, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana-custom/provisioning/dashboards/dashboards.yaml b/grafana-custom/provisioning/dashboards/dashboards.yaml new file mode 100644 index 00000000..6cd5057b --- /dev/null +++ b/grafana-custom/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 # how often Grafana will scan for changes + options: + path: /var/lib/grafana/dashboards + \ No newline at end of file diff --git a/grafana-custom/provisioning/datasource/mqtt-datasource.yaml b/grafana-custom/provisioning/datasource/mqtt-datasource.yaml new file mode 100644 index 00000000..84fbe455 --- /dev/null +++ b/grafana-custom/provisioning/datasource/mqtt-datasource.yaml @@ -0,0 +1,21 @@ +apiVersion: 1 + +datasources: + - name: MQTT-GRAFANA + type: grafana-mqtt-datasource + uid: feb8cw2js8tmoc + access: proxy + # Base URL of the MQTT broker + url: tcp://mqtt-broker_2:1883 + isDefault: false + editable: true + jsonData: + # Required by the plugin + protocol: "tcp" + + server: tcp://mqtt-broker_2:1883 + # Unique client ID for MQTT connection + clientId: grafana_mqtt_client_1 + # username: your_username + # password: your_password + secureJsonData: \ No newline at end of file diff --git a/helm/Chart.yaml b/helm/Chart.yaml deleted file mode 100644 index c2064e29..00000000 --- a/helm/Chart.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright © 2024 Intel Corporation. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: object-detection-service-low-code -description: DLStreamer based object detection pipeline -version: 3.0.0 -apiVersion: v1 -keywords: - - dlstreamer - - object-detection -sources: -home: diff --git a/helm/Dockerfile b/helm/Dockerfile deleted file mode 100644 index 765ac2dc..00000000 --- a/helm/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -# -# Copyright (C) 2024 Intel Corporation. -# -# SPDX-License-Identifier: Apache-2.0 -# - -FROM intel/dlstreamer:2024.1.1-dev-ubuntu22 as base -USER root - -COPY helm/requirements.txt /requirements.txt -RUN pip3 install --upgrade pip --no-cache-dir -r /requirements.txt -WORKDIR / -COPY src/extensions /home/pipeline-server/extensions -COPY src/pipelines /home/pipeline-server/pipelines -COPY /download_models/downloadModels.sh /home/pipeline-server/models/downloadModels.sh -WORKDIR /home/pipeline-server/models -RUN sh downloadModels.sh -WORKDIR / - -COPY src/entrypoint.sh /script/entrypoint.sh -RUN mkdir /tmp/results \ No newline at end of file diff --git a/helm/Makefile b/helm/Makefile deleted file mode 100644 index 72c73ed0..00000000 --- a/helm/Makefile +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright © 2024 Intel Corporation. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -.PHONY: build build-all build-minikube run-minikube-demo stop-minikube-demo helm-convert install-helm uninstall-helm - -build: - docker build --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -t dlstreamer:deploy -f Dockerfile .. - -build-minikube: build - minikube start - minikube image build -t dlstreamer:deploy -f helm/Dockerfile .. - -run-minikube-demo: build-minikube - kompose -f docker-compose.yml convert -o kubernetes/ && \ - kubectl apply -f kubernetes - -stop-minikube-demo: - kubectl delete -f kubernetes - minikube stop - -helm-convert: - kompose -f docker-compose.yml convert -o kubernetes -c - -install-helm: - helm install -f kubernetes/Chart.yaml dls kubernetes - -uninstall-helm: - helm uninstall dls \ No newline at end of file diff --git a/helm/docker-compose.yml b/helm/docker-compose.yml deleted file mode 100644 index bd375426..00000000 --- a/helm/docker-compose.yml +++ /dev/null @@ -1,71 +0,0 @@ -# -# Copyright (C) 2024 Intel Corporation. -# -# SPDX-License-Identifier: Apache-2.0 -# - - - ## Current Developer Toolbox doesn't support environment files, make sure to remove any files or environment variables starting with $ -version: '3.7' -services: - camera-simulator: - container_name: camera-simulator - image: aler9/rtsp-simple-server - ports: - - "8554:8554" - env_file: - - .env - camera-simulator0: - image: jrottenberg/ffmpeg:4.1-alpine - container_name: camera-simulator0 - entrypoint: ["/bin/sh", "-c"] - command: | - if [ ! -f /home/pipeline-server/sample-media/coca-cola-4465029-1920-15-bench.mp4 ]; then - mkdir -p /home/pipeline-server/sample-media - wget -O /home/pipeline-server/sample-media/coca-cola-4465029-1920-15-bench.mp4 https://www.pexels.com/download/video/4465029 - fi - ffmpeg -nostdin -re -stream_loop -1 -i /home/pipeline-server/sample-media/coca-cola-4465029-1920-15-bench.mp4 -c copy -f rtsp -rtsp_transport tcp rtsp://camera-simulator:8554/camera_0 - depends_on: - - camera-simulator - env_file: - - .env - - dlstreamer: - image: dlstreamer:deploy - deploy: - mode: replicated - replicas: 1 - entrypoint: /script/entrypoint.sh --pipeline_script_choice yolov5s.sh - environment: - INPUTSRC: ${RTSP_INPUTSRC} - BATCH_SIZE: ${BATCH_SIZE} - GST_DEBUG: ${GST_DEBUG} - LOG_LEVEL: ${LOG_LEVEL} - CLASSIFICATION_OPTIONS: ${CLASSIFICATION_OPTIONS} - DETECTION_OPTIONS: ${DETECTION_OPTIONS} - OCR_RECLASSIFY_INTERVAL: ${OCR_RECLASSIFY_INTERVAL} - BARCODE_RECLASSIFY_INTERVAL: ${BARCODE_RECLASSIFY_INTERVAL} - RENDER_MODE: ${RENDER_MODE} - PUBLISH: ${PUBLISH} - depends_on: - - camera-simulator0 - env_file: - - .env - - mqtt-broker: - image: eclipse-mosquitto:2.0.18 - ports: - - "${MQTT_PORT}:${MQTT_PORT}" - command: - - /usr/sbin/mosquitto - - -c - - /mosquitto-no-auth.conf - container_name: mqtt-broker - hostname: mqtt-broker - read_only: true - restart: always - security_opt: - - no-new-privileges:true - user: 2002:2001 - env_file: - - .env \ No newline at end of file diff --git a/helm/requirements.txt b/helm/requirements.txt deleted file mode 100644 index d40d0daa..00000000 --- a/helm/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -pyzbar==0.1.9 -zxing-cpp==2.2.0 -numpy==2.1.3 \ No newline at end of file diff --git a/mosquitto-custom/mosquitto.conf b/mosquitto-custom/mosquitto.conf new file mode 100644 index 00000000..6dc5917c --- /dev/null +++ b/mosquitto-custom/mosquitto.conf @@ -0,0 +1,2 @@ +listener 1883 0.0.0.0 +allow_anonymous true diff --git a/mqtt/fps_extracter.py b/mqtt/fps_extracter.py new file mode 100644 index 00000000..0f0a19cb --- /dev/null +++ b/mqtt/fps_extracter.py @@ -0,0 +1,80 @@ +import re +import json +import time +import os +from paho.mqtt.client import Client + +# Configuration +current_directory = os.getcwd() +folder_path = os.path.join( + current_directory, 'performance-tools/benchmark-scripts/results' +) + +# Define the pattern for matching log files +pattern = r'^gst-launch_.*_gst\.log$' + +# List all matching log files +file_names = os.listdir(folder_path) +matching_files = [file for file in file_names if re.match(pattern, file)] +relative_path = ( + "performance-tools/benchmark-scripts/results/" + matching_files[0] +) +file_path = os.path.join(current_directory, relative_path) + +# MQTT Configuration +mqtt_broker = "mqtt-broker_2" +mqtt_port = 1883 +mqtt_topic = "grafana/log_data" + +# Regex pattern to extract relevant information +data_pattern = ( + r"latency_tracer_pipeline, frame_latency=\(double\)([\d.]+), " + r"avg=\(double\)([\d.]+), min=\(double\)([\d.]+), " + r"max=\(double\)([\d.]+), latency=\(double\)([\d.]+), " + r"fps=\(double\)([\d.]+), frame_num=\(uint\)(\d+);" +) + + +def extract_data(file_path): + """Extract relevant information from the log file.""" + extracted_data = [] + with open(file_path, "r") as file: + for line in file: + match = re.search(data_pattern, line) + if match: + data = { + "frame_latency": float(match.group(1)), + "avg": float(match.group(2)), + "min": float(match.group(3)), + "max": float(match.group(4)), + "latency": float(match.group(5)), + "fps": float(match.group(6)), + "frame_num": int(match.group(7)) + } + extracted_data.append(data) + return extracted_data + + +def publish_data_to_mqtt(extracted_data): + """Publish extracted data to MQTT broker.""" + client = Client() + try: + client.connect(mqtt_broker, mqtt_port) + client.loop_start() + for data in extracted_data: + payload = json.dumps(data) + client.publish(mqtt_topic, payload) + time.sleep(1) + except Exception as e: + print(f"Failed to publish to MQTT: {e}") + finally: + client.loop_stop() + client.disconnect() + + +if __name__ == "__main__": + extracted_data = extract_data(file_path) + if extracted_data: + publish_data_to_mqtt(extracted_data) + else: + print("No relevant data found in the log file.") diff --git a/mqtt/publisher_intel.py b/mqtt/publisher_intel.py new file mode 100644 index 00000000..e7d7c42e --- /dev/null +++ b/mqtt/publisher_intel.py @@ -0,0 +1,111 @@ +import itertools +import json +import os +import time +from paho.mqtt.client import Client + +# MQTT Configuration +BROKER_ADDRESS = "mqtt-broker_2" # Replace with your MQTT broker address +MQTT_TOPIC = "grafana/log_data" # Topic to publish the logs +MQTT_PORT = 1883 # Default MQTT port + +# Get the current working directory +current_directory = os.getcwd() + +# Define the relative paths to the files +relative_paths = { + "cpu": "performance-tools/benchmark-scripts/results/cpu_usage.log", + "memory": "performance-tools/benchmark-scripts/results/memory_usage.log", + "disk": "performance-tools/benchmark-scripts/results/disk_bandwidth.log", +} + +# Combine the current directory with the relative paths +file_paths = { + key: os.path.join(current_directory, path) + for key, path in relative_paths.items() +} + +# Initialize file handlers +cpu_file = open(file_paths["cpu"], 'r') +memory_file = open(file_paths["memory"], 'r') +disk_file = open(file_paths["disk"], 'r') + +# Initialize MQTT client +mqtt_client = Client() +mqtt_client.connect(BROKER_ADDRESS, MQTT_PORT) + + +def process_cpu_line(line): + """Process a line from the CPU usage file.""" + parts = line.split() + if len(parts) == 8 and parts[1] == "all": + return { + "Cpu_user": float(parts[2]), + "Cpu_idle": float(parts[7]), + "Cpu_iowait": float(parts[4]), + } + return None + + +def process_memory_line(line): + """Process a line from the memory usage file.""" + if line.startswith("Mem:"): + parts = line.split() + return {"Memory_total": int(parts[1]), "Memory_used": int(parts[2])} + return None + + +def process_disk_line(line): + """Process a line from the disk bandwidth file.""" + total_read = None + total_write = None + + if line.startswith("Total DISK READ"): + total_read = line.split('|')[0].split(':')[-1].strip() + total_write = line.split('|')[1].split(':')[-1].strip() + elif line.startswith("Current DISK READ"): + current_read = line.split('|')[0].split(':')[-1].strip() + current_write = line.split('|')[1].split(':')[-1].strip() + return { + "total_read": total_read, + "current_read": current_read, + "total_write": total_write, + "current_write": current_write, + } + return None + + +# Using itertools.zip_longest to iterate over all files simultaneously +for cpu_line, mem_line, disk_line in itertools.zip_longest( + cpu_file, memory_file, disk_file +): + # Process each line + cpu_data = process_cpu_line(cpu_line) if cpu_line else None + memory_data = process_memory_line(mem_line) if mem_line else None + disk_data = process_disk_line(disk_line) if disk_line else None + + # Combine data into a single payload + log_data = {} + if cpu_data: + log_data.update(cpu_data) + if memory_data: + log_data.update(memory_data) + if disk_data: + log_data.update(disk_data) + + if log_data: + # Convert to JSON and publish to MQTT + payload = json.dumps(log_data) + mqtt_client.publish(MQTT_TOPIC, payload) + print(f"Published: {payload}") + + # Sleep to simulate real-time log streaming (adjust as needed) + time.sleep(1) + +# Close all file handlers +cpu_file.close() +memory_file.close() +disk_file.close() + +# Disconnect MQTT client +mqtt_client.disconnect() diff --git a/performance-tools b/performance-tools index 197209f8..981fcc0b 160000 --- a/performance-tools +++ b/performance-tools @@ -1 +1 @@ -Subproject commit 197209f8f7bee712df5728d7cfae1605a3820d07 +Subproject commit 981fcc0b5e9ccf3742b2c3aeafdc28965d7fab0a diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..c543356e --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +paho-mqtt==1.6.0 \ No newline at end of file diff --git a/src/grafana-custom/dashboards/dashboards.yaml b/src/grafana-custom/dashboards/dashboards.yaml new file mode 100644 index 00000000..e69de29b diff --git a/src/grafana-custom/provisioning/datasources/mqtt-datasource.yaml b/src/grafana-custom/provisioning/datasources/mqtt-datasource.yaml index 961a023d..b0e95d97 100644 --- a/src/grafana-custom/provisioning/datasources/mqtt-datasource.yaml +++ b/src/grafana-custom/provisioning/datasources/mqtt-datasource.yaml @@ -6,14 +6,14 @@ datasources: uid: mqtt-datasource-uid access: proxy # Base URL of the MQTT broker - url: tcp://mqtt-broker_1:1883 + url: tcp://mqtt-broker_2:1883 isDefault: false editable: true jsonData: # Required by the plugin protocol: "tcp" - server: tcp://mqtt-broker_1:1883 + server: tcp://mqtt-broker_2:1883 # Unique client ID for MQTT connection clientId: grafana_mqtt_client_1 # username: your_username diff --git a/src/pipeline-server/Dockerfile.pipeline-server b/src/pipeline-server/Dockerfile.pipeline-server index dd67124c..08f7e12c 100644 --- a/src/pipeline-server/Dockerfile.pipeline-server +++ b/src/pipeline-server/Dockerfile.pipeline-server @@ -4,7 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 # -FROM intel/edge_video_analytics_microservice:2.0.0 +FROM intel/edge_video_analytics_microservice:1.0.0-standalone COPY ./requirements.txt /requirements.txt RUN pip3 install --upgrade pip --no-cache-dir -r /requirements.txt \ No newline at end of file