feat: Prometheus integration (#35)

* refactor: move Prometheus integration to extension

* chore: update devenv to include Grafana provisioning config for Prometheus

* chore: update config for PromEx

* feat: add Telemetry events

* feat: add promex plugin

* feat: add Grafana dashboard for deliveries

devenv.lock

@@ -31,10 +31,31 @@
         "type": "github"
       }
     },
+    "git-hooks": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1737465171,
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "9364dc02281ce2d37a1f55b6e51f7c0f65a75f17",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "type": "github"
+      }
+    },
     "gitignore": {
       "inputs": {
         "nixpkgs": [
-          "pre-commit-hooks",
+          "git-hooks",
           "nixpkgs"
         ]
       },
@@ -81,33 +102,15 @@
         "type": "github"
       }
     },
-    "pre-commit-hooks": {
-      "inputs": {
-        "flake-compat": "flake-compat",
-        "gitignore": "gitignore",
-        "nixpkgs": [
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1735882644,
-        "owner": "cachix",
-        "repo": "pre-commit-hooks.nix",
-        "rev": "a5a961387e75ae44cc20f0a57ae463da5e959656",
-        "type": "github"
-      },
-      "original": {
-        "owner": "cachix",
-        "repo": "pre-commit-hooks.nix",
-        "type": "github"
-      }
-    },
     "root": {
       "inputs": {
         "devenv": "devenv",
+        "git-hooks": "git-hooks",
         "nixpkgs": "nixpkgs",
         "nixpkgs-unstable": "nixpkgs-unstable",
-        "pre-commit-hooks": "pre-commit-hooks"
+        "pre-commit-hooks": [
+          "git-hooks"
+        ]
       }
     }
   },

devenv.nix

@@ -8,6 +8,23 @@ let
     printf $((RANDOM % 10000)) > $out
   '');
   localtunnel-subdomain = "tololo-" + randomSuffix;
+  grafana-provisioning = pkgs.runCommand "grafana-provisioning" {
+     buildInputs = [ pkgs.jq ];
+   } ''
+     mkdir -p $out/datasources
+     cat > $out/datasources/prometheus.yml <<EOF
+     apiVersion: 1
+     datasources:
+       - name: "Prometheus Tololo"
+         type: prometheus
+         uid: "prometheus-tololo"
+         access: proxy
+         url: http://localhost:9090
+         isDefault: true
+         jsonData:
+           timeInterval: "15s"
+     EOF
+   '';
 in
 {
   devcontainer.enable = true;
@@ -64,6 +81,11 @@ in
       chmod 777 -R .devenv/state/grafana
     fi
 
+    mkdir -p .devenv/state/grafana/conf/provisioning
+    cp -rL ${grafana-provisioning}/datasources .devenv/state/grafana/conf/provisioning/
+    chmod 777 -R .devenv/state/grafana/conf/provisioning/
+    
+
     if [ ! -d ".devenv/state/prometheus" ]; then
       mkdir .devenv/state/prometheus
     fi

prometheus.yml

@@ -19,11 +19,10 @@ rule_files:
 # A scrape configuration containing exactly one endpoint to scrape:
 # Here it's Prometheus itself.
 scrape_configs:
-  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
-  - job_name: "prometheus"
+  - job_name: 'tololo'
 
     # metrics_path defaults to '/metrics'
     # scheme defaults to 'http'.
 
     static_configs:
-      - targets: ["localhost:9090"]
+      - targets: ["localhost:9090", "localhost:4000"]

tololo/config/config.exs

@@ -49,7 +49,7 @@ config :spark,
 config :tololo,
   ecto_repos: [Tololo.Repo],
   generators: [timestamp_type: :utc_datetime],
-  extensions: [Tololo.Extensions.TelegramBot],
+  extensions: [Tololo.Extensions.TelegramBot, Tololo.Extensions.Prometheus],
   pubsub: Tololo.PubSub,
   geocoding_endpoint: "https://nominatim.openstreetmap.org/search",
   business_name: "Sushi",

tololo/config/dev.exs

@@ -31,7 +31,7 @@ config :tololo, TololoWeb.Endpoint,
 
 config :tololo, :kafka_driver, driver: Tololo.Kafka.Driver
 
-config :tololo, Tololo.Prometheus,
+config :tololo, Tololo.Extensions.Prometheus.PromEx,
   disabled: false,
   manual_metrics_start_delay: :no_delay,
   drop_metrics_groups: [],

tololo/config/prod.exs

@@ -18,8 +18,6 @@ config :logger, level: :info
 
 config :tololo, :kafka_driver, driver: Tololo.Kafka.Noop
 
-config :tololo, Tololo.Prometheus, disabled: true
-
 config :kafka_ex,
   brokers: [
     {"localhost", 9092},

tololo/config/test.exs

@@ -39,7 +39,6 @@ config :phoenix_live_view,
   enable_expensive_runtime_checks: true
 
 config :tololo, :kafka_driver, driver: Tololo.Kafka.Noop
-config :tololo, Tololo.Prometheus, disabled: true
 
 config :kafka_ex,
   brokers: [

tololo/core/lib/deliveries/delivery.ex

@@ -247,6 +247,12 @@ defmodule TololoCore.Deliveries.Delivery do
               comment
             )
 
+            :telemetry.execute([:ash, :deliveries, :update, :state], %{count: 1}, %{
+              action: :done_with_distance_check,
+              old_state: old_state,
+              new_state: new_state
+            })
+
             {:ok, result}
 
           _changeset, error ->

tololo/core/lib/deliveries/update_history.ex

@@ -22,6 +22,13 @@ defmodule TololoCore.Deliveries.UpdateHistory do
       |> Ash.Changeset.after_transaction(fn
         _changeset, {:ok, result} ->
           DeliveryStateChanges.add_to_state_history!(id, old_state, new_state, comment)
+
+          :telemetry.execute([:ash, :deliveries, :update, :state], %{count: 1}, %{
+            action: :update_state,
+            old_state: old_state,
+            new_state: new_state
+          })
+
           {:ok, result}
 
         _changeset, error ->

tololo/core/lib/extension.ex

@@ -3,7 +3,28 @@ defmodule TololoCore.Extension do
   Defines the extension behaviour.
   """
   @callback routes() :: Macro.t()
+  @callback endpoint() :: Macro.t()
   @callback ash_domains() :: [Ash.Domain.t()]
   @callback child_spec(any()) :: Supervisor.child_spec()
   # @callback payment()
+
+  @extensions Application.compile_env(:tololo, :extensions)
+
+  defmacro __using__(:routes) do
+    quote do
+      unquote(
+        @extensions
+        |> Enum.map(fn extension_module -> extension_module.routes() end)
+      )
+    end
+  end
+
+  defmacro __using__(:endpoint) do
+    quote do
+      unquote(
+        @extensions
+        |> Enum.map(fn extension_module -> extension_module.endpoint() end)
+      )
+    end
+  end
 end

tololo/extensions/prometheus/.gitignore

@@ -0,0 +1,23 @@
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where third-party dependencies like ExDoc output generated docs.
+/doc/
+
+# If the VM crashes, it generates a dump, let's ignore it too.
+erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
+*.ez
+
+# Ignore package tarball (built via "mix hex.build").
+tololo_extension_telegram_bot-*.tar
+
+# Temporary files, for example, from tests.
+/tmp/

tololo/extensions/prometheus/lib/extension.ex

@@ -0,0 +1,44 @@
+defmodule Tololo.Extensions.Prometheus do
+  @moduledoc false
+  use Supervisor
+  @behaviour TololoCore.Extension
+
+  @impl true
+  def child_spec(init_arg) do
+    %{
+      id: __MODULE__,
+      start: {__MODULE__, :start_link, [init_arg]},
+      type: :supervisor,
+      restart: :permanent
+    }
+  end
+
+  def start_link(_init_arg) do
+    Supervisor.start_link(__MODULE__, nil, name: __MODULE__)
+  end
+
+  @impl true
+  def init(_) do
+    children = [
+      Tololo.Extensions.Prometheus.PromEx
+    ]
+
+    Supervisor.init(children, strategy: :one_for_all)
+  end
+
+  @impl true
+  def routes() do
+    quote do
+    end
+  end
+
+  @impl true
+  def endpoint() do
+    quote do
+      plug PromEx.Plug, prom_ex_module: Tololo.Extensions.Prometheus.PromEx
+    end
+  end
+
+  @impl true
+  def ash_domains(), do: []
+end

tololo/lib/tololo/prometheus/prometheus.ex → tololo/extensions/prometheus/lib/promex.ex

@@ -1,4 +1,4 @@
-defmodule Tololo.Prometheus do
+defmodule Tololo.Extensions.Prometheus.PromEx do
   @moduledoc """
   PromEx module, handles integration with Prometheus.
   """
@@ -15,18 +15,15 @@ defmodule Tololo.Prometheus do
       Plugins.Beam,
       {Plugins.Phoenix, router: TololoWeb.Router, endpoint: TololoWeb.Endpoint},
       Plugins.Ecto,
-      Plugins.PhoenixLiveView
-
-      # Add your own PromEx metrics plugins
-      # Tololo.Users.PromExPlugin
-      # TODO: implement plugin for metrics described in https://github.com/ElixirCL/tololo/issues/1#issuecomment-2582911263
+      Plugins.PhoenixLiveView,
+      Tololo.Extensions.Prometheus.PromExPlugin
     ]
   end
 
   @impl true
   def dashboard_assigns do
     [
-      datasource_id: "prometheus",
+      datasource_id: "prometheus-tololo",
       default_selected_interval: "30s"
     ]
   end
@@ -39,11 +36,9 @@ defmodule Tololo.Prometheus do
       {:prom_ex, "beam.json"},
       {:prom_ex, "phoenix.json"},
       {:prom_ex, "ecto.json"},
-      {:prom_ex, "phoenix_live_view.json"}
+      {:prom_ex, "phoenix_live_view.json"},
 
-      # Add your dashboard definitions here with the format: {:otp_app, "path_in_priv"}
-      # {:tololo, "/grafana_dashboards/user_metrics.json"}
-      # TODO: add dashboards for custom Tololo metrics
+      {:tololo_extension_prometheus, "/dashboards/deliveries.json"}
     ]
   end
 end

tololo/extensions/prometheus/lib/promex_plugin.ex

@@ -0,0 +1,67 @@
+defmodule Tololo.Extensions.Prometheus.PromExPlugin do
+  use PromEx.Plugin
+
+  @impl true
+  def event_metrics(_opts) do
+    Event.build(
+      :tololo_deliveries,
+      [
+        counter(
+          [:ash, :deliveries, :create, :count],
+          event_name: [:ash, :deliveries, :create, :stop],
+          description: "Total number of delivery creations",
+          measurement: :count,
+          tags: [:action],
+          tag_values: fn metadata ->
+            %{
+              action: metadata.action
+            }
+          end
+        ),
+        counter(
+          [:ash, :deliveries, :update, :state, :count],
+          event_name: [:ash, :deliveries, :update, :stop],
+          description: "Total number of state changes",
+          measurement: :count,
+          tags: [:action],
+          tag_values: fn metadata ->
+            %{
+              action: metadata.action
+            }
+          end
+        ),
+        counter(
+          [:ash, :deliveries, :update, :state, :individual, :count],
+          event_name: [:ash, :deliveries, :update, :state],
+          description: "Total number of state changes for individual state",
+          measurement: :count,
+          tags: [:action, :state],
+          tag_values: fn metadata ->
+            %{
+              action: metadata.action,
+              state: metadata.new_state
+            }
+          end
+        ),
+        counter(
+          [:ash, :users, :approve, :count],
+          event_name: [:ash, :users, :approve],
+          description: "Total number of approved users",
+          measurement: :count
+        ),
+        counter(
+          [:ash, :users, :delivery, :assigned, :count],
+          event_name: [:ash, :users, :delivery, :assigned],
+          description: "Total number of assigned deliveries to users",
+          measurement: :count
+        ),
+        counter(
+          [:ash, :users, :delivery, :done, :count],
+          event_name: [:ash, :users, :delivery, :done],
+          description: "Total number of deliveries marked as done by users",
+          measurement: :count
+        )
+      ]
+    )
+  end
+end