diff --git a/dashboards/README.md b/dashboards/README.md new file mode 100644 index 000000000..e27e23851 --- /dev/null +++ b/dashboards/README.md @@ -0,0 +1,70 @@ +# Compass Observability — Grafana Dashboards + +## Overview + +These are the Plane-B operator dashboards for Compass observability: in-repo +Grafana dashboard JSON that renders signals the shipped `compass-agent` emits +today. They ship as JSON only — there is **no provisioning automation** (no +datasource YAML, no docker-compose, no `grafana.ini`) by design, per decision +D3/T5 of the frozen record +[`docs/designs/platform/compass-observability-architecture/design.md`](../docs/designs/platform/compass-observability-architecture/design.md). +Import is the operator's one manual step. + +Each dashboard declares its datasource as a templating variable +(`${DS_PROMETHEUS}` or `${DS_TEMPO}`) rather than a hard-coded uid, so the JSON +imports into any Grafana: you pick the concrete datasource at import time. + +## `compass-agent-transport.json` + +Transport-plane metrics for the agent (Prometheus datasource variable +`DS_PROMETHEUS`). OTLP metrics are exported through the OTel Prometheus +exporter, which mangles names (`.` → `_`) and appends `_total` to monotonic +counters; gauges keep their dotless-to-underscore name with no suffix. Panels +are authored against the Prometheus-mangled names. + +Rendered signals (metric constants → Prometheus-mangled name): + +- `compass_agent.transport.publish.trace_frames_lost` → `..._total` (by `reason`) +- `compass_agent.transport.publish.priority_frames_lost` → `..._total` (never-drop contract; any nonzero is a breach) +- `compass_agent.transport.publish.priority_batch_retries` → `..._total` +- `compass_agent.transport.publish.priority_retry_depth` (gauge) +- `compass_agent.transport.publish.trace_queue_depth` (gauge) +- `compass_agent.transport.frame_sink.durable_attempts` → `..._total` +- `compass_agent.transport.frame_sink.durable_give_ups` → `..._total` +- `compass_agent.transport.control.reconnects` → `..._total` +- `compass_agent.transport.control.no_progress_depth` (gauge) +- `compass_agent.transport.control.flap_resets` → `..._total` +- `compass_agent.transport.control.unmapped` → `..._total` (by `event_type`) + +## `compass-agent-loop-traces.json` + +Agent-loop GenAI traces (Tempo datasource variable `DS_TEMPO`). The +pi-agent-core loop emits GenAI-semconv spans; the span hierarchy is +`invoke_agent {agent.name}` → children `chat {model}` / `execute_tool +{tool.name}` / `handoff`. Correlate a trace to a session via span attribute +`gen_ai.conversation.id` (= session id) and resource attribute +`compass.session.id` (stamped via `OTEL_RESOURCE_ATTRIBUTES` on both loop and +transport signals). + +Rendered signals (span names, TraceQL filtered to `service.name = compass-agent`): + +- `invoke_agent.*` root spans — recent turns table +- `execute_tool.*` spans — tool-call table + +## Import + +To import a dashboard: + +1. In Grafana, go to **Dashboards → New → Import**. +2. Upload the dashboard JSON file (or paste its contents). +3. When prompted, pick the concrete datasource for the dashboard's datasource + variable (`DS_PROMETHEUS` for the transport dashboard, `DS_TEMPO` for the + loop-traces dashboard). +4. Click **Import**. + +## Server/runner panels — follow-up (T4b) + +The Go server/runner observability signals (task T4b of the frozen record) do +not exist yet. Panels for those signals are a follow-up and will be added once +those metrics ship; this directory currently binds only the shipped +`compass-agent` signals listed above. diff --git a/dashboards/compass-agent-loop-traces.json b/dashboards/compass-agent-loop-traces.json new file mode 100644 index 000000000..b251f2d26 --- /dev/null +++ b/dashboards/compass-agent-loop-traces.json @@ -0,0 +1,116 @@ +{ + "title": "Compass Agent — Loop Traces", + "uid": "compass-agent-loop-traces", + "tags": ["compass", "observability"], + "schemaVersion": 39, + "editable": true, + "graphTooltip": 0, + "time": { + "from": "now-6h", + "to": "now" + }, + "refresh": "30s", + "timezone": "browser", + "templating": { + "list": [ + { + "name": "DS_TEMPO", + "label": "Tempo", + "type": "datasource", + "query": "tempo", + "current": {}, + "hide": 0, + "refresh": 1, + "regex": "", + "includeAll": false, + "multi": false + } + ] + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "panels": [ + { + "id": 1, + "type": "table", + "title": "Recent invoke_agent root spans", + "description": "GenAI-semconv agent-loop spans (service.name=compass-agent). Root span `invoke_agent {agent.name}` opens each turn; its children are `chat {model}`, `execute_tool {tool.name}`, and `handoff`. Correlate a trace to a session via span attribute gen_ai.conversation.id (= session id) and resource attribute compass.session.id (stamped via OTEL_RESOURCE_ATTRIBUTES on both loop and transport signals). TraceQL filters the Tempo search to invoke_agent root spans.", + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{ resource.service.name = \"compass-agent\" && name =~ \"invoke_agent.*\" }", + "limit": 20, + "tableType": "traces" + } + ] + }, + { + "id": 2, + "type": "table", + "title": "Tool-call spans (execute_tool)", + "description": "Child spans `execute_tool {tool.name}` under an invoke_agent root, per the GenAI-semconv hierarchy (invoke_agent -> chat/execute_tool/handoff). Attributes follow OTel GenAI semconv: gen_ai.operation.name, gen_ai.request.model, gen_ai.usage.input_tokens/output_tokens. Correlate to a session via gen_ai.conversation.id / compass.session.id. TraceQL filters the Tempo search to execute_tool spans.", + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 12 + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {}, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{ resource.service.name = \"compass-agent\" && name =~ \"execute_tool.*\" }", + "limit": 20, + "tableType": "traces" + } + ] + } + ] +} diff --git a/dashboards/compass-agent-transport.json b/dashboards/compass-agent-transport.json new file mode 100644 index 000000000..83b5a4d64 --- /dev/null +++ b/dashboards/compass-agent-transport.json @@ -0,0 +1,537 @@ +{ + "title": "Compass Agent — Transport", + "uid": "compass-agent-transport", + "tags": ["compass", "observability"], + "schemaVersion": 39, + "editable": true, + "graphTooltip": 0, + "time": { + "from": "now-6h", + "to": "now" + }, + "refresh": "30s", + "timezone": "browser", + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "type": "datasource", + "query": "prometheus", + "current": {}, + "hide": 0, + "refresh": 1, + "regex": "", + "includeAll": false, + "multi": false + } + ] + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Trace frames lost by reason", + "description": "Metric constant: compass_agent.transport.publish.trace_frames_lost (counter, label reason=overflow|failed_batch). Prometheus-mangled name: compass_agent_transport_publish_trace_frames_lost_total (. -> _, monotonic counter gets _total).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10 + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (reason) (rate(compass_agent_transport_publish_trace_frames_lost_total[$__rate_interval]))", + "legendFormat": "{{reason}}" + } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "Priority frames lost", + "description": "Metric constant: compass_agent.transport.publish.priority_frames_lost (counter). Prometheus-mangled: compass_agent_transport_publish_priority_frames_lost_total. Priority frames are a never-drop contract — ANY nonzero value is a contract breach.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10 + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(compass_agent_transport_publish_priority_frames_lost_total[$__rate_interval]))", + "legendFormat": "priority frames lost" + } + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "Priority batch retries", + "description": "Metric constant: compass_agent.transport.publish.priority_batch_retries (counter). Prometheus-mangled: compass_agent_transport_publish_priority_batch_retries_total.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10 + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(compass_agent_transport_publish_priority_batch_retries_total[$__rate_interval]))", + "legendFormat": "priority batch retries" + } + ] + }, + { + "id": 4, + "type": "timeseries", + "title": "Priority retry depth", + "description": "Metric constant: compass_agent.transport.publish.priority_retry_depth (gauge). Prometheus-mangled: compass_agent_transport_publish_priority_retry_depth (gauge keeps its name, no _total suffix).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10 + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max(compass_agent_transport_publish_priority_retry_depth)", + "legendFormat": "priority retry depth" + } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Trace queue depth", + "description": "Metric constant: compass_agent.transport.publish.trace_queue_depth (gauge). Prometheus-mangled: compass_agent_transport_publish_trace_queue_depth (gauge keeps its name, no _total suffix).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10 + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max(compass_agent_transport_publish_trace_queue_depth)", + "legendFormat": "trace queue depth" + } + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "Frame-sink durable attempts vs give-ups", + "description": "Metric constants: compass_agent.transport.frame_sink.durable_attempts and compass_agent.transport.frame_sink.durable_give_ups (counters). Prometheus-mangled: compass_agent_transport_frame_sink_durable_attempts_total and compass_agent_transport_frame_sink_durable_give_ups_total.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10 + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(compass_agent_transport_frame_sink_durable_attempts_total[$__rate_interval]))", + "legendFormat": "durable attempts" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(compass_agent_transport_frame_sink_durable_give_ups_total[$__rate_interval]))", + "legendFormat": "durable give-ups" + } + ] + }, + { + "id": 7, + "type": "timeseries", + "title": "Control reconnects", + "description": "Metric constant: compass_agent.transport.control.reconnects (counter). Prometheus-mangled: compass_agent_transport_control_reconnects_total.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10 + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(compass_agent_transport_control_reconnects_total[$__rate_interval]))", + "legendFormat": "reconnects" + } + ] + }, + { + "id": 8, + "type": "timeseries", + "title": "Control no-progress depth", + "description": "Metric constant: compass_agent.transport.control.no_progress_depth (gauge). Prometheus-mangled: compass_agent_transport_control_no_progress_depth (gauge keeps its name, no _total suffix).", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10 + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max(compass_agent_transport_control_no_progress_depth)", + "legendFormat": "no-progress depth" + } + ] + }, + { + "id": 9, + "type": "timeseries", + "title": "Control flap resets", + "description": "Metric constant: compass_agent.transport.control.flap_resets (counter). Prometheus-mangled: compass_agent_transport_control_flap_resets_total.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10 + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(compass_agent_transport_control_flap_resets_total[$__rate_interval]))", + "legendFormat": "flap resets" + } + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "Control unmapped by event_type", + "description": "Metric constant: compass_agent.transport.control.unmapped (counter, label event_type dynamic e.g. control:steer, control:replay). Prometheus-mangled: compass_agent_transport_control_unmapped_total.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10 + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (event_type) (rate(compass_agent_transport_control_unmapped_total[$__rate_interval]))", + "legendFormat": "{{event_type}}" + } + ] + } + ] +}