feat(observability): add diagnostic dashboards (#27)

2026-07-17 00:22:46 +02:00
parent 06c4ecf8f3
commit 99885f8c8c
30 changed files with 3173 additions and 18 deletions
@@ -0,0 +1,292 @@
+{
+  "annotations": {"list": []},
+  "description": "Privacy-safe operational view of the single-active Rendezvous service. Capacity percentages use the approved 25,000 listing and 10,000 active-attempt launch envelope.",
+  "editable": false,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "id": 1,
+      "title": "Service",
+      "description": "Prometheus can authenticate to and scrape the Rendezvous process.",
+      "type": "stat",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 4, "w": 4, "x": 0, "y": 0},
+      "fieldConfig": {"defaults": {"mappings": [{"options": {"0": {"color": "red", "text": "DOWN"}, "1": {"color": "green", "text": "UP"}}, "type": "value"}], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}, "overrides": []},
+      "options": {"colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true},
+      "targets": [{"editorMode": "code", "expr": "up{job=\"rendezvous\"}", "legendFormat": "Rendezvous", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 2,
+      "title": "Store",
+      "type": "stat",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 4, "w": 4, "x": 4, "y": 0},
+      "fieldConfig": {"defaults": {"mappings": [{"options": {"0": {"color": "red", "text": "UNAVAILABLE"}, "1": {"color": "green", "text": "AVAILABLE"}}, "type": "value"}], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}, "overrides": []},
+      "options": {"colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true},
+      "targets": [{"editorMode": "code", "expr": "rendezvous_store_available", "legendFormat": "Store", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 3,
+      "title": "Drain",
+      "type": "stat",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 4, "w": 4, "x": 8, "y": 0},
+      "fieldConfig": {"defaults": {"mappings": [{"options": {"0": {"color": "green", "text": "ACCEPTING"}, "1": {"color": "orange", "text": "DRAINING"}}, "type": "value"}], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 1}]}}, "overrides": []},
+      "options": {"colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true},
+      "targets": [{"editorMode": "code", "expr": "rendezvous_store_draining", "legendFormat": "Drain", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 4,
+      "title": "Listings",
+      "description": "Percentage of the approved 25,000-listing single-process envelope.",
+      "type": "gauge",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
+      "fieldConfig": {"defaults": {"max": 100, "min": 0, "unit": "percent", "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 70}, {"color": "red", "value": 90}]}}, "overrides": []},
+      "options": {"minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto"},
+      "targets": [{"editorMode": "code", "expr": "100 * rendezvous_store_active_listings / 25000", "legendFormat": "Listings", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 5,
+      "title": "Join attempts",
+      "description": "Percentage of the approved 10,000 active-attempt single-process envelope.",
+      "type": "gauge",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
+      "fieldConfig": {"defaults": {"max": 100, "min": 0, "unit": "percent", "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 70}, {"color": "red", "value": 90}]}}, "overrides": []},
+      "options": {"minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto"},
+      "targets": [{"editorMode": "code", "expr": "100 * rendezvous_store_active_attempts / 10000", "legendFormat": "Attempts", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 6,
+      "title": "HTTP request rate by operation",
+      "description": "Registration, renewal, browse, and join issuance appear as bounded operation names.",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 8, "w": 8, "x": 0, "y": 4},
+      "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "sum by (operation) (rate(rendezvous_http_requests_total[5m]))", "legendFormat": "{{operation}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 7,
+      "title": "HTTP p95 latency",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 8, "w": 8, "x": 8, "y": 4},
+      "fieldConfig": {"defaults": {"unit": "ms", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 200}]}}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(rendezvous_http_duration_milliseconds_bucket[5m])))", "legendFormat": "{{operation}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 8,
+      "title": "HTTP error and shedding rate",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 8, "w": 8, "x": 16, "y": 4},
+      "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "sum by (status_code) (rate(rendezvous_http_requests_total{status_code=~\"4..|5..\"}[5m]))", "legendFormat": "HTTP {{status_code}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 9,
+      "title": "Browser live-update load",
+      "description": "Active public SSE clients, tenant scopes, and bounded replay entries.",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 0, "y": 12},
+      "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [
+        {"editorMode": "code", "expr": "rendezvous_browser_sse_subscribers", "legendFormat": "Subscribers", "range": true, "refId": "A"},
+        {"editorMode": "code", "expr": "rendezvous_browser_sse_tenants", "legendFormat": "Tenant scopes", "range": true, "refId": "B"},
+        {"editorMode": "code", "expr": "rendezvous_browser_replay_entries", "legendFormat": "Replay entries", "range": true, "refId": "C"}
+      ]
+    },
+    {
+      "id": 10,
+      "title": "UDP mediation results",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 8, "y": 12},
+      "fieldConfig": {"defaults": {"unit": "pps"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "sum by (operation, result) (rate(rendezvous_udp_results_total[5m]))", "legendFormat": "{{operation}} · {{result}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 11,
+      "title": "UDP p95 processing latency",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 16, "y": 12},
+      "fieldConfig": {"defaults": {"unit": "ms", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 100}]}}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(rendezvous_udp_duration_milliseconds_bucket[5m])))", "legendFormat": "{{operation}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 12,
+      "title": "UDP ingress and admitted response budget",
+      "description": "Traffic bytes observed by the process and the conservative maximum response budget admitted for successful introductions; this is not actual egress.",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 0, "y": 19},
+      "fieldConfig": {"defaults": {"unit": "Bps"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [
+        {"editorMode": "code", "expr": "sum by (operation) (rate(rendezvous_udp_received_bytes_total[5m]))", "legendFormat": "Ingress · {{operation}}", "range": true, "refId": "A"},
+        {"editorMode": "code", "expr": "sum by (operation) (rate(rendezvous_udp_response_budget_bytes_total[5m]))", "legendFormat": "Response budget · {{operation}}", "range": true, "refId": "B"}
+      ]
+    },
+    {
+      "id": 13,
+      "title": "Admission-control drops",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 8, "y": 19},
+      "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "sum by (transport, partition) (rate(rendezvous_limiter_drops_total[5m]))", "legendFormat": "{{transport}} · {{partition}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 14,
+      "title": "Direct-connect outcomes",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 16, "y": 19},
+      "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "sum by (outcome, elapsed_bucket) (rate(rendezvous_connection_outcomes_total[5m]))", "legendFormat": "{{outcome}} · {{elapsed_bucket}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 15,
+      "title": "Pairing p95 latency",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 0, "y": 26},
+      "fieldConfig": {"defaults": {"unit": "ms", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 100}]}}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "histogram_quantile(0.95, rate(rendezvous_pairing_latency_milliseconds_bucket[5m]))", "legendFormat": "Pairing p95", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 16,
+      "title": "Presence and expiry state",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 8, "y": 26},
+      "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [
+        {"editorMode": "code", "expr": "rendezvous_store_fresh_presence_bindings", "legendFormat": "Fresh presence", "range": true, "refId": "A"},
+        {"editorMode": "code", "expr": "rendezvous_store_awaiting_presence_listings", "legendFormat": "Awaiting presence", "range": true, "refId": "B"},
+        {"editorMode": "code", "expr": "rate(rendezvous_store_expiry_churn_total[5m])", "legendFormat": "Expiry churn/s", "range": true, "refId": "C"}
+      ]
+    },
+    {
+      "id": 17,
+      "title": "Signing key state",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 16, "y": 26},
+      "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "rendezvous_signing_keys", "legendFormat": "{{state}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 18,
+      "title": "Minimum signing window",
+      "description": "Page below seven days; escalate below 24 hours.",
+      "type": "stat",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 5, "w": 6, "x": 0, "y": 33},
+      "fieldConfig": {"defaults": {"unit": "s", "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "orange", "value": 86400}, {"color": "green", "value": 604800}]}}, "overrides": []},
+      "options": {"colorMode": "background", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true},
+      "targets": [{"editorMode": "code", "expr": "rendezvous_signing_key_sign_seconds_remaining", "legendFormat": "Signing window", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 19,
+      "title": "Operator authentication",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 5, "w": 6, "x": 6, "y": 33},
+      "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "sum by (result) (rate(rendezvous_operator_authentication_total[5m]))", "legendFormat": "{{result}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 20,
+      "title": "Privileged audit outcomes",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 5, "w": 6, "x": 12, "y": 33},
+      "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "sum by (action, result) (rate(rendezvous_audit_events_total[5m]))", "legendFormat": "{{action}} · {{result}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 21,
+      "title": "Metrics access",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 5, "w": 6, "x": 18, "y": 33},
+      "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "sum by (result) (rate(rendezvous_metrics_scrapes_total[5m]))", "legendFormat": "{{result}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 22,
+      "title": "Process memory",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 0, "y": 38},
+      "fieldConfig": {"defaults": {"unit": "bytes", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 1073741824}, {"color": "red", "value": 1610612736}]}}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [
+        {"editorMode": "code", "expr": "process_resident_memory_bytes", "legendFormat": "Resident", "range": true, "refId": "A"},
+        {"editorMode": "code", "expr": "dotnet_gc_heap_size_bytes", "legendFormat": "Managed heap", "range": true, "refId": "B"}
+      ]
+    },
+    {
+      "id": 23,
+      "title": "Process CPU and threads",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 8, "y": 38},
+      "fieldConfig": {"defaults": {"unit": "short"}, "overrides": [{"matcher": {"id": "byName", "options": "CPU cores"}, "properties": [{"id": "unit", "value": "cores"}]}]},
+      "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [
+        {"editorMode": "code", "expr": "rate(process_cpu_seconds_total[5m])", "legendFormat": "CPU cores", "range": true, "refId": "A"},
+        {"editorMode": "code", "expr": "process_threads", "legendFormat": "Process threads", "range": true, "refId": "B"},
+        {"editorMode": "code", "expr": "dotnet_thread_pool_threads", "legendFormat": "Thread-pool threads", "range": true, "refId": "C"}
+      ]
+    },
+    {
+      "id": 24,
+      "title": "Descriptor and GC pressure",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 16, "y": 38},
+      "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [
+        {"editorMode": "code", "expr": "process_open_file_descriptors", "legendFormat": "Open descriptors", "range": true, "refId": "A"},
+        {"editorMode": "code", "expr": "sum(rate(dotnet_gc_collections_total[5m]))", "legendFormat": "GC collections/s", "range": true, "refId": "B"},
+        {"editorMode": "code", "expr": "dotnet_thread_pool_available_worker_threads", "legendFormat": "Available workers", "range": true, "refId": "C"}
+      ]
+    }
+  ],
+  "refresh": "15s",
+  "schemaVersion": 41,
+  "tags": ["rendezvous", "operations", "privacy-safe"],
+  "templating": {"list": []},
+  "time": {"from": "now-1h", "to": "now"},
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Rendezvous operational overview",
+  "uid": "rendezvous-overview",
+  "version": 1,
+  "weekStart": ""
+}
@@ -0,0 +1,13 @@
+apiVersion: 1
+
+providers:
+  - name: Rendezvous
+    orgId: 1
+    folder: Rendezvous
+    type: file
+    disableDeletion: true
+    allowUiUpdates: false
+    updateIntervalSeconds: 30
+    options:
+      path: /var/lib/grafana/dashboards
+      foldersFromFilesStructure: false
@@ -0,0 +1,17 @@
+apiVersion: 1
+
+deleteDatasources:
+  - name: Rendezvous Prometheus
+    orgId: 1
+
+datasources:
+  - name: Rendezvous Prometheus
+    uid: rendezvous-prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false
+    jsonData:
+      httpMethod: POST
+      timeInterval: 15s