feat(observability): add diagnostic dashboards (#27)

deploy/compose/appsettings.Production.json

@@ -14,6 +14,29 @@
       "ListenAddress": "0.0.0.0",
       "Port": 9050
     },
+    "Diagnostics": {
+      "Enabled": false,
+      "PollIntervalSeconds": 10,
+      "MaximumRenderedSessions": 100,
+      "Scopes": [
+        {
+          "GameId": "space-game",
+          "EnvironmentId": "smoke",
+          "ProtocolVersions": [1, 2],
+          "Regions": ["local"]
+        },
+        {
+          "GameId": "unscouted",
+          "EnvironmentId": "smoke",
+          "ProtocolVersions": [1],
+          "Regions": ["local"]
+        }
+      ]
+    },
+    "Metrics": {
+      "Enabled": false,
+      "BearerTokenSecretReference": "file:/run/secrets/rendezvous-metrics-token"
+    },
     "AbuseProtection": {
       "TrustedProxyAddresses": ["127.0.0.1"],
       "OperatorAllowedAddresses": ["127.0.0.1"]

deploy/compose/compose.yaml

@@ -27,6 +27,8 @@ services:
     environment:
       ASPNETCORE_ENVIRONMENT: Production
       ASPNETCORE_HTTP_PORTS: "8080"
+      Rendezvous__Diagnostics__Enabled: "${RENDEZVOUS_DIAGNOSTICS_ENABLED:-false}"
+      Rendezvous__Metrics__Enabled: "${RENDEZVOUS_METRICS_ENABLED:-false}"
     volumes:
       - ./appsettings.Production.json:/app/appsettings.Production.json:ro
       - ./secrets/signing-key:/run/secrets/rendezvous-signing-key:ro

deploy/observability/compose.yaml

@@ -0,0 +1,71 @@
+services:
+  rendezvous:
+    environment:
+      Rendezvous__Metrics__Enabled: "true"
+    volumes:
+      - ../observability/secrets/rendezvous-metrics-token:/run/secrets/rendezvous-metrics-token:ro
+
+  prometheus:
+    image: prom/prometheus:v3.13.1@sha256:3c42b892cf723fa54d2f262c37a0e1f80aa8c8ddb1da7b9b0df9455a35a7f893
+    user: "65534:65534"
+    read_only: true
+    init: true
+    cap_drop:
+      - ALL
+    security_opt:
+      - no-new-privileges:true
+    pids_limit: 128
+    mem_limit: 512m
+    cpus: 1.0
+    restart: unless-stopped
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+      - --storage.tsdb.retention.time=15d
+    volumes:
+      - ../observability/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ../observability/secrets/rendezvous-metrics-token:/run/secrets/rendezvous-metrics-token:ro
+      - prometheus-data:/prometheus
+    depends_on:
+      - rendezvous
+
+  grafana:
+    image: grafana/grafana:13.1.0@sha256:121a7a9ece6dc10b969f1f96eed64b4f07dfac0d0b8abc070f7cb83bbde86f63
+    user: "472:472"
+    read_only: true
+    init: true
+    cap_drop:
+      - ALL
+    security_opt:
+      - no-new-privileges:true
+    pids_limit: 128
+    mem_limit: 512m
+    cpus: 1.0
+    restart: unless-stopped
+    environment:
+      GF_ANALYTICS_REPORTING_ENABLED: "false"
+      GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
+      GF_PLUGINS_PREINSTALL_DISABLED: "true"
+      GF_PLUGINS_PREINSTALL_AUTO_UPDATE: "false"
+      GF_SECURITY_ADMIN_USER: "rendezvous-admin"
+      GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana-admin-password
+      GF_USERS_ALLOW_SIGN_UP: "false"
+      GF_AUTH_ANONYMOUS_ENABLED: "false"
+      GF_SERVER_DOMAIN: localhost
+      GF_SERVER_ROOT_URL: http://localhost:3000
+    tmpfs:
+      - /tmp:rw,noexec,nosuid,nodev,size=32m,uid=472,gid=472,mode=0700
+    volumes:
+      - ../observability/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
+      - ../observability/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
+      - ../observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
+      - ../observability/secrets/grafana-admin-password:/run/secrets/grafana-admin-password:ro
+      - grafana-data:/var/lib/grafana
+    ports:
+      - "127.0.0.1:3000:3000/tcp"
+    depends_on:
+      - prometheus
+
+volumes:
+  prometheus-data:
+  grafana-data:

deploy/observability/grafana/dashboards/rendezvous-overview.json

@@ -0,0 +1,292 @@
+{
+  "annotations": {"list": []},
+  "description": "Privacy-safe operational view of the single-active Rendezvous service. Capacity percentages use the approved 25,000 listing and 10,000 active-attempt launch envelope.",
+  "editable": false,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "id": 1,
+      "title": "Service",
+      "description": "Prometheus can authenticate to and scrape the Rendezvous process.",
+      "type": "stat",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 4, "w": 4, "x": 0, "y": 0},
+      "fieldConfig": {"defaults": {"mappings": [{"options": {"0": {"color": "red", "text": "DOWN"}, "1": {"color": "green", "text": "UP"}}, "type": "value"}], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}, "overrides": []},
+      "options": {"colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true},
+      "targets": [{"editorMode": "code", "expr": "up{job=\"rendezvous\"}", "legendFormat": "Rendezvous", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 2,
+      "title": "Store",
+      "type": "stat",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 4, "w": 4, "x": 4, "y": 0},
+      "fieldConfig": {"defaults": {"mappings": [{"options": {"0": {"color": "red", "text": "UNAVAILABLE"}, "1": {"color": "green", "text": "AVAILABLE"}}, "type": "value"}], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}, "overrides": []},
+      "options": {"colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true},
+      "targets": [{"editorMode": "code", "expr": "rendezvous_store_available", "legendFormat": "Store", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 3,
+      "title": "Drain",
+      "type": "stat",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 4, "w": 4, "x": 8, "y": 0},
+      "fieldConfig": {"defaults": {"mappings": [{"options": {"0": {"color": "green", "text": "ACCEPTING"}, "1": {"color": "orange", "text": "DRAINING"}}, "type": "value"}], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 1}]}}, "overrides": []},
+      "options": {"colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true},
+      "targets": [{"editorMode": "code", "expr": "rendezvous_store_draining", "legendFormat": "Drain", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 4,
+      "title": "Listings",
+      "description": "Percentage of the approved 25,000-listing single-process envelope.",
+      "type": "gauge",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
+      "fieldConfig": {"defaults": {"max": 100, "min": 0, "unit": "percent", "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 70}, {"color": "red", "value": 90}]}}, "overrides": []},
+      "options": {"minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto"},
+      "targets": [{"editorMode": "code", "expr": "100 * rendezvous_store_active_listings / 25000", "legendFormat": "Listings", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 5,
+      "title": "Join attempts",
+      "description": "Percentage of the approved 10,000 active-attempt single-process envelope.",
+      "type": "gauge",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
+      "fieldConfig": {"defaults": {"max": 100, "min": 0, "unit": "percent", "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 70}, {"color": "red", "value": 90}]}}, "overrides": []},
+      "options": {"minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto"},
+      "targets": [{"editorMode": "code", "expr": "100 * rendezvous_store_active_attempts / 10000", "legendFormat": "Attempts", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 6,
+      "title": "HTTP request rate by operation",
+      "description": "Registration, renewal, browse, and join issuance appear as bounded operation names.",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 8, "w": 8, "x": 0, "y": 4},
+      "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "sum by (operation) (rate(rendezvous_http_requests_total[5m]))", "legendFormat": "{{operation}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 7,
+      "title": "HTTP p95 latency",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 8, "w": 8, "x": 8, "y": 4},
+      "fieldConfig": {"defaults": {"unit": "ms", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 200}]}}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(rendezvous_http_duration_milliseconds_bucket[5m])))", "legendFormat": "{{operation}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 8,
+      "title": "HTTP error and shedding rate",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 8, "w": 8, "x": 16, "y": 4},
+      "fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "sum by (status_code) (rate(rendezvous_http_requests_total{status_code=~\"4..|5..\"}[5m]))", "legendFormat": "HTTP {{status_code}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 9,
+      "title": "Browser live-update load",
+      "description": "Active public SSE clients, tenant scopes, and bounded replay entries.",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 0, "y": 12},
+      "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [
+        {"editorMode": "code", "expr": "rendezvous_browser_sse_subscribers", "legendFormat": "Subscribers", "range": true, "refId": "A"},
+        {"editorMode": "code", "expr": "rendezvous_browser_sse_tenants", "legendFormat": "Tenant scopes", "range": true, "refId": "B"},
+        {"editorMode": "code", "expr": "rendezvous_browser_replay_entries", "legendFormat": "Replay entries", "range": true, "refId": "C"}
+      ]
+    },
+    {
+      "id": 10,
+      "title": "UDP mediation results",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 8, "y": 12},
+      "fieldConfig": {"defaults": {"unit": "pps"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "sum by (operation, result) (rate(rendezvous_udp_results_total[5m]))", "legendFormat": "{{operation}} · {{result}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 11,
+      "title": "UDP p95 processing latency",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 16, "y": 12},
+      "fieldConfig": {"defaults": {"unit": "ms", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 100}]}}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(rendezvous_udp_duration_milliseconds_bucket[5m])))", "legendFormat": "{{operation}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 12,
+      "title": "UDP ingress and admitted response budget",
+      "description": "Traffic bytes observed by the process and the conservative maximum response budget admitted for successful introductions; this is not actual egress.",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 0, "y": 19},
+      "fieldConfig": {"defaults": {"unit": "Bps"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [
+        {"editorMode": "code", "expr": "sum by (operation) (rate(rendezvous_udp_received_bytes_total[5m]))", "legendFormat": "Ingress · {{operation}}", "range": true, "refId": "A"},
+        {"editorMode": "code", "expr": "sum by (operation) (rate(rendezvous_udp_response_budget_bytes_total[5m]))", "legendFormat": "Response budget · {{operation}}", "range": true, "refId": "B"}
+      ]
+    },
+    {
+      "id": 13,
+      "title": "Admission-control drops",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 8, "y": 19},
+      "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "sum by (transport, partition) (rate(rendezvous_limiter_drops_total[5m]))", "legendFormat": "{{transport}} · {{partition}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 14,
+      "title": "Direct-connect outcomes",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 16, "y": 19},
+      "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "sum by (outcome, elapsed_bucket) (rate(rendezvous_connection_outcomes_total[5m]))", "legendFormat": "{{outcome}} · {{elapsed_bucket}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 15,
+      "title": "Pairing p95 latency",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 0, "y": 26},
+      "fieldConfig": {"defaults": {"unit": "ms", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 100}]}}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "histogram_quantile(0.95, rate(rendezvous_pairing_latency_milliseconds_bucket[5m]))", "legendFormat": "Pairing p95", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 16,
+      "title": "Presence and expiry state",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 8, "y": 26},
+      "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [
+        {"editorMode": "code", "expr": "rendezvous_store_fresh_presence_bindings", "legendFormat": "Fresh presence", "range": true, "refId": "A"},
+        {"editorMode": "code", "expr": "rendezvous_store_awaiting_presence_listings", "legendFormat": "Awaiting presence", "range": true, "refId": "B"},
+        {"editorMode": "code", "expr": "rate(rendezvous_store_expiry_churn_total[5m])", "legendFormat": "Expiry churn/s", "range": true, "refId": "C"}
+      ]
+    },
+    {
+      "id": 17,
+      "title": "Signing key state",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 16, "y": 26},
+      "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "rendezvous_signing_keys", "legendFormat": "{{state}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 18,
+      "title": "Minimum signing window",
+      "description": "Page below seven days; escalate below 24 hours.",
+      "type": "stat",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 5, "w": 6, "x": 0, "y": 33},
+      "fieldConfig": {"defaults": {"unit": "s", "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "orange", "value": 86400}, {"color": "green", "value": 604800}]}}, "overrides": []},
+      "options": {"colorMode": "background", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true},
+      "targets": [{"editorMode": "code", "expr": "rendezvous_signing_key_sign_seconds_remaining", "legendFormat": "Signing window", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 19,
+      "title": "Operator authentication",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 5, "w": 6, "x": 6, "y": 33},
+      "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "sum by (result) (rate(rendezvous_operator_authentication_total[5m]))", "legendFormat": "{{result}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 20,
+      "title": "Privileged audit outcomes",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 5, "w": 6, "x": 12, "y": 33},
+      "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "sum by (action, result) (rate(rendezvous_audit_events_total[5m]))", "legendFormat": "{{action}} · {{result}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 21,
+      "title": "Metrics access",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 5, "w": 6, "x": 18, "y": 33},
+      "fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [{"editorMode": "code", "expr": "sum by (result) (rate(rendezvous_metrics_scrapes_total[5m]))", "legendFormat": "{{result}}", "range": true, "refId": "A"}]
+    },
+    {
+      "id": 22,
+      "title": "Process memory",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 0, "y": 38},
+      "fieldConfig": {"defaults": {"unit": "bytes", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 1073741824}, {"color": "red", "value": 1610612736}]}}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [
+        {"editorMode": "code", "expr": "process_resident_memory_bytes", "legendFormat": "Resident", "range": true, "refId": "A"},
+        {"editorMode": "code", "expr": "dotnet_gc_heap_size_bytes", "legendFormat": "Managed heap", "range": true, "refId": "B"}
+      ]
+    },
+    {
+      "id": 23,
+      "title": "Process CPU and threads",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 8, "y": 38},
+      "fieldConfig": {"defaults": {"unit": "short"}, "overrides": [{"matcher": {"id": "byName", "options": "CPU cores"}, "properties": [{"id": "unit", "value": "cores"}]}]},
+      "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [
+        {"editorMode": "code", "expr": "rate(process_cpu_seconds_total[5m])", "legendFormat": "CPU cores", "range": true, "refId": "A"},
+        {"editorMode": "code", "expr": "process_threads", "legendFormat": "Process threads", "range": true, "refId": "B"},
+        {"editorMode": "code", "expr": "dotnet_thread_pool_threads", "legendFormat": "Thread-pool threads", "range": true, "refId": "C"}
+      ]
+    },
+    {
+      "id": 24,
+      "title": "Descriptor and GC pressure",
+      "type": "timeseries",
+      "datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
+      "gridPos": {"h": 7, "w": 8, "x": 16, "y": 38},
+      "fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
+      "options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
+      "targets": [
+        {"editorMode": "code", "expr": "process_open_file_descriptors", "legendFormat": "Open descriptors", "range": true, "refId": "A"},
+        {"editorMode": "code", "expr": "sum(rate(dotnet_gc_collections_total[5m]))", "legendFormat": "GC collections/s", "range": true, "refId": "B"},
+        {"editorMode": "code", "expr": "dotnet_thread_pool_available_worker_threads", "legendFormat": "Available workers", "range": true, "refId": "C"}
+      ]
+    }
+  ],
+  "refresh": "15s",
+  "schemaVersion": 41,
+  "tags": ["rendezvous", "operations", "privacy-safe"],
+  "templating": {"list": []},
+  "time": {"from": "now-1h", "to": "now"},
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Rendezvous operational overview",
+  "uid": "rendezvous-overview",
+  "version": 1,
+  "weekStart": ""
+}

deploy/observability/grafana/provisioning/dashboards/rendezvous.yaml

@@ -0,0 +1,13 @@
+apiVersion: 1
+
+providers:
+  - name: Rendezvous
+    orgId: 1
+    folder: Rendezvous
+    type: file
+    disableDeletion: true
+    allowUiUpdates: false
+    updateIntervalSeconds: 30
+    options:
+      path: /var/lib/grafana/dashboards
+      foldersFromFilesStructure: false

deploy/observability/grafana/provisioning/datasources/prometheus.yaml

@@ -0,0 +1,17 @@
+apiVersion: 1
+
+deleteDatasources:
+  - name: Rendezvous Prometheus
+    orgId: 1
+
+datasources:
+  - name: Rendezvous Prometheus
+    uid: rendezvous-prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false
+    jsonData:
+      httpMethod: POST
+      timeInterval: 15s

deploy/observability/prometheus/prometheus.yml

@@ -0,0 +1,14 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+  external_labels:
+    service: rendezvous
+
+scrape_configs:
+  - job_name: rendezvous
+    scheme: http
+    metrics_path: /metrics
+    bearer_token_file: /run/secrets/rendezvous-metrics-token
+    static_configs:
+      - targets:
+          - rendezvous:8080

deploy/observability/secrets/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore