feat(observability): add diagnostic dashboards (#27)
This commit is contained in:
KyuubiYoru
@@ -45,6 +45,12 @@ jobs:
|
||||
- name: Verify formatting and analyzers
|
||||
run: dotnet format Rendezvous.slnx --verify-no-changes --no-restore
|
||||
|
||||
- name: Test dependency-free diagnostic dashboard
|
||||
run: ./scripts/test-diagnostic-dashboard.sh
|
||||
|
||||
- name: Test observability dashboard provisioning
|
||||
run: ./scripts/test-observability-assets.sh
|
||||
|
||||
- name: Build
|
||||
run: dotnet build Rendezvous.slnx --configuration Release --no-restore
|
||||
|
||||
|
||||
@@ -84,10 +84,10 @@ directory leases, authenticated join attempts, LiteNetLib mediator, caller-owned
|
||||
SDK coordination, typed connection outcomes, thin public-SDK diagnostic client,
|
||||
deterministic NAT topology harness, hostile-input controls,
|
||||
observability/operator surface, secure single-active Linux deployment, and
|
||||
numeric capacity/resilience gates, and reproducible signed release pipeline are
|
||||
implemented. Consumer pilots and final production-readiness gates remain in progress;
|
||||
participating games must not treat the current repository as a finished production
|
||||
service until those gates land.
|
||||
numeric capacity/resilience gates, reproducible signed release pipeline, consumer
|
||||
pilot integrations, and production-readiness decision framework are implemented.
|
||||
Deployment-specific external canaries and optional roadmap follow-ups remain;
|
||||
participating games must not treat a checkout alone as production approval.
|
||||
|
||||
The ratified v1 boundaries, trust decisions, privacy rules, safety budgets, and
|
||||
threat model are indexed in [the architecture documentation](docs/architecture/README.md).
|
||||
@@ -100,6 +100,8 @@ defined in [hostile-input and overload protection](docs/security/abuse-protectio
|
||||
Health semantics, bounded telemetry, alerting, audit privacy, and the authenticated
|
||||
operator controls are defined in the
|
||||
[observability and operator runbook](docs/operations/observability-and-operator-runbook.md).
|
||||
The optional public session diagnostic and the private provisioned Grafana stack
|
||||
are described in [diagnostic dashboards](docs/operations/diagnostic-dashboards.md).
|
||||
Concrete detect/contain/recover/verify procedures are in the
|
||||
[incident and change runbooks](docs/operations/incident-runbooks.md).
|
||||
The pinned non-root container, production topology, graceful drain, Linux
|
||||
|
||||
@@ -14,6 +14,29 @@
|
||||
"ListenAddress": "0.0.0.0",
|
||||
"Port": 9050
|
||||
},
|
||||
"Diagnostics": {
|
||||
"Enabled": false,
|
||||
"PollIntervalSeconds": 10,
|
||||
"MaximumRenderedSessions": 100,
|
||||
"Scopes": [
|
||||
{
|
||||
"GameId": "space-game",
|
||||
"EnvironmentId": "smoke",
|
||||
"ProtocolVersions": [1, 2],
|
||||
"Regions": ["local"]
|
||||
},
|
||||
{
|
||||
"GameId": "unscouted",
|
||||
"EnvironmentId": "smoke",
|
||||
"ProtocolVersions": [1],
|
||||
"Regions": ["local"]
|
||||
}
|
||||
]
|
||||
},
|
||||
"Metrics": {
|
||||
"Enabled": false,
|
||||
"BearerTokenSecretReference": "file:/run/secrets/rendezvous-metrics-token"
|
||||
},
|
||||
"AbuseProtection": {
|
||||
"TrustedProxyAddresses": ["127.0.0.1"],
|
||||
"OperatorAllowedAddresses": ["127.0.0.1"]
|
||||
|
||||
@@ -27,6 +27,8 @@ services:
|
||||
environment:
|
||||
ASPNETCORE_ENVIRONMENT: Production
|
||||
ASPNETCORE_HTTP_PORTS: "8080"
|
||||
Rendezvous__Diagnostics__Enabled: "${RENDEZVOUS_DIAGNOSTICS_ENABLED:-false}"
|
||||
Rendezvous__Metrics__Enabled: "${RENDEZVOUS_METRICS_ENABLED:-false}"
|
||||
volumes:
|
||||
- ./appsettings.Production.json:/app/appsettings.Production.json:ro
|
||||
- ./secrets/signing-key:/run/secrets/rendezvous-signing-key:ro
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
services:
|
||||
rendezvous:
|
||||
environment:
|
||||
Rendezvous__Metrics__Enabled: "true"
|
||||
volumes:
|
||||
- ../observability/secrets/rendezvous-metrics-token:/run/secrets/rendezvous-metrics-token:ro
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.13.1@sha256:3c42b892cf723fa54d2f262c37a0e1f80aa8c8ddb1da7b9b0df9455a35a7f893
|
||||
user: "65534:65534"
|
||||
read_only: true
|
||||
init: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
pids_limit: 128
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=15d
|
||||
volumes:
|
||||
- ../observability/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- ../observability/secrets/rendezvous-metrics-token:/run/secrets/rendezvous-metrics-token:ro
|
||||
- prometheus-data:/prometheus
|
||||
depends_on:
|
||||
- rendezvous
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:13.1.0@sha256:121a7a9ece6dc10b969f1f96eed64b4f07dfac0d0b8abc070f7cb83bbde86f63
|
||||
user: "472:472"
|
||||
read_only: true
|
||||
init: true
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
pids_limit: 128
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||
GF_PLUGINS_PREINSTALL_DISABLED: "true"
|
||||
GF_PLUGINS_PREINSTALL_AUTO_UPDATE: "false"
|
||||
GF_SECURITY_ADMIN_USER: "rendezvous-admin"
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana-admin-password
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "false"
|
||||
GF_SERVER_DOMAIN: localhost
|
||||
GF_SERVER_ROOT_URL: http://localhost:3000
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,nodev,size=32m,uid=472,gid=472,mode=0700
|
||||
volumes:
|
||||
- ../observability/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
|
||||
- ../observability/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
|
||||
- ../observability/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- ../observability/secrets/grafana-admin-password:/run/secrets/grafana-admin-password:ro
|
||||
- grafana-data:/var/lib/grafana
|
||||
ports:
|
||||
- "127.0.0.1:3000:3000/tcp"
|
||||
depends_on:
|
||||
- prometheus
|
||||
|
||||
volumes:
|
||||
prometheus-data:
|
||||
grafana-data:
|
||||
@@ -0,0 +1,292 @@
|
||||
{
|
||||
"annotations": {"list": []},
|
||||
"description": "Privacy-safe operational view of the single-active Rendezvous service. Capacity percentages use the approved 25,000 listing and 10,000 active-attempt launch envelope.",
|
||||
"editable": false,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Service",
|
||||
"description": "Prometheus can authenticate to and scrape the Rendezvous process.",
|
||||
"type": "stat",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 0},
|
||||
"fieldConfig": {"defaults": {"mappings": [{"options": {"0": {"color": "red", "text": "DOWN"}, "1": {"color": "green", "text": "UP"}}, "type": "value"}], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}, "overrides": []},
|
||||
"options": {"colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true},
|
||||
"targets": [{"editorMode": "code", "expr": "up{job=\"rendezvous\"}", "legendFormat": "Rendezvous", "range": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Store",
|
||||
"type": "stat",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 0},
|
||||
"fieldConfig": {"defaults": {"mappings": [{"options": {"0": {"color": "red", "text": "UNAVAILABLE"}, "1": {"color": "green", "text": "AVAILABLE"}}, "type": "value"}], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}, "overrides": []},
|
||||
"options": {"colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true},
|
||||
"targets": [{"editorMode": "code", "expr": "rendezvous_store_available", "legendFormat": "Store", "range": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Drain",
|
||||
"type": "stat",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 4, "w": 4, "x": 8, "y": 0},
|
||||
"fieldConfig": {"defaults": {"mappings": [{"options": {"0": {"color": "green", "text": "ACCEPTING"}, "1": {"color": "orange", "text": "DRAINING"}}, "type": "value"}], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 1}]}}, "overrides": []},
|
||||
"options": {"colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true},
|
||||
"targets": [{"editorMode": "code", "expr": "rendezvous_store_draining", "legendFormat": "Drain", "range": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Listings",
|
||||
"description": "Percentage of the approved 25,000-listing single-process envelope.",
|
||||
"type": "gauge",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
|
||||
"fieldConfig": {"defaults": {"max": 100, "min": 0, "unit": "percent", "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 70}, {"color": "red", "value": 90}]}}, "overrides": []},
|
||||
"options": {"minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto"},
|
||||
"targets": [{"editorMode": "code", "expr": "100 * rendezvous_store_active_listings / 25000", "legendFormat": "Listings", "range": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Join attempts",
|
||||
"description": "Percentage of the approved 10,000 active-attempt single-process envelope.",
|
||||
"type": "gauge",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
|
||||
"fieldConfig": {"defaults": {"max": 100, "min": 0, "unit": "percent", "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 70}, {"color": "red", "value": 90}]}}, "overrides": []},
|
||||
"options": {"minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto"},
|
||||
"targets": [{"editorMode": "code", "expr": "100 * rendezvous_store_active_attempts / 10000", "legendFormat": "Attempts", "range": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "HTTP request rate by operation",
|
||||
"description": "Registration, renewal, browse, and join issuance appear as bounded operation names.",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 4},
|
||||
"fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
|
||||
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [{"editorMode": "code", "expr": "sum by (operation) (rate(rendezvous_http_requests_total[5m]))", "legendFormat": "{{operation}}", "range": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "HTTP p95 latency",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 4},
|
||||
"fieldConfig": {"defaults": {"unit": "ms", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 200}]}}, "overrides": []},
|
||||
"options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [{"editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(rendezvous_http_duration_milliseconds_bucket[5m])))", "legendFormat": "{{operation}}", "range": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "HTTP error and shedding rate",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 4},
|
||||
"fieldConfig": {"defaults": {"unit": "reqps"}, "overrides": []},
|
||||
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [{"editorMode": "code", "expr": "sum by (status_code) (rate(rendezvous_http_requests_total{status_code=~\"4..|5..\"}[5m]))", "legendFormat": "HTTP {{status_code}}", "range": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Browser live-update load",
|
||||
"description": "Active public SSE clients, tenant scopes, and bounded replay entries.",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 0, "y": 12},
|
||||
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
|
||||
"options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
{"editorMode": "code", "expr": "rendezvous_browser_sse_subscribers", "legendFormat": "Subscribers", "range": true, "refId": "A"},
|
||||
{"editorMode": "code", "expr": "rendezvous_browser_sse_tenants", "legendFormat": "Tenant scopes", "range": true, "refId": "B"},
|
||||
{"editorMode": "code", "expr": "rendezvous_browser_replay_entries", "legendFormat": "Replay entries", "range": true, "refId": "C"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "UDP mediation results",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 8, "y": 12},
|
||||
"fieldConfig": {"defaults": {"unit": "pps"}, "overrides": []},
|
||||
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [{"editorMode": "code", "expr": "sum by (operation, result) (rate(rendezvous_udp_results_total[5m]))", "legendFormat": "{{operation}} · {{result}}", "range": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "UDP p95 processing latency",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 16, "y": 12},
|
||||
"fieldConfig": {"defaults": {"unit": "ms", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 100}]}}, "overrides": []},
|
||||
"options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [{"editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(rendezvous_udp_duration_milliseconds_bucket[5m])))", "legendFormat": "{{operation}}", "range": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "UDP ingress and admitted response budget",
|
||||
"description": "Traffic bytes observed by the process and the conservative maximum response budget admitted for successful introductions; this is not actual egress.",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 0, "y": 19},
|
||||
"fieldConfig": {"defaults": {"unit": "Bps"}, "overrides": []},
|
||||
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
{"editorMode": "code", "expr": "sum by (operation) (rate(rendezvous_udp_received_bytes_total[5m]))", "legendFormat": "Ingress · {{operation}}", "range": true, "refId": "A"},
|
||||
{"editorMode": "code", "expr": "sum by (operation) (rate(rendezvous_udp_response_budget_bytes_total[5m]))", "legendFormat": "Response budget · {{operation}}", "range": true, "refId": "B"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"title": "Admission-control drops",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 8, "y": 19},
|
||||
"fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
|
||||
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [{"editorMode": "code", "expr": "sum by (transport, partition) (rate(rendezvous_limiter_drops_total[5m]))", "legendFormat": "{{transport}} · {{partition}}", "range": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"title": "Direct-connect outcomes",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 16, "y": 19},
|
||||
"fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
|
||||
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [{"editorMode": "code", "expr": "sum by (outcome, elapsed_bucket) (rate(rendezvous_connection_outcomes_total[5m]))", "legendFormat": "{{outcome}} · {{elapsed_bucket}}", "range": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"title": "Pairing p95 latency",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 0, "y": 26},
|
||||
"fieldConfig": {"defaults": {"unit": "ms", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 100}]}}, "overrides": []},
|
||||
"options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [{"editorMode": "code", "expr": "histogram_quantile(0.95, rate(rendezvous_pairing_latency_milliseconds_bucket[5m]))", "legendFormat": "Pairing p95", "range": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"title": "Presence and expiry state",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 8, "y": 26},
|
||||
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
|
||||
"options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
{"editorMode": "code", "expr": "rendezvous_store_fresh_presence_bindings", "legendFormat": "Fresh presence", "range": true, "refId": "A"},
|
||||
{"editorMode": "code", "expr": "rendezvous_store_awaiting_presence_listings", "legendFormat": "Awaiting presence", "range": true, "refId": "B"},
|
||||
{"editorMode": "code", "expr": "rate(rendezvous_store_expiry_churn_total[5m])", "legendFormat": "Expiry churn/s", "range": true, "refId": "C"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"title": "Signing key state",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 16, "y": 26},
|
||||
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
|
||||
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [{"editorMode": "code", "expr": "rendezvous_signing_keys", "legendFormat": "{{state}}", "range": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"id": 18,
|
||||
"title": "Minimum signing window",
|
||||
"description": "Page below seven days; escalate below 24 hours.",
|
||||
"type": "stat",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 5, "w": 6, "x": 0, "y": 33},
|
||||
"fieldConfig": {"defaults": {"unit": "s", "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "orange", "value": 86400}, {"color": "green", "value": 604800}]}}, "overrides": []},
|
||||
"options": {"colorMode": "background", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto", "wideLayout": true},
|
||||
"targets": [{"editorMode": "code", "expr": "rendezvous_signing_key_sign_seconds_remaining", "legendFormat": "Signing window", "range": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"id": 19,
|
||||
"title": "Operator authentication",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 5, "w": 6, "x": 6, "y": 33},
|
||||
"fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
|
||||
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [{"editorMode": "code", "expr": "sum by (result) (rate(rendezvous_operator_authentication_total[5m]))", "legendFormat": "{{result}}", "range": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"id": 20,
|
||||
"title": "Privileged audit outcomes",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 5, "w": 6, "x": 12, "y": 33},
|
||||
"fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
|
||||
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [{"editorMode": "code", "expr": "sum by (action, result) (rate(rendezvous_audit_events_total[5m]))", "legendFormat": "{{action}} · {{result}}", "range": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"id": 21,
|
||||
"title": "Metrics access",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 5, "w": 6, "x": 18, "y": 33},
|
||||
"fieldConfig": {"defaults": {"unit": "ops"}, "overrides": []},
|
||||
"options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [{"editorMode": "code", "expr": "sum by (result) (rate(rendezvous_metrics_scrapes_total[5m]))", "legendFormat": "{{result}}", "range": true, "refId": "A"}]
|
||||
},
|
||||
{
|
||||
"id": 22,
|
||||
"title": "Process memory",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 0, "y": 38},
|
||||
"fieldConfig": {"defaults": {"unit": "bytes", "custom": {"thresholdsStyle": {"mode": "line"}}, "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 1073741824}, {"color": "red", "value": 1610612736}]}}, "overrides": []},
|
||||
"options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
{"editorMode": "code", "expr": "process_resident_memory_bytes", "legendFormat": "Resident", "range": true, "refId": "A"},
|
||||
{"editorMode": "code", "expr": "dotnet_gc_heap_size_bytes", "legendFormat": "Managed heap", "range": true, "refId": "B"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 23,
|
||||
"title": "Process CPU and threads",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 8, "y": 38},
|
||||
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": [{"matcher": {"id": "byName", "options": "CPU cores"}, "properties": [{"id": "unit", "value": "cores"}]}]},
|
||||
"options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
{"editorMode": "code", "expr": "rate(process_cpu_seconds_total[5m])", "legendFormat": "CPU cores", "range": true, "refId": "A"},
|
||||
{"editorMode": "code", "expr": "process_threads", "legendFormat": "Process threads", "range": true, "refId": "B"},
|
||||
{"editorMode": "code", "expr": "dotnet_thread_pool_threads", "legendFormat": "Thread-pool threads", "range": true, "refId": "C"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 24,
|
||||
"title": "Descriptor and GC pressure",
|
||||
"type": "timeseries",
|
||||
"datasource": {"type": "prometheus", "uid": "rendezvous-prometheus"},
|
||||
"gridPos": {"h": 7, "w": 8, "x": 16, "y": 38},
|
||||
"fieldConfig": {"defaults": {"unit": "short"}, "overrides": []},
|
||||
"options": {"legend": {"calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
|
||||
"targets": [
|
||||
{"editorMode": "code", "expr": "process_open_file_descriptors", "legendFormat": "Open descriptors", "range": true, "refId": "A"},
|
||||
{"editorMode": "code", "expr": "sum(rate(dotnet_gc_collections_total[5m]))", "legendFormat": "GC collections/s", "range": true, "refId": "B"},
|
||||
{"editorMode": "code", "expr": "dotnet_thread_pool_available_worker_threads", "legendFormat": "Available workers", "range": true, "refId": "C"}
|
||||
]
|
||||
}
|
||||
],
|
||||
"refresh": "15s",
|
||||
"schemaVersion": 41,
|
||||
"tags": ["rendezvous", "operations", "privacy-safe"],
|
||||
"templating": {"list": []},
|
||||
"time": {"from": "now-1h", "to": "now"},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "Rendezvous operational overview",
|
||||
"uid": "rendezvous-overview",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: Rendezvous
|
||||
orgId: 1
|
||||
folder: Rendezvous
|
||||
type: file
|
||||
disableDeletion: true
|
||||
allowUiUpdates: false
|
||||
updateIntervalSeconds: 30
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
foldersFromFilesStructure: false
|
||||
@@ -0,0 +1,17 @@
|
||||
apiVersion: 1
|
||||
|
||||
deleteDatasources:
|
||||
- name: Rendezvous Prometheus
|
||||
orgId: 1
|
||||
|
||||
datasources:
|
||||
- name: Rendezvous Prometheus
|
||||
uid: rendezvous-prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
timeInterval: 15s
|
||||
@@ -0,0 +1,14 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
external_labels:
|
||||
service: rendezvous
|
||||
|
||||
scrape_configs:
|
||||
- job_name: rendezvous
|
||||
scheme: http
|
||||
metrics_path: /metrics
|
||||
bearer_token_file: /run/secrets/rendezvous-metrics-token
|
||||
static_configs:
|
||||
- targets:
|
||||
- rendezvous:8080
|
||||
@@ -0,0 +1,2 @@
|
||||
*
|
||||
!.gitignore
|
||||
@@ -0,0 +1,123 @@
|
||||
# Diagnostic dashboards
|
||||
|
||||
Tracking: #27
|
||||
|
||||
Rendezvous provides two deliberately separate, optional views. The public
|
||||
session diagnostic helps a player or integration operator understand safe
|
||||
session-list state using only the public browse contract. The private Grafana
|
||||
dashboard exposes aggregate operational health through authenticated metrics.
|
||||
Neither view grants operator privileges or exposes player identity, endpoints,
|
||||
credentials, capabilities, or raw session metadata beyond the explicitly
|
||||
allowlisted public browse fields.
|
||||
|
||||
## Public read-only session diagnostic
|
||||
|
||||
The static diagnostic is disabled by default. Enable it only for approved
|
||||
game/environment scopes and keep the allowlist narrow:
|
||||
|
||||
```json
|
||||
"Diagnostics": {
|
||||
"Enabled": true,
|
||||
"PollIntervalSeconds": 10,
|
||||
"MaximumRenderedSessions": 100,
|
||||
"Scopes": [
|
||||
{
|
||||
"GameId": "space-game",
|
||||
"EnvironmentId": "smoke",
|
||||
"ProtocolVersions": [1, 2],
|
||||
"Regions": ["local"]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Open `/diagnostics` on the same origin as Rendezvous. The page cannot choose a
|
||||
different backend, request private visibility, join a session, or call the
|
||||
operator surface. It renders a bounded snapshot, then applies ordered SSE
|
||||
updates. A replay reset, corrupt cursor, incomplete snapshot, transport failure,
|
||||
or deliberate reconnect returns to a fresh authoritative snapshot and bounded
|
||||
polling. Apply filter changes explicitly; **Reset filters** restores the
|
||||
configured defaults, while **Reconnect now** tests recovery without changing
|
||||
the selection.
|
||||
|
||||
The page uses semantic HTML, labelled controls, visible keyboard focus, status
|
||||
text in addition to color, a reduced-motion mode, and a 320-pixel reflow. It
|
||||
creates untrusted content with `textContent` only. The endpoint sets a restrictive
|
||||
same-origin content-security policy, denies framing, disables MIME sniffing and
|
||||
browser capabilities, and marks every asset/config response `no-store`.
|
||||
|
||||
This is a diagnostics convenience, not a game browser, management console, or
|
||||
availability monitor. Disable it independently by setting `Enabled` to `false`;
|
||||
all diagnostic paths then return `404` without affecting game traffic, metrics,
|
||||
or health endpoints.
|
||||
|
||||
## Private Prometheus and Grafana view
|
||||
|
||||
The observability overlay pins Prometheus 3.13.1 and Grafana 13.1.0 by immutable
|
||||
multi-platform image digest. Prometheus is not published to the host. Grafana is
|
||||
bound to host loopback, disables anonymous access and sign-up, and reads its
|
||||
administrator password from a file. The service and Prometheus share only the
|
||||
metrics bearer-token file. All three secrets remain ignored by Git.
|
||||
|
||||
Create independent random secrets, then start the base service and overlay:
|
||||
|
||||
```bash
|
||||
install -d -m 0700 deploy/compose/secrets deploy/observability/secrets
|
||||
umask 077
|
||||
openssl rand -out deploy/compose/secrets/signing-key 32
|
||||
openssl rand -hex 32 >deploy/observability/secrets/rendezvous-metrics-token
|
||||
openssl rand -base64 36 >deploy/observability/secrets/grafana-admin-password
|
||||
export RENDEZVOUS_UID="$(id -u)"
|
||||
export RENDEZVOUS_GID="$(id -g)"
|
||||
test "$RENDEZVOUS_UID" -ne 0
|
||||
docker compose \
|
||||
-f deploy/compose/compose.yaml \
|
||||
-f deploy/observability/compose.yaml \
|
||||
up --build --detach
|
||||
```
|
||||
|
||||
Visit `http://127.0.0.1:3000`, sign in as `rendezvous-admin`, and open the
|
||||
**Rendezvous operational overview** folder/dashboard. The provisioned panels
|
||||
cover scrape/store/drain health, listing and join capacity, HTTP volume/errors
|
||||
and p95, browse/SSE load, lease and join operations, UDP results/latency/bytes,
|
||||
admission drops, connection outcomes, pairing latency, presence/expiry state,
|
||||
signing windows, security/audit results, and process/GC/descriptor pressure.
|
||||
Capacity gauges use the approved single-process envelope of 25,000 listings and
|
||||
10,000 active attempts, with 70% warning and 90% critical thresholds. The UDP
|
||||
response series is a conservative admitted maximum, not observed egress.
|
||||
|
||||
Validate merged configuration and checked-in dashboard structure before every
|
||||
rollout:
|
||||
|
||||
```bash
|
||||
RENDEZVOUS_UID="$(id -u)" RENDEZVOUS_GID="$(id -g)" \
|
||||
docker compose \
|
||||
-f deploy/compose/compose.yaml \
|
||||
-f deploy/observability/compose.yaml \
|
||||
config --quiet
|
||||
./scripts/test-observability-assets.sh
|
||||
```
|
||||
|
||||
For a real deployment, keep Grafana on a private authenticated management
|
||||
network instead of host loopback, replace the local admin login with the
|
||||
organization's supported identity boundary, enforce TLS at the edge, and set
|
||||
retention to the approved operational period. Do not make Prometheus public.
|
||||
Provisioning is read-only so local UI edits cannot silently drift from source.
|
||||
|
||||
## Verify, rotate, and disable
|
||||
|
||||
After startup, verify the dashboard shows `UP`, store `AVAILABLE`, a nonzero
|
||||
signing window, and changing request/UDP panels during a smoke run. Confirm an
|
||||
unauthenticated `/metrics` request returns `404`, the bearer-authenticated
|
||||
collector target is healthy, Prometheus is not bound on a host port, Grafana is
|
||||
not anonymously accessible, and dashboard query labels contain no identifiers.
|
||||
|
||||
Rotate metrics access by writing a new 32-128 character token to the secret file
|
||||
with private permissions and restarting Rendezvous and Prometheus together.
|
||||
Rotate the Grafana administrator password through the same protected secret
|
||||
workflow. Delete both secret files after a disposable local run.
|
||||
|
||||
To disable aggregate observability independently, stop/remove the overlay and
|
||||
set `Rendezvous:Metrics:Enabled` to `false`; `/metrics` returns `404` and the
|
||||
core service continues. To disable only the public session diagnostic, leave the
|
||||
overlay running and set `Rendezvous:Diagnostics:Enabled` to `false`.
|
||||
@@ -2,9 +2,10 @@
|
||||
|
||||
This runbook defines the production signals and privileged controls for the
|
||||
Rendezvous service. The service emits `System.Diagnostics.Metrics` instruments
|
||||
from the `FinalFactory.Rendezvous` meter and distributed-tracing activities from
|
||||
`FinalFactory.Rendezvous.Server`. Connect those sources to the deployment's
|
||||
OpenTelemetry or equivalent collector. Do not add identifiers to metric labels.
|
||||
from the `FinalFactory.Rendezvous` meter, distributed-tracing activities from
|
||||
`FinalFactory.Rendezvous.Server`, and an optional bearer-protected Prometheus
|
||||
endpoint. Connect only a private collector network. Do not add identifiers to
|
||||
metric labels.
|
||||
|
||||
Concrete detect/contain/recover/verify procedures for abuse, key compromise,
|
||||
targeted revocation, restart, rollback, saturation, privacy incidents, and
|
||||
@@ -38,6 +39,25 @@ dependency upgrades are in the [incident and change runbooks](incident-runbooks.
|
||||
| `rendezvous.store.expiry_churn` | Cumulative natural expiry activity | none |
|
||||
| `rendezvous.store.available` | Store health (`1` available, `0` unavailable) | none |
|
||||
|
||||
The Prometheus exporter additionally exposes active/fresh/awaiting store state,
|
||||
drain state, SSE subscriber/tenant/replay gauges, bounded HTTP and UDP
|
||||
histograms, UDP ingress and conservative admitted-response budgets, signing-key
|
||||
state/window gauges, and process/.NET pressure. Its only labels are the fixed
|
||||
operation, status, result, transport, partition, action, outcome, elapsed-bucket,
|
||||
key-state, and GC-generation dimensions. Unknown or unsafe values normalize to
|
||||
`other`; identifiers, metadata, addresses, endpoints, tokens, and capabilities
|
||||
are never labels.
|
||||
|
||||
The exporter is disabled by default. Enabling `Rendezvous:Metrics:Enabled`
|
||||
requires `BearerTokenSecretReference` to be an external `env:` or absolute
|
||||
`file:` secret containing 32-128 visible ASCII bytes. Unauthorized requests get
|
||||
the same `404` as a disabled endpoint, and accepted responses are `no-store`.
|
||||
Expose `/metrics` only to the private collector network, rotate its token as a
|
||||
deployment secret, and never place the token in a URL, Compose environment
|
||||
value, dashboard, log, or issue. The checked-in Prometheus/Grafana provisioning
|
||||
and its verification procedure are in
|
||||
[diagnostic dashboards](diagnostic-dashboards.md).
|
||||
|
||||
HTTP responses include `X-Rendezvous-Correlation-ID`. It is a generated trace ID
|
||||
or random value, never a caller-supplied session or player identifier. UDP and
|
||||
HTTP activities contain operation-level data only. Logs and traces must not add
|
||||
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$root"
|
||||
node --test tests/Diagnostics/*.test.mjs
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
node --test tests/Observability/observability-assets.test.mjs
|
||||
@@ -143,6 +143,39 @@ internal sealed class SessionChangeJournal
|
||||
|
||||
public SessionChangeJournalOptions Options => _options;
|
||||
|
||||
public int ReplayCount
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _changes.Count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public int SubscriberCount
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _subscribers;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public int SubscribedTenantCount
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _subscribersByTenant.Count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public long CurrentRevision
|
||||
{
|
||||
get
|
||||
|
||||
@@ -0,0 +1,615 @@
|
||||
const CONTRACT_VERSION = 1;
|
||||
const BROWSER_PAGE_LIMIT = 100;
|
||||
const REQUEST_TIMEOUT_MS = 10_000;
|
||||
const STREAM_FAILURE_LIMIT = 3;
|
||||
|
||||
export class SessionProjection {
|
||||
constructor(maximumSessions = 100) {
|
||||
if (!Number.isInteger(maximumSessions) || maximumSessions < 1 || maximumSessions > 500) {
|
||||
throw new RangeError("maximumSessions must be between 1 and 500");
|
||||
}
|
||||
this.maximumSessions = maximumSessions;
|
||||
this.entries = new Map();
|
||||
this.cursor = "";
|
||||
this.hasMore = false;
|
||||
}
|
||||
|
||||
replace(items, cursor, hasMore = false, updatedAt = Date.now()) {
|
||||
if (!Array.isArray(items) || items.length > this.maximumSessions) {
|
||||
return "overflow";
|
||||
}
|
||||
const next = new Map();
|
||||
for (const session of items) {
|
||||
if (!isPublicSession(session) || next.has(session.listingId)) {
|
||||
return "invalid";
|
||||
}
|
||||
next.set(session.listingId, { session, updatedAt });
|
||||
}
|
||||
this.entries = next;
|
||||
this.cursor = validCursor(cursor) ? cursor : "";
|
||||
this.hasMore = Boolean(hasMore);
|
||||
return "applied";
|
||||
}
|
||||
|
||||
apply(event, updatedAt = Date.now()) {
|
||||
if (!event || event.contractVersion !== CONTRACT_VERSION || !validCursor(event.cursor)) {
|
||||
return "invalid";
|
||||
}
|
||||
if (event.kind === "sessionUpsert" && isPublicSession(event.session) && event.listingId == null) {
|
||||
if (!this.entries.has(event.session.listingId)
|
||||
&& this.entries.size >= this.maximumSessions) {
|
||||
return "overflow";
|
||||
}
|
||||
this.entries.set(event.session.listingId, { session: event.session, updatedAt });
|
||||
this.cursor = event.cursor;
|
||||
return "applied";
|
||||
}
|
||||
if (event.kind === "sessionRemove"
|
||||
&& typeof event.listingId === "string"
|
||||
&& event.listingId.length > 0
|
||||
&& event.session == null) {
|
||||
this.entries.delete(event.listingId);
|
||||
this.cursor = event.cursor;
|
||||
return this.hasMore ? "refresh" : "applied";
|
||||
}
|
||||
if (event.kind === "keepalive" && event.session == null && event.listingId == null) {
|
||||
this.cursor = event.cursor;
|
||||
return "keepalive";
|
||||
}
|
||||
if (event.kind === "reset" && event.session == null && event.listingId == null) {
|
||||
this.cursor = event.cursor;
|
||||
return "reset";
|
||||
}
|
||||
return "invalid";
|
||||
}
|
||||
|
||||
sessions() {
|
||||
return [...this.entries.values()]
|
||||
.sort((left, right) => left.session.listingId.localeCompare(right.session.listingId));
|
||||
}
|
||||
}
|
||||
|
||||
export function safeText(value, maximumLength = 160) {
|
||||
const text = typeof value === "string" ? value : String(value ?? "");
|
||||
const visible = text.replace(/[\u0000-\u001f\u007f]/gu, "�");
|
||||
return visible.length <= maximumLength
|
||||
? visible
|
||||
: `${visible.slice(0, Math.max(0, maximumLength - 1))}…`;
|
||||
}
|
||||
|
||||
export function buildBrowseUrl(filter) {
|
||||
validateFilter(filter);
|
||||
const query = new URLSearchParams({
|
||||
contractVersion: String(CONTRACT_VERSION),
|
||||
gameId: filter.gameId,
|
||||
environmentId: filter.environmentId,
|
||||
protocolVersion: String(filter.protocolVersion),
|
||||
regionId: filter.regionId,
|
||||
pageSize: String(Math.min(BROWSER_PAGE_LIMIT, filter.pageSize ?? BROWSER_PAGE_LIMIT)),
|
||||
excludeFull: String(Boolean(filter.excludeFull)),
|
||||
});
|
||||
return `/v1/sessions?${query}`;
|
||||
}
|
||||
|
||||
export function buildStreamUrl(filter, cursor) {
|
||||
validateFilter(filter);
|
||||
if (!validCursor(cursor)) {
|
||||
throw new TypeError("A bounded stream cursor is required");
|
||||
}
|
||||
const query = new URLSearchParams({
|
||||
contractVersion: String(CONTRACT_VERSION),
|
||||
gameId: filter.gameId,
|
||||
environmentId: filter.environmentId,
|
||||
protocolVersion: String(filter.protocolVersion),
|
||||
regionId: filter.regionId,
|
||||
excludeFull: String(Boolean(filter.excludeFull)),
|
||||
streamCursor: cursor,
|
||||
});
|
||||
return `/v1/sessions/stream?${query}`;
|
||||
}
|
||||
|
||||
export function chooseSnapshotTransport(hasMore, pollingOnly) {
|
||||
if (pollingOnly) {
|
||||
return "pollingOnly";
|
||||
}
|
||||
return hasMore ? "boundedPolling" : "stream";
|
||||
}
|
||||
|
||||
function validCursor(value) {
|
||||
return typeof value === "string"
|
||||
&& value.length > 0
|
||||
&& value.length <= 1024
|
||||
&& /^[\x21-\x7e]+$/u.test(value);
|
||||
}
|
||||
|
||||
function isPublicSession(session) {
|
||||
return session != null
|
||||
&& session.contractVersion === CONTRACT_VERSION
|
||||
&& typeof session.listingId === "string"
|
||||
&& session.listingId.length > 0
|
||||
&& typeof session.gameId === "string"
|
||||
&& typeof session.environmentId === "string"
|
||||
&& typeof session.regionId === "string"
|
||||
&& Number.isInteger(session.protocolVersion)
|
||||
&& session.protocolVersion > 0
|
||||
&& typeof session.buildVersion === "string"
|
||||
&& typeof session.displayName === "string"
|
||||
&& session.visibility === "public"
|
||||
&& session.capacity != null
|
||||
&& Number.isInteger(session.capacity.currentPlayers)
|
||||
&& Number.isInteger(session.capacity.maximumPlayers)
|
||||
&& session.capacity.currentPlayers >= 0
|
||||
&& session.capacity.maximumPlayers >= 1
|
||||
&& session.capacity.currentPlayers <= session.capacity.maximumPlayers
|
||||
&& session.metadata != null
|
||||
&& typeof session.metadata === "object"
|
||||
&& !Array.isArray(session.metadata);
|
||||
}
|
||||
|
||||
function validateFilter(filter) {
|
||||
if (!filter
|
||||
|| typeof filter.gameId !== "string"
|
||||
|| typeof filter.environmentId !== "string"
|
||||
|| typeof filter.regionId !== "string"
|
||||
|| !Number.isInteger(filter.protocolVersion)
|
||||
|| filter.protocolVersion <= 0) {
|
||||
throw new TypeError("The selected diagnostic filter is invalid");
|
||||
}
|
||||
}
|
||||
|
||||
function startDashboard() {
|
||||
const elements = {
|
||||
form: document.querySelector("#filters"),
|
||||
game: document.querySelector("#game-filter"),
|
||||
environment: document.querySelector("#environment-filter"),
|
||||
protocol: document.querySelector("#protocol-filter"),
|
||||
region: document.querySelector("#region-filter"),
|
||||
capacity: document.querySelector("#capacity-filter"),
|
||||
error: document.querySelector("#filter-error"),
|
||||
reconnect: document.querySelector("#reconnect-button"),
|
||||
reset: document.querySelector("#reset-button"),
|
||||
poll: document.querySelector("#poll-button"),
|
||||
streamStatus: document.querySelector("#stream-status"),
|
||||
transportState: document.querySelector("#transport-state"),
|
||||
projectionState: document.querySelector("#projection-state"),
|
||||
lastUpdate: document.querySelector("#last-update"),
|
||||
sessionCount: document.querySelector("#session-count"),
|
||||
results: document.querySelector(".results-panel"),
|
||||
resultsSummary: document.querySelector("#results-summary"),
|
||||
list: document.querySelector("#session-list"),
|
||||
empty: document.querySelector("#empty-state"),
|
||||
};
|
||||
|
||||
const state = {
|
||||
configuration: null,
|
||||
projection: null,
|
||||
activeFilter: null,
|
||||
source: null,
|
||||
pollingTimer: null,
|
||||
requestController: null,
|
||||
streamFailures: 0,
|
||||
pollingOnly: false,
|
||||
renderPending: false,
|
||||
lastSuccess: 0,
|
||||
};
|
||||
|
||||
function setStatus(kind, message, transport = message) {
|
||||
document.body.dataset.streamState = kind;
|
||||
elements.streamStatus.textContent = message;
|
||||
elements.transportState.textContent = transport;
|
||||
}
|
||||
|
||||
function setFormError(message = "") {
|
||||
elements.error.textContent = message;
|
||||
elements.error.hidden = message.length === 0;
|
||||
}
|
||||
|
||||
function populate(select, values, previous) {
|
||||
select.replaceChildren();
|
||||
for (const value of values) {
|
||||
const option = document.createElement("option");
|
||||
option.value = String(value);
|
||||
option.textContent = safeText(value, 80);
|
||||
select.append(option);
|
||||
}
|
||||
if (values.some((value) => String(value) === previous)) {
|
||||
select.value = previous;
|
||||
}
|
||||
}
|
||||
|
||||
function selectedScope() {
|
||||
return state.configuration?.scopes.find((scope) =>
|
||||
scope.gameId === elements.game.value
|
||||
&& scope.environmentId === elements.environment.value) ?? null;
|
||||
}
|
||||
|
||||
function updateEnvironmentOptions() {
|
||||
const prior = elements.environment.value;
|
||||
const environments = state.configuration.scopes
|
||||
.filter((scope) => scope.gameId === elements.game.value)
|
||||
.map((scope) => scope.environmentId);
|
||||
populate(elements.environment, environments, prior);
|
||||
updateScopeOptions();
|
||||
}
|
||||
|
||||
function updateScopeOptions() {
|
||||
const scope = selectedScope();
|
||||
populate(elements.protocol, scope?.protocolVersions ?? [], elements.protocol.value);
|
||||
populate(elements.region, scope?.regions ?? [], elements.region.value);
|
||||
}
|
||||
|
||||
function currentFilter() {
|
||||
const scope = selectedScope();
|
||||
const protocolVersion = Number(elements.protocol.value);
|
||||
if (!scope
|
||||
|| !scope.protocolVersions.includes(protocolVersion)
|
||||
|| !scope.regions.includes(elements.region.value)) {
|
||||
throw new TypeError("Choose one of the configured game, environment, protocol, and region combinations.");
|
||||
}
|
||||
return {
|
||||
gameId: scope.gameId,
|
||||
environmentId: scope.environmentId,
|
||||
protocolVersion,
|
||||
regionId: elements.region.value,
|
||||
excludeFull: elements.capacity.value === "open",
|
||||
pageSize: Math.min(BROWSER_PAGE_LIMIT, state.configuration.maximumRenderedSessions),
|
||||
};
|
||||
}
|
||||
|
||||
function stopLiveWork() {
|
||||
if (state.source) {
|
||||
state.source.close();
|
||||
state.source = null;
|
||||
}
|
||||
if (state.pollingTimer) {
|
||||
clearTimeout(state.pollingTimer);
|
||||
state.pollingTimer = null;
|
||||
}
|
||||
if (state.requestController) {
|
||||
state.requestController.abort();
|
||||
state.requestController = null;
|
||||
}
|
||||
}
|
||||
|
||||
function markSuccess(message) {
|
||||
state.lastSuccess = Date.now();
|
||||
elements.lastUpdate.dateTime = new Date(state.lastSuccess).toISOString();
|
||||
elements.lastUpdate.textContent = "Just now";
|
||||
elements.resultsSummary.textContent = message;
|
||||
}
|
||||
|
||||
function scheduleRender() {
|
||||
if (state.renderPending) {
|
||||
return;
|
||||
}
|
||||
state.renderPending = true;
|
||||
requestAnimationFrame(() => {
|
||||
state.renderPending = false;
|
||||
renderProjection();
|
||||
});
|
||||
}
|
||||
|
||||
function appendDetail(list, term, description) {
|
||||
const dt = document.createElement("dt");
|
||||
dt.textContent = term;
|
||||
const dd = document.createElement("dd");
|
||||
dd.textContent = safeText(description, 160);
|
||||
list.append(dt, dd);
|
||||
}
|
||||
|
||||
function sessionCard(entry) {
|
||||
const session = entry.session;
|
||||
const article = document.createElement("article");
|
||||
article.className = "session-card";
|
||||
article.setAttribute("role", "listitem");
|
||||
article.dataset.updatedAt = String(entry.updatedAt);
|
||||
|
||||
const heading = document.createElement("div");
|
||||
heading.className = "card-heading";
|
||||
const title = document.createElement("h3");
|
||||
title.textContent = safeText(session.displayName, 120);
|
||||
const region = document.createElement("span");
|
||||
region.className = "region-badge";
|
||||
region.textContent = safeText(session.regionId, 48);
|
||||
heading.append(title, region);
|
||||
|
||||
const details = document.createElement("dl");
|
||||
details.className = "session-detail";
|
||||
appendDetail(details, "Build", session.buildVersion);
|
||||
appendDetail(details, "Protocol", session.protocolVersion);
|
||||
appendDetail(details, "Visibility", "Public");
|
||||
appendDetail(details, "Presence", "Recently verified");
|
||||
|
||||
const capacity = document.createElement("div");
|
||||
capacity.className = "capacity-row";
|
||||
const capacityCopy = document.createElement("div");
|
||||
capacityCopy.className = "capacity-copy";
|
||||
const capacityLabel = document.createElement("span");
|
||||
capacityLabel.textContent = "Advisory capacity";
|
||||
const capacityValue = document.createElement("span");
|
||||
capacityValue.textContent = `${session.capacity.currentPlayers} / ${session.capacity.maximumPlayers}`;
|
||||
capacityCopy.append(capacityLabel, capacityValue);
|
||||
const meter = document.createElement("meter");
|
||||
meter.min = 0;
|
||||
meter.max = session.capacity.maximumPlayers;
|
||||
meter.value = session.capacity.currentPlayers;
|
||||
meter.setAttribute("aria-label", `Advisory capacity ${capacityValue.textContent}`);
|
||||
capacity.append(capacityCopy, meter);
|
||||
|
||||
const metadata = document.createElement("ul");
|
||||
metadata.className = "metadata-list";
|
||||
for (const [key, value] of Object.entries(session.metadata).slice(0, 16)) {
|
||||
const item = document.createElement("li");
|
||||
item.textContent = `${safeText(key, 48)}: ${safeText(value, 96)}`;
|
||||
metadata.append(item);
|
||||
}
|
||||
if (metadata.childElementCount === 0) {
|
||||
const item = document.createElement("li");
|
||||
item.textContent = "No public metadata";
|
||||
metadata.append(item);
|
||||
}
|
||||
|
||||
const updated = document.createElement("p");
|
||||
updated.className = "updated-age";
|
||||
updated.textContent = "Updated just now";
|
||||
article.append(heading, details, capacity, metadata, updated);
|
||||
return article;
|
||||
}
|
||||
|
||||
function renderProjection() {
|
||||
const entries = state.projection?.sessions() ?? [];
|
||||
const fragment = document.createDocumentFragment();
|
||||
for (const entry of entries) {
|
||||
fragment.append(sessionCard(entry));
|
||||
}
|
||||
elements.list.replaceChildren(fragment);
|
||||
elements.sessionCount.textContent = String(entries.length);
|
||||
elements.empty.hidden = entries.length !== 0;
|
||||
elements.projectionState.textContent = state.projection?.hasMore
|
||||
? "Bounded snapshot; polling"
|
||||
: "Snapshot plus ordered deltas";
|
||||
elements.results.setAttribute("aria-busy", "false");
|
||||
}
|
||||
|
||||
function updateAges() {
|
||||
const now = Date.now();
|
||||
if (state.lastSuccess > 0) {
|
||||
const age = Math.max(0, Math.floor((now - state.lastSuccess) / 1000));
|
||||
elements.lastUpdate.textContent = age < 5 ? "Just now" : `${age} seconds ago`;
|
||||
}
|
||||
for (const card of elements.list.querySelectorAll(".session-card")) {
|
||||
const age = Math.max(0, Math.floor((now - Number(card.dataset.updatedAt)) / 1000));
|
||||
const copy = card.querySelector(".updated-age");
|
||||
copy.textContent = age < 5 ? "Updated just now" : `Updated ${age} seconds ago`;
|
||||
}
|
||||
}
|
||||
|
||||
async function readJson(url) {
|
||||
const controller = new AbortController();
|
||||
state.requestController = controller;
|
||||
const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
cache: "no-store",
|
||||
credentials: "same-origin",
|
||||
headers: { Accept: "application/json" },
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`The service returned HTTP ${response.status}.`);
|
||||
}
|
||||
return await response.json();
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
if (state.requestController === controller) {
|
||||
state.requestController = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function schedulePolling() {
|
||||
if (state.pollingTimer) {
|
||||
clearTimeout(state.pollingTimer);
|
||||
}
|
||||
state.pollingTimer = setTimeout(
|
||||
() => fetchSnapshot("poll").catch(showServiceError),
|
||||
state.configuration.pollIntervalSeconds * 1000,
|
||||
);
|
||||
}
|
||||
|
||||
function usePolling(message = "Polling fallback active") {
|
||||
if (state.source) {
|
||||
state.source.close();
|
||||
state.source = null;
|
||||
}
|
||||
setStatus("polling", message, "Polling fallback");
|
||||
schedulePolling();
|
||||
}
|
||||
|
||||
function showServiceError(error) {
|
||||
const message = error?.name === "AbortError"
|
||||
? "The service did not answer within the request deadline."
|
||||
: safeText(error?.message || "The service is unavailable.", 200);
|
||||
setStatus("error", "Service unavailable", "Unavailable");
|
||||
elements.resultsSummary.textContent = `${message} Retrying with bounded polling.`;
|
||||
elements.results.setAttribute("aria-busy", "false");
|
||||
usePolling("Service unavailable; polling retry scheduled");
|
||||
}
|
||||
|
||||
async function fetchSnapshot(reason = "manual") {
|
||||
if (!state.activeFilter) {
|
||||
return;
|
||||
}
|
||||
if (state.requestController) {
|
||||
state.requestController.abort();
|
||||
}
|
||||
elements.results.setAttribute("aria-busy", "true");
|
||||
if (reason !== "poll") {
|
||||
setStatus("reconnecting", "Loading a fresh snapshot", "Snapshot request");
|
||||
}
|
||||
const response = await readJson(buildBrowseUrl(state.activeFilter));
|
||||
if (response.contractVersion !== CONTRACT_VERSION
|
||||
|| !Array.isArray(response.items)
|
||||
|| !validCursor(response.streamCursor)) {
|
||||
throw new Error("The service returned an invalid browser snapshot.");
|
||||
}
|
||||
const outcome = state.projection.replace(
|
||||
response.items,
|
||||
response.streamCursor,
|
||||
Boolean(response.nextCursor),
|
||||
);
|
||||
if (outcome !== "applied") {
|
||||
throw new Error("The bounded browser snapshot could not be applied safely.");
|
||||
}
|
||||
scheduleRender();
|
||||
markSuccess(`${response.items.length} public session${response.items.length === 1 ? "" : "s"} in the fresh snapshot.`);
|
||||
const transport = chooseSnapshotTransport(Boolean(response.nextCursor), state.pollingOnly);
|
||||
if (transport !== "stream") {
|
||||
if (transport === "pollingOnly") {
|
||||
usePolling("Polling only selected");
|
||||
return;
|
||||
}
|
||||
usePolling("More sessions exist than this bounded view; polling keeps it authoritative");
|
||||
return;
|
||||
}
|
||||
connectStream(response.streamCursor);
|
||||
}
|
||||
|
||||
function handleStreamEvent(serialized, expectedKind) {
|
||||
let event;
|
||||
try {
|
||||
event = JSON.parse(serialized);
|
||||
} catch {
|
||||
usePolling("Invalid stream data; polling fallback active");
|
||||
return;
|
||||
}
|
||||
if (event.kind !== expectedKind) {
|
||||
usePolling("Unexpected stream event; polling fallback active");
|
||||
return;
|
||||
}
|
||||
const outcome = state.projection.apply(event);
|
||||
if (outcome === "invalid" || outcome === "overflow" || outcome === "refresh") {
|
||||
fetchSnapshot("stream-recovery").catch(showServiceError);
|
||||
return;
|
||||
}
|
||||
if (outcome === "reset") {
|
||||
setStatus("reset", "Cursor reset; refreshing snapshot", "Cursor reset");
|
||||
fetchSnapshot("reset").catch(showServiceError);
|
||||
return;
|
||||
}
|
||||
if (outcome === "keepalive") {
|
||||
markSuccess("Live connection healthy; no listing changes.");
|
||||
return;
|
||||
}
|
||||
state.streamFailures = 0;
|
||||
scheduleRender();
|
||||
markSuccess(expectedKind === "sessionUpsert"
|
||||
? "Applied a live session add or update."
|
||||
: "Applied a live session removal.");
|
||||
}
|
||||
|
||||
function connectStream(cursor, corrupt = false) {
|
||||
if (state.pollingTimer) {
|
||||
clearTimeout(state.pollingTimer);
|
||||
state.pollingTimer = null;
|
||||
}
|
||||
if (state.source) {
|
||||
state.source.close();
|
||||
}
|
||||
const selectedCursor = corrupt
|
||||
? `${cursor.slice(0, -1)}${cursor.endsWith("a") ? "b" : "a"}`
|
||||
: cursor;
|
||||
setStatus(corrupt ? "reset" : "reconnecting",
|
||||
corrupt ? "Testing cursor reset" : "Connecting live updates",
|
||||
corrupt ? "Cursor reset test" : "SSE reconnecting");
|
||||
const source = new EventSource(buildStreamUrl(state.activeFilter, selectedCursor));
|
||||
state.source = source;
|
||||
source.addEventListener("open", () => {
|
||||
if (state.source !== source) {
|
||||
return;
|
||||
}
|
||||
state.streamFailures = 0;
|
||||
setStatus("connected", "Live updates connected", "SSE live");
|
||||
});
|
||||
for (const [name, kind] of [
|
||||
["session_upsert", "sessionUpsert"],
|
||||
["session_remove", "sessionRemove"],
|
||||
["reset", "reset"],
|
||||
["keepalive", "keepalive"],
|
||||
]) {
|
||||
source.addEventListener(name, (message) => {
|
||||
if (state.source === source) {
|
||||
handleStreamEvent(message.data, kind);
|
||||
}
|
||||
});
|
||||
}
|
||||
source.addEventListener("error", () => {
|
||||
if (state.source !== source) {
|
||||
return;
|
||||
}
|
||||
state.streamFailures += 1;
|
||||
if (state.streamFailures >= STREAM_FAILURE_LIMIT || source.readyState === EventSource.CLOSED) {
|
||||
usePolling("Live stream unavailable; polling fallback active");
|
||||
} else {
|
||||
setStatus("reconnecting", "Live stream interrupted; reconnecting", "SSE reconnecting");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function loadConfiguration() {
|
||||
const configuration = await readJson("/diagnostics/config.json");
|
||||
if (configuration.contractVersion !== CONTRACT_VERSION
|
||||
|| !Array.isArray(configuration.scopes)
|
||||
|| configuration.scopes.length < 1
|
||||
|| !Number.isInteger(configuration.pollIntervalSeconds)
|
||||
|| !Number.isInteger(configuration.maximumRenderedSessions)) {
|
||||
throw new Error("The diagnostic configuration is invalid.");
|
||||
}
|
||||
state.configuration = configuration;
|
||||
state.projection = new SessionProjection(configuration.maximumRenderedSessions);
|
||||
populate(elements.game, [...new Set(configuration.scopes.map((scope) => scope.gameId))], "");
|
||||
updateEnvironmentOptions();
|
||||
state.activeFilter = currentFilter();
|
||||
await fetchSnapshot("startup");
|
||||
}
|
||||
|
||||
elements.game.addEventListener("change", updateEnvironmentOptions);
|
||||
elements.environment.addEventListener("change", updateScopeOptions);
|
||||
elements.form.addEventListener("submit", (event) => {
|
||||
event.preventDefault();
|
||||
try {
|
||||
setFormError();
|
||||
stopLiveWork();
|
||||
state.pollingOnly = false;
|
||||
state.activeFilter = currentFilter();
|
||||
state.projection = new SessionProjection(state.configuration.maximumRenderedSessions);
|
||||
fetchSnapshot("filter").catch(showServiceError);
|
||||
} catch (error) {
|
||||
setFormError(safeText(error.message, 200));
|
||||
}
|
||||
});
|
||||
elements.reconnect.addEventListener("click", () => {
|
||||
if (state.projection?.cursor) {
|
||||
state.pollingOnly = false;
|
||||
connectStream(state.projection.cursor);
|
||||
}
|
||||
});
|
||||
elements.reset.addEventListener("click", () => {
|
||||
if (state.projection?.cursor) {
|
||||
state.pollingOnly = false;
|
||||
connectStream(state.projection.cursor, true);
|
||||
}
|
||||
});
|
||||
elements.poll.addEventListener("click", () => {
|
||||
state.pollingOnly = true;
|
||||
usePolling("Polling only selected deliberately");
|
||||
fetchSnapshot("poll").catch(showServiceError);
|
||||
});
|
||||
window.addEventListener("pagehide", stopLiveWork, { once: true });
|
||||
setInterval(updateAges, 5000);
|
||||
loadConfiguration().catch(showServiceError);
|
||||
}
|
||||
|
||||
if (typeof document !== "undefined") {
|
||||
startDashboard();
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="color-scheme" content="dark">
|
||||
<title>Session diagnostics — Rendezvous</title>
|
||||
<link rel="stylesheet" href="/diagnostics/styles.css">
|
||||
<script type="module" src="/diagnostics/app.mjs"></script>
|
||||
</head>
|
||||
<body>
|
||||
<a class="skip-link" href="#main-content">Skip to session results</a>
|
||||
<header class="site-header">
|
||||
<div>
|
||||
<p class="eyebrow">Final Factory infrastructure</p>
|
||||
<h1>Rendezvous session diagnostics</h1>
|
||||
<p class="lede">A read-only view of public session discovery. Capacity is advisory; joining always revalidates current state.</p>
|
||||
</div>
|
||||
<div class="connection-state" aria-live="polite" aria-atomic="true">
|
||||
<span class="status-dot" aria-hidden="true"></span>
|
||||
<span id="stream-status">Loading configuration</span>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main id="main-content" tabindex="-1">
|
||||
<section class="filter-panel" aria-labelledby="filter-heading">
|
||||
<div class="section-heading">
|
||||
<div>
|
||||
<p class="eyebrow">Public browser scope</p>
|
||||
<h2 id="filter-heading">Choose a configured session view</h2>
|
||||
</div>
|
||||
<p class="boundary-note">This page has no join, publish, or operator authority.</p>
|
||||
</div>
|
||||
|
||||
<form id="filters" novalidate>
|
||||
<div class="filter-grid">
|
||||
<label>
|
||||
<span>Game</span>
|
||||
<select id="game-filter" name="game" required></select>
|
||||
</label>
|
||||
<label>
|
||||
<span>Environment</span>
|
||||
<select id="environment-filter" name="environment" required></select>
|
||||
</label>
|
||||
<label>
|
||||
<span>Protocol</span>
|
||||
<select id="protocol-filter" name="protocol" required></select>
|
||||
</label>
|
||||
<label>
|
||||
<span>Region</span>
|
||||
<select id="region-filter" name="region" required></select>
|
||||
</label>
|
||||
<label>
|
||||
<span>Availability</span>
|
||||
<select id="capacity-filter" name="capacity">
|
||||
<option value="open">Open slots only</option>
|
||||
<option value="all">Include full sessions</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
<span>Visibility</span>
|
||||
<select id="visibility-filter" name="visibility" disabled>
|
||||
<option value="public">Public only</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
<p id="filter-error" class="form-error" role="alert" hidden></p>
|
||||
<div class="button-row">
|
||||
<button class="button primary" type="submit">Apply filters</button>
|
||||
<button class="button" id="reconnect-button" type="button">Reconnect now</button>
|
||||
<button class="button" id="reset-button" type="button">Test reset recovery</button>
|
||||
<button class="button" id="poll-button" type="button">Use polling only</button>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<section class="summary-panel" aria-labelledby="summary-heading">
|
||||
<h2 id="summary-heading" class="visually-hidden">Current diagnostic state</h2>
|
||||
<dl class="summary-grid">
|
||||
<div>
|
||||
<dt>Sessions shown</dt>
|
||||
<dd id="session-count">0</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Transport</dt>
|
||||
<dd id="transport-state">Starting</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Last successful update</dt>
|
||||
<dd><time id="last-update">Not yet</time></dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Projection</dt>
|
||||
<dd id="projection-state">Waiting for snapshot</dd>
|
||||
</div>
|
||||
</dl>
|
||||
</section>
|
||||
|
||||
<section class="results-panel" aria-labelledby="results-heading" aria-busy="true">
|
||||
<div class="section-heading results-heading">
|
||||
<div>
|
||||
<p class="eyebrow">Bounded public projection</p>
|
||||
<h2 id="results-heading">Available sessions</h2>
|
||||
</div>
|
||||
<p id="results-summary" role="status" aria-live="polite">Loading a fresh snapshot…</p>
|
||||
</div>
|
||||
<div id="session-list" class="session-grid" role="list"></div>
|
||||
<div id="empty-state" class="empty-state" hidden>
|
||||
<h3>No compatible public sessions</h3>
|
||||
<p>The service answered successfully, but this configured filter currently has no listings.</p>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
<p>Read-only diagnostics · no credentials stored · no gameplay or administration</p>
|
||||
</footer>
|
||||
<noscript>This diagnostic requires JavaScript to consume the public snapshot and event stream.</noscript>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,505 @@
|
||||
:root {
|
||||
color-scheme: dark;
|
||||
--bg: #071019;
|
||||
--surface: #101c28;
|
||||
--surface-raised: #172737;
|
||||
--border: #375066;
|
||||
--text: #f2f7fb;
|
||||
--muted: #b6c8d6;
|
||||
--accent: #55d7b5;
|
||||
--accent-strong: #7ce9cc;
|
||||
--accent-ink: #04231c;
|
||||
--warning: #ffd479;
|
||||
--danger: #ff9f9f;
|
||||
--focus: #ffd479;
|
||||
--radius: 0.75rem;
|
||||
--space-1: 0.5rem;
|
||||
--space-2: 0.75rem;
|
||||
--space-3: 1rem;
|
||||
--space-4: 1.5rem;
|
||||
--space-5: 2rem;
|
||||
--space-6: 3rem;
|
||||
font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
||||
font-size: 100%;
|
||||
line-height: 1.55;
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
html {
|
||||
min-width: 20rem;
|
||||
background: var(--bg);
|
||||
}
|
||||
|
||||
body {
|
||||
min-height: 100vh;
|
||||
margin: 0;
|
||||
color: var(--text);
|
||||
background:
|
||||
radial-gradient(circle at 10% -10%, rgb(36 103 104 / 35%), transparent 32rem),
|
||||
linear-gradient(180deg, #09141f 0%, var(--bg) 50%);
|
||||
}
|
||||
|
||||
button,
|
||||
select {
|
||||
font: inherit;
|
||||
}
|
||||
|
||||
button,
|
||||
select,
|
||||
a {
|
||||
-webkit-tap-highlight-color: transparent;
|
||||
}
|
||||
|
||||
:focus-visible {
|
||||
outline: 0.2rem solid var(--focus);
|
||||
outline-offset: 0.2rem;
|
||||
}
|
||||
|
||||
.skip-link {
|
||||
position: fixed;
|
||||
z-index: 10;
|
||||
top: var(--space-2);
|
||||
left: var(--space-2);
|
||||
padding: var(--space-2) var(--space-3);
|
||||
color: #071019;
|
||||
background: var(--focus);
|
||||
border-radius: 0.4rem;
|
||||
font-weight: 800;
|
||||
transform: translateY(-200%);
|
||||
}
|
||||
|
||||
.skip-link:focus {
|
||||
transform: translateY(0);
|
||||
}
|
||||
|
||||
.site-header,
|
||||
main,
|
||||
footer {
|
||||
width: min(80rem, calc(100% - 2rem));
|
||||
margin-inline: auto;
|
||||
}
|
||||
|
||||
.site-header {
|
||||
display: flex;
|
||||
align-items: end;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-5);
|
||||
padding-block: var(--space-6) var(--space-5);
|
||||
}
|
||||
|
||||
h1,
|
||||
h2,
|
||||
h3,
|
||||
p {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
h1 {
|
||||
max-width: 18ch;
|
||||
margin-bottom: var(--space-2);
|
||||
font-size: clamp(2rem, 6vw, 4.25rem);
|
||||
line-height: 1.02;
|
||||
letter-spacing: -0.045em;
|
||||
}
|
||||
|
||||
h2 {
|
||||
margin-bottom: var(--space-1);
|
||||
font-size: clamp(1.35rem, 3vw, 2rem);
|
||||
line-height: 1.2;
|
||||
}
|
||||
|
||||
h3 {
|
||||
margin-bottom: var(--space-1);
|
||||
font-size: 1.1rem;
|
||||
}
|
||||
|
||||
.eyebrow {
|
||||
margin-bottom: var(--space-1);
|
||||
color: var(--accent-strong);
|
||||
font-size: 0.78rem;
|
||||
font-weight: 800;
|
||||
letter-spacing: 0.14em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.lede {
|
||||
max-width: 65ch;
|
||||
margin-bottom: 0;
|
||||
color: var(--muted);
|
||||
font-size: 1.05rem;
|
||||
}
|
||||
|
||||
.connection-state {
|
||||
display: inline-flex;
|
||||
min-height: 2.75rem;
|
||||
align-items: center;
|
||||
gap: var(--space-2);
|
||||
padding: var(--space-2) var(--space-3);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 999px;
|
||||
background: rgb(16 28 40 / 88%);
|
||||
color: var(--muted);
|
||||
font-weight: 700;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.status-dot {
|
||||
width: 0.7rem;
|
||||
height: 0.7rem;
|
||||
border: 2px solid currentColor;
|
||||
border-radius: 50%;
|
||||
background: currentColor;
|
||||
}
|
||||
|
||||
body[data-stream-state="connected"] .connection-state {
|
||||
color: var(--accent-strong);
|
||||
}
|
||||
|
||||
body[data-stream-state="reconnecting"] .connection-state,
|
||||
body[data-stream-state="polling"] .connection-state,
|
||||
body[data-stream-state="reset"] .connection-state {
|
||||
color: var(--warning);
|
||||
}
|
||||
|
||||
body[data-stream-state="error"] .connection-state {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
main {
|
||||
display: grid;
|
||||
gap: var(--space-4);
|
||||
}
|
||||
|
||||
.filter-panel,
|
||||
.summary-panel,
|
||||
.results-panel {
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
background: rgb(16 28 40 / 94%);
|
||||
box-shadow: 0 1rem 3rem rgb(0 0 0 / 18%);
|
||||
}
|
||||
|
||||
.filter-panel,
|
||||
.results-panel {
|
||||
padding: clamp(1rem, 4vw, 2rem);
|
||||
}
|
||||
|
||||
.section-heading {
|
||||
display: flex;
|
||||
align-items: start;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-4);
|
||||
margin-bottom: var(--space-4);
|
||||
}
|
||||
|
||||
.boundary-note,
|
||||
#results-summary {
|
||||
max-width: 34rem;
|
||||
margin-bottom: 0;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.filter-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
gap: var(--space-3);
|
||||
}
|
||||
|
||||
label {
|
||||
display: grid;
|
||||
gap: 0.35rem;
|
||||
color: var(--muted);
|
||||
font-size: 0.9rem;
|
||||
font-weight: 750;
|
||||
}
|
||||
|
||||
select {
|
||||
width: 100%;
|
||||
min-height: 2.75rem;
|
||||
padding: 0.6rem 2.4rem 0.6rem 0.75rem;
|
||||
border: 1px solid #587189;
|
||||
border-radius: 0.45rem;
|
||||
color: var(--text);
|
||||
background: #0b1722;
|
||||
}
|
||||
|
||||
select:disabled {
|
||||
color: #9fb0bd;
|
||||
border-style: dashed;
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.button-row {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: var(--space-2);
|
||||
margin-top: var(--space-4);
|
||||
}
|
||||
|
||||
.button {
|
||||
min-height: 2.75rem;
|
||||
padding: 0.65rem 1rem;
|
||||
border: 1px solid #6a8298;
|
||||
border-radius: 0.45rem;
|
||||
color: var(--text);
|
||||
background: var(--surface-raised);
|
||||
cursor: pointer;
|
||||
font-weight: 800;
|
||||
}
|
||||
|
||||
.button:hover {
|
||||
border-color: var(--accent-strong);
|
||||
background: #21384a;
|
||||
}
|
||||
|
||||
.button:active {
|
||||
transform: translateY(1px);
|
||||
}
|
||||
|
||||
.button.primary {
|
||||
color: var(--accent-ink);
|
||||
border-color: var(--accent);
|
||||
background: var(--accent);
|
||||
}
|
||||
|
||||
.button.primary:hover {
|
||||
background: var(--accent-strong);
|
||||
}
|
||||
|
||||
.form-error {
|
||||
margin: var(--space-3) 0 0;
|
||||
color: var(--danger);
|
||||
font-weight: 750;
|
||||
}
|
||||
|
||||
.summary-panel {
|
||||
padding: 0;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.summary-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.summary-grid > div {
|
||||
min-width: 0;
|
||||
padding: var(--space-3) var(--space-4);
|
||||
border-right: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.summary-grid > div:last-child {
|
||||
border-right: 0;
|
||||
}
|
||||
|
||||
.summary-grid dt {
|
||||
color: var(--muted);
|
||||
font-size: 0.78rem;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.summary-grid dd {
|
||||
margin: 0.25rem 0 0;
|
||||
overflow-wrap: anywhere;
|
||||
font-size: 1.05rem;
|
||||
font-weight: 800;
|
||||
}
|
||||
|
||||
.results-heading {
|
||||
align-items: end;
|
||||
}
|
||||
|
||||
.session-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(min(100%, 19rem), 1fr));
|
||||
gap: var(--space-3);
|
||||
}
|
||||
|
||||
.session-card {
|
||||
min-width: 0;
|
||||
padding: var(--space-3);
|
||||
border: 1px solid #496177;
|
||||
border-radius: 0.6rem;
|
||||
background: #0b1722;
|
||||
}
|
||||
|
||||
.card-heading {
|
||||
display: flex;
|
||||
align-items: start;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-2);
|
||||
}
|
||||
|
||||
.card-heading h3 {
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.region-badge {
|
||||
flex: 0 0 auto;
|
||||
padding: 0.15rem 0.5rem;
|
||||
border: 1px solid #597187;
|
||||
border-radius: 999px;
|
||||
color: var(--muted);
|
||||
font-size: 0.75rem;
|
||||
font-weight: 750;
|
||||
}
|
||||
|
||||
.session-detail {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(5rem, auto) 1fr;
|
||||
gap: 0.25rem var(--space-2);
|
||||
margin: var(--space-3) 0 0;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.session-detail dt {
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.session-detail dd {
|
||||
min-width: 0;
|
||||
margin: 0;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.capacity-row {
|
||||
margin-top: var(--space-3);
|
||||
}
|
||||
|
||||
.capacity-copy {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-2);
|
||||
margin-bottom: 0.35rem;
|
||||
color: var(--muted);
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
meter {
|
||||
width: 100%;
|
||||
height: 0.65rem;
|
||||
accent-color: var(--accent);
|
||||
}
|
||||
|
||||
.metadata-list {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 0.35rem;
|
||||
margin: var(--space-3) 0 0;
|
||||
padding: 0;
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
.metadata-list li {
|
||||
max-width: 100%;
|
||||
padding: 0.2rem 0.45rem;
|
||||
overflow-wrap: anywhere;
|
||||
border-radius: 0.3rem;
|
||||
color: #d6e4ed;
|
||||
background: #1b2b39;
|
||||
font-size: 0.78rem;
|
||||
}
|
||||
|
||||
.updated-age {
|
||||
margin: var(--space-3) 0 0;
|
||||
color: var(--muted);
|
||||
font-size: 0.78rem;
|
||||
}
|
||||
|
||||
.empty-state {
|
||||
padding: var(--space-6) var(--space-3);
|
||||
text-align: center;
|
||||
border: 1px dashed #587189;
|
||||
border-radius: 0.6rem;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.empty-state h3 {
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
footer {
|
||||
padding-block: var(--space-5);
|
||||
color: var(--muted);
|
||||
font-size: 0.85rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.visually-hidden {
|
||||
position: absolute;
|
||||
width: 1px;
|
||||
height: 1px;
|
||||
padding: 0;
|
||||
overflow: hidden;
|
||||
clip: rect(0, 0, 0, 0);
|
||||
white-space: nowrap;
|
||||
border: 0;
|
||||
}
|
||||
|
||||
[hidden] {
|
||||
display: none !important;
|
||||
}
|
||||
|
||||
@media (max-width: 56rem) {
|
||||
.site-header,
|
||||
.section-heading {
|
||||
align-items: start;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.filter-grid,
|
||||
.summary-grid {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.summary-grid > div:nth-child(2) {
|
||||
border-right: 0;
|
||||
}
|
||||
|
||||
.summary-grid > div:nth-child(-n + 2) {
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 36rem) {
|
||||
.site-header,
|
||||
main,
|
||||
footer {
|
||||
width: min(100% - 1rem, 80rem);
|
||||
}
|
||||
|
||||
.site-header {
|
||||
padding-block: var(--space-5) var(--space-4);
|
||||
}
|
||||
|
||||
.filter-grid,
|
||||
.summary-grid {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.summary-grid > div {
|
||||
border-right: 0;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.summary-grid > div:last-child {
|
||||
border-bottom: 0;
|
||||
}
|
||||
|
||||
.button {
|
||||
width: 100%;
|
||||
}
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
*,
|
||||
*::before,
|
||||
*::after {
|
||||
scroll-behavior: auto !important;
|
||||
transition-duration: 0.01ms !important;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
using System.Reflection;
|
||||
using FinalFactory.Rendezvous.Contracts;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace FinalFactory.Rendezvous.Server.Diagnostics;
|
||||
|
||||
internal static class DiagnosticDashboardEndpoints
|
||||
{
|
||||
private const string ContentSecurityPolicy =
|
||||
"default-src 'none'; base-uri 'none'; connect-src 'self'; "
|
||||
+ "font-src 'self'; form-action 'none'; frame-ancestors 'none'; "
|
||||
+ "img-src 'self'; manifest-src 'none'; object-src 'none'; "
|
||||
+ "script-src 'self'; style-src 'self'";
|
||||
private static readonly byte[] Index = ReadAsset("index.html");
|
||||
private static readonly byte[] Script = ReadAsset("app.mjs");
|
||||
private static readonly byte[] Styles = ReadAsset("styles.css");
|
||||
|
||||
public static IEndpointRouteBuilder MapDiagnosticDashboardEndpoints(
|
||||
this IEndpointRouteBuilder endpoints)
|
||||
{
|
||||
RouteGroupBuilder dashboard = endpoints.MapGroup("/diagnostics")
|
||||
.ExcludeFromDescription();
|
||||
dashboard.MapGet("", ServeIndex);
|
||||
dashboard.MapGet("/app.mjs", ServeScript);
|
||||
dashboard.MapGet("/styles.css", ServeStyles);
|
||||
dashboard.MapGet("/config.json", ServeConfiguration);
|
||||
return endpoints;
|
||||
}
|
||||
|
||||
private static IResult ServeIndex(
|
||||
HttpContext context,
|
||||
[FromServices] IOptions<DiagnosticDashboardOptions> configured) =>
|
||||
ServeAsset(context, configured.Value, Index, "text/html; charset=utf-8");
|
||||
|
||||
private static IResult ServeScript(
|
||||
HttpContext context,
|
||||
[FromServices] IOptions<DiagnosticDashboardOptions> configured) =>
|
||||
ServeAsset(context, configured.Value, Script, "text/javascript; charset=utf-8");
|
||||
|
||||
private static IResult ServeStyles(
|
||||
HttpContext context,
|
||||
[FromServices] IOptions<DiagnosticDashboardOptions> configured) =>
|
||||
ServeAsset(context, configured.Value, Styles, "text/css; charset=utf-8");
|
||||
|
||||
private static IResult ServeConfiguration(
|
||||
HttpContext context,
|
||||
[FromServices] IOptions<DiagnosticDashboardOptions> configured)
|
||||
{
|
||||
DiagnosticDashboardOptions options = configured.Value;
|
||||
if (!options.Enabled)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
ApplySecurityHeaders(context.Response);
|
||||
return Results.Json(new
|
||||
{
|
||||
contractVersion = ContractLimits.ContractVersion,
|
||||
pollIntervalSeconds = options.PollIntervalSeconds,
|
||||
maximumRenderedSessions = options.MaximumRenderedSessions,
|
||||
scopes = options.Scopes.Select(static scope => new
|
||||
{
|
||||
gameId = scope.GameId,
|
||||
environmentId = scope.EnvironmentId,
|
||||
protocolVersions = scope.ProtocolVersions,
|
||||
regions = scope.Regions,
|
||||
visibility = "public",
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
private static IResult ServeAsset(
|
||||
HttpContext context,
|
||||
DiagnosticDashboardOptions options,
|
||||
byte[] content,
|
||||
string contentType)
|
||||
{
|
||||
if (!options.Enabled)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
ApplySecurityHeaders(context.Response);
|
||||
return Results.Bytes(content, contentType);
|
||||
}
|
||||
|
||||
private static void ApplySecurityHeaders(HttpResponse response)
|
||||
{
|
||||
response.Headers.CacheControl = "no-store";
|
||||
response.Headers["Content-Security-Policy"] = ContentSecurityPolicy;
|
||||
response.Headers["X-Content-Type-Options"] = "nosniff";
|
||||
response.Headers["X-Frame-Options"] = "DENY";
|
||||
response.Headers["Cross-Origin-Opener-Policy"] = "same-origin";
|
||||
response.Headers["Cross-Origin-Resource-Policy"] = "same-origin";
|
||||
response.Headers["Permissions-Policy"] =
|
||||
"camera=(), geolocation=(), microphone=(), payment=(), usb=()";
|
||||
response.Headers["Referrer-Policy"] = "no-referrer";
|
||||
}
|
||||
|
||||
private static byte[] ReadAsset(string fileName)
|
||||
{
|
||||
Assembly assembly = typeof(DiagnosticDashboardEndpoints).Assembly;
|
||||
string resourceName = $"FinalFactory.Rendezvous.Server.Diagnostics.Assets.{fileName}";
|
||||
using Stream source = assembly.GetManifestResourceStream(resourceName)
|
||||
?? throw new InvalidOperationException($"Missing embedded dashboard asset {fileName}.");
|
||||
using MemoryStream destination = new();
|
||||
source.CopyTo(destination);
|
||||
return destination.ToArray();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
using FinalFactory.Rendezvous.Contracts;
|
||||
|
||||
namespace FinalFactory.Rendezvous.Server.Diagnostics;
|
||||
|
||||
internal sealed record DiagnosticDashboardOptions
|
||||
{
|
||||
public const string SectionName = "Rendezvous:Diagnostics";
|
||||
|
||||
public bool Enabled { get; init; }
|
||||
public int PollIntervalSeconds { get; init; } = 10;
|
||||
public int MaximumRenderedSessions { get; init; } = 100;
|
||||
public DiagnosticDashboardScope[] Scopes { get; init; } = [];
|
||||
|
||||
public IReadOnlyList<string> Validate()
|
||||
{
|
||||
List<string> errors = [];
|
||||
if (PollIntervalSeconds is < 5 or > 60)
|
||||
{
|
||||
errors.Add($"{SectionName}:PollIntervalSeconds must be between 5 and 60.");
|
||||
}
|
||||
if (MaximumRenderedSessions is < 10 or > 500)
|
||||
{
|
||||
errors.Add($"{SectionName}:MaximumRenderedSessions must be between 10 and 500.");
|
||||
}
|
||||
if (!Enabled)
|
||||
{
|
||||
return errors;
|
||||
}
|
||||
if (Scopes is null || Scopes.Length is < 1 or > 32)
|
||||
{
|
||||
errors.Add($"{SectionName}:Scopes must contain between 1 and 32 allow-listed scopes when enabled.");
|
||||
return errors;
|
||||
}
|
||||
|
||||
HashSet<string> identities = new(StringComparer.Ordinal);
|
||||
foreach (DiagnosticDashboardScope? scope in Scopes)
|
||||
{
|
||||
if (scope is null)
|
||||
{
|
||||
errors.Add($"{SectionName}:Scopes cannot contain null entries.");
|
||||
continue;
|
||||
}
|
||||
string gameId = scope.GameId ?? string.Empty;
|
||||
string environmentId = scope.EnvironmentId ?? string.Empty;
|
||||
uint[] protocolVersions = scope.ProtocolVersions ?? [];
|
||||
string[] regions = scope.Regions ?? [];
|
||||
if (!GameId.TryParse(gameId, out _)
|
||||
|| !EnvironmentId.TryParse(environmentId, out _))
|
||||
{
|
||||
errors.Add($"{SectionName}:Scopes contains an invalid game or environment identifier.");
|
||||
}
|
||||
if (protocolVersions.Length is < 1 or > 16
|
||||
|| protocolVersions.Any(static version => version == 0)
|
||||
|| protocolVersions.Distinct().Count() != protocolVersions.Length)
|
||||
{
|
||||
errors.Add($"{SectionName}:Scopes protocol versions must contain 1-16 unique positive values.");
|
||||
}
|
||||
if (regions.Length is < 1 or > 16
|
||||
|| regions.Any(static region => !RegionId.TryParse(region ?? string.Empty, out _))
|
||||
|| regions.Distinct(StringComparer.Ordinal).Count() != regions.Length)
|
||||
{
|
||||
errors.Add($"{SectionName}:Scopes regions must contain 1-16 unique valid identifiers.");
|
||||
}
|
||||
|
||||
string identity = $"{gameId}\n{environmentId}";
|
||||
if (!identities.Add(identity))
|
||||
{
|
||||
errors.Add($"{SectionName}:Scopes contains a duplicate game/environment pair.");
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record DiagnosticDashboardScope
|
||||
{
|
||||
public string GameId { get; init; } = string.Empty;
|
||||
public string EnvironmentId { get; init; } = string.Empty;
|
||||
public uint[] ProtocolVersions { get; init; } = [];
|
||||
public string[] Regions { get; init; } = [];
|
||||
}
|
||||
@@ -53,6 +53,8 @@
|
||||
<Compile Include="ConnectionOutcomes/ConnectionOutcomeService.cs" />
|
||||
<Compile Include="Deployment/DeploymentOptions.cs" />
|
||||
<Compile Include="Deployment/GracefulDrainService.cs" />
|
||||
<Compile Include="Diagnostics/DiagnosticDashboardEndpoints.cs" />
|
||||
<Compile Include="Diagnostics/DiagnosticDashboardOptions.cs" />
|
||||
<Compile Include="Http/ContractEndpoints.cs" />
|
||||
<Compile Include="Http/RendezvousExceptionHandler.cs" />
|
||||
<Compile Include="JoinAttempts/JoinAttemptCursorCodec.cs" />
|
||||
@@ -60,6 +62,7 @@
|
||||
<Compile Include="Observability/AuditOptions.cs" />
|
||||
<Compile Include="Observability/AuditTrail.cs" />
|
||||
<Compile Include="Observability/HealthEndpoints.cs" />
|
||||
<Compile Include="Observability/PrometheusMetricsEndpoint.cs" />
|
||||
<Compile Include="Observability/RendezvousReadiness.cs" />
|
||||
<Compile Include="Observability/RendezvousTelemetry.cs" />
|
||||
<Compile Include="Observability/TelemetryMiddleware.cs" />
|
||||
@@ -88,4 +91,15 @@
|
||||
<Compile Include="Transport/UdpMediatorOptions.cs" />
|
||||
<Compile Include="Transport/UdpMediatorService.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<EmbeddedResource Include="Diagnostics/Assets/app.mjs">
|
||||
<LogicalName>FinalFactory.Rendezvous.Server.Diagnostics.Assets.app.mjs</LogicalName>
|
||||
</EmbeddedResource>
|
||||
<EmbeddedResource Include="Diagnostics/Assets/index.html">
|
||||
<LogicalName>FinalFactory.Rendezvous.Server.Diagnostics.Assets.index.html</LogicalName>
|
||||
</EmbeddedResource>
|
||||
<EmbeddedResource Include="Diagnostics/Assets/styles.css">
|
||||
<LogicalName>FinalFactory.Rendezvous.Server.Diagnostics.Assets.styles.css</LogicalName>
|
||||
</EmbeddedResource>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using FinalFactory.Rendezvous.Server.Provisioning;
|
||||
|
||||
namespace FinalFactory.Rendezvous.Server.Observability;
|
||||
|
||||
internal sealed record PrometheusMetricsOptions
|
||||
{
|
||||
public const string SectionName = "Rendezvous:Metrics";
|
||||
|
||||
public bool Enabled { get; init; }
|
||||
public string BearerTokenSecretReference { get; init; } = string.Empty;
|
||||
|
||||
public IReadOnlyList<string> Validate()
|
||||
{
|
||||
if (!Enabled)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
string reference = BearerTokenSecretReference ?? string.Empty;
|
||||
bool environmentReference = reference.StartsWith("env:", StringComparison.Ordinal)
|
||||
&& reference.Length > "env:".Length;
|
||||
bool absoluteFileReference = reference.StartsWith("file:", StringComparison.Ordinal)
|
||||
&& Path.IsPathFullyQualified(reference["file:".Length..]);
|
||||
return reference.Length is < 5 or > 512
|
||||
|| !(environmentReference || absoluteFileReference)
|
||||
? [$"{SectionName}:BearerTokenSecretReference must be a bounded env: or absolute file: secret reference when metrics are enabled."]
|
||||
: [];
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class MetricsAccessCredential : IDisposable
|
||||
{
|
||||
private byte[]? _token;
|
||||
|
||||
private MetricsAccessCredential(byte[] token) => _token = token;
|
||||
|
||||
public static bool TryCreate(
|
||||
PrometheusMetricsOptions options,
|
||||
ISecretProvider secrets,
|
||||
out MetricsAccessCredential? credential)
|
||||
{
|
||||
credential = null;
|
||||
if (!options.Enabled
|
||||
|| !secrets.TryGetSecret(options.BearerTokenSecretReference, out SecretMaterial? material)
|
||||
|| material is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
using (material)
|
||||
{
|
||||
byte[] bytes = material.CopyBytes();
|
||||
int length = bytes.Length;
|
||||
while (length > 0 && bytes[length - 1] is (byte)'\r' or (byte)'\n')
|
||||
{
|
||||
length--;
|
||||
}
|
||||
bool valid = length is >= 32 and <= 128
|
||||
&& bytes.AsSpan(0, length).IndexOfAnyExceptInRange((byte)0x21, (byte)0x7e) < 0;
|
||||
if (!valid)
|
||||
{
|
||||
CryptographicOperations.ZeroMemory(bytes);
|
||||
return false;
|
||||
}
|
||||
|
||||
byte[] token = bytes.AsSpan(0, length).ToArray();
|
||||
CryptographicOperations.ZeroMemory(bytes);
|
||||
credential = new(token);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public bool Authorizes(HttpRequest request)
|
||||
{
|
||||
byte[]? expected = _token;
|
||||
string authorization = request.Headers.Authorization.ToString();
|
||||
const string prefix = "Bearer ";
|
||||
if (expected is null
|
||||
|| !authorization.StartsWith(prefix, StringComparison.Ordinal)
|
||||
|| authorization.Length - prefix.Length is < 32 or > 128)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
byte[] supplied = Encoding.UTF8.GetBytes(authorization[prefix.Length..]);
|
||||
try
|
||||
{
|
||||
return supplied.Length == expected.Length
|
||||
&& CryptographicOperations.FixedTimeEquals(supplied, expected);
|
||||
}
|
||||
finally
|
||||
{
|
||||
CryptographicOperations.ZeroMemory(supplied);
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
byte[]? token = Interlocked.Exchange(ref _token, null);
|
||||
if (token is not null)
|
||||
{
|
||||
CryptographicOperations.ZeroMemory(token);
|
||||
}
|
||||
}
|
||||
|
||||
public override string ToString() => "[MetricsAccessCredential: REDACTED]";
|
||||
}
|
||||
|
||||
internal static class PrometheusMetricsEndpoint
|
||||
{
|
||||
public static IEndpointRouteBuilder MapPrometheusMetricsEndpoint(
|
||||
this IEndpointRouteBuilder endpoints,
|
||||
PrometheusMetricsOptions options,
|
||||
MetricsAccessCredential? credential)
|
||||
{
|
||||
endpoints.MapGet("/metrics", (HttpContext context, RendezvousTelemetry telemetry) =>
|
||||
Export(context, telemetry, options, credential))
|
||||
.WithName("MetricsScrape")
|
||||
.ExcludeFromDescription();
|
||||
return endpoints;
|
||||
}
|
||||
|
||||
private static IResult Export(
|
||||
HttpContext context,
|
||||
RendezvousTelemetry telemetry,
|
||||
PrometheusMetricsOptions options,
|
||||
MetricsAccessCredential? credential)
|
||||
{
|
||||
if (!options.Enabled)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
if (credential is null || !credential.Authorizes(context.Request))
|
||||
{
|
||||
telemetry.RecordMetricsScrape("rejected");
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
telemetry.RecordMetricsScrape("accepted");
|
||||
context.Response.Headers.CacheControl = "no-store";
|
||||
context.Response.Headers["X-Content-Type-Options"] = "nosniff";
|
||||
return Results.Text(
|
||||
telemetry.RenderPrometheus(),
|
||||
"text/plain; version=0.0.4; charset=utf-8",
|
||||
Encoding.UTF8);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,10 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using FinalFactory.Rendezvous.Server.Browser;
|
||||
using FinalFactory.Rendezvous.Server.Provisioning;
|
||||
using FinalFactory.Rendezvous.Server.State;
|
||||
|
||||
namespace FinalFactory.Rendezvous.Server.Observability;
|
||||
@@ -10,6 +15,10 @@ internal sealed class RendezvousTelemetry : IDisposable
|
||||
public const string ActivitySourceName = "FinalFactory.Rendezvous.Server";
|
||||
|
||||
private readonly InMemoryEphemeralRendezvousStore _store;
|
||||
private readonly SessionChangeJournal? _sessionChanges;
|
||||
private readonly ProvisioningRuntime? _provisioning;
|
||||
private readonly ConcurrentDictionary<MetricSeriesKey, long> _prometheusCounters = new();
|
||||
private readonly ConcurrentDictionary<MetricSeriesKey, PrometheusHistogram> _prometheusHistograms = new();
|
||||
private readonly Meter _meter = new(MeterName, "1.0.0");
|
||||
private readonly ActivitySource _activities = new(ActivitySourceName, "1.0.0");
|
||||
private readonly Counter<long> _httpRequests;
|
||||
@@ -22,9 +31,14 @@ internal sealed class RendezvousTelemetry : IDisposable
|
||||
private readonly Counter<long> _operatorAuthentication;
|
||||
private readonly Histogram<double> _pairingLatency;
|
||||
|
||||
public RendezvousTelemetry(InMemoryEphemeralRendezvousStore store)
|
||||
public RendezvousTelemetry(
|
||||
InMemoryEphemeralRendezvousStore store,
|
||||
SessionChangeJournal? sessionChanges = null,
|
||||
ProvisioningRuntime? provisioning = null)
|
||||
{
|
||||
_store = store;
|
||||
_sessionChanges = sessionChanges;
|
||||
_provisioning = provisioning;
|
||||
_httpRequests = _meter.CreateCounter<long>("rendezvous.http.requests");
|
||||
_httpDuration = _meter.CreateHistogram<double>(
|
||||
"rendezvous.http.duration",
|
||||
@@ -75,9 +89,21 @@ internal sealed class RendezvousTelemetry : IDisposable
|
||||
};
|
||||
_httpRequests.Add(1, tags);
|
||||
_httpDuration.Record(elapsedMilliseconds, tags);
|
||||
IncrementPrometheus(
|
||||
"rendezvous_http_requests_total",
|
||||
Labels(("operation", operation), ("status_code", statusCode.ToString(CultureInfo.InvariantCulture))));
|
||||
ObservePrometheus(
|
||||
"rendezvous_http_duration_milliseconds",
|
||||
elapsedMilliseconds,
|
||||
Labels(("operation", operation), ("status_code", statusCode.ToString(CultureInfo.InvariantCulture))));
|
||||
}
|
||||
|
||||
public void RecordUdp(string operation, string result, double elapsedMilliseconds)
|
||||
public void RecordUdp(
|
||||
string operation,
|
||||
string result,
|
||||
double elapsedMilliseconds,
|
||||
int receivedBytes = 0,
|
||||
int responseBudgetBytes = 0)
|
||||
{
|
||||
TagList tags = new()
|
||||
{
|
||||
@@ -86,41 +112,315 @@ internal sealed class RendezvousTelemetry : IDisposable
|
||||
};
|
||||
_udpResults.Add(1, tags);
|
||||
_udpDuration.Record(elapsedMilliseconds, tags);
|
||||
IncrementPrometheus(
|
||||
"rendezvous_udp_results_total",
|
||||
Labels(("operation", operation), ("result", result)));
|
||||
ObservePrometheus(
|
||||
"rendezvous_udp_duration_milliseconds",
|
||||
elapsedMilliseconds,
|
||||
Labels(("operation", operation), ("result", result)));
|
||||
AddPrometheus(
|
||||
"rendezvous_udp_received_bytes_total",
|
||||
Math.Max(0, receivedBytes),
|
||||
Labels(("operation", operation)));
|
||||
AddPrometheus(
|
||||
"rendezvous_udp_response_budget_bytes_total",
|
||||
Math.Max(0, responseBudgetBytes),
|
||||
Labels(("operation", operation)));
|
||||
}
|
||||
|
||||
public void RecordLimiterDrop(string transport, string partition) =>
|
||||
public void RecordLimiterDrop(string transport, string partition)
|
||||
{
|
||||
_limiterDrops.Add(1, new TagList
|
||||
{
|
||||
{ "transport", transport },
|
||||
{ "partition", partition },
|
||||
});
|
||||
IncrementPrometheus(
|
||||
"rendezvous_limiter_drops_total",
|
||||
Labels(("transport", transport), ("partition", partition)));
|
||||
}
|
||||
|
||||
public void RecordAudit(string action, string result) =>
|
||||
public void RecordAudit(string action, string result)
|
||||
{
|
||||
_auditEvents.Add(1, new TagList
|
||||
{
|
||||
{ "action", action },
|
||||
{ "result", result },
|
||||
});
|
||||
IncrementPrometheus(
|
||||
"rendezvous_audit_events_total",
|
||||
Labels(("action", action), ("result", result)));
|
||||
}
|
||||
|
||||
public void RecordConnectionOutcome(string outcome, string elapsedBucket) =>
|
||||
public void RecordConnectionOutcome(string outcome, string elapsedBucket)
|
||||
{
|
||||
_connectionOutcomes.Add(1, new TagList
|
||||
{
|
||||
{ "outcome", outcome },
|
||||
{ "elapsed_bucket", elapsedBucket },
|
||||
});
|
||||
IncrementPrometheus(
|
||||
"rendezvous_connection_outcomes_total",
|
||||
Labels(("outcome", outcome), ("elapsed_bucket", elapsedBucket)));
|
||||
}
|
||||
|
||||
public void RecordOperatorAuthentication(string result) =>
|
||||
public void RecordOperatorAuthentication(string result)
|
||||
{
|
||||
_operatorAuthentication.Add(1, new TagList
|
||||
{
|
||||
{ "result", result },
|
||||
});
|
||||
IncrementPrometheus(
|
||||
"rendezvous_operator_authentication_total",
|
||||
Labels(("result", result)));
|
||||
}
|
||||
|
||||
public void RecordPairingLatency(double elapsedMilliseconds) =>
|
||||
public void RecordPairingLatency(double elapsedMilliseconds)
|
||||
{
|
||||
_pairingLatency.Record(elapsedMilliseconds);
|
||||
ObservePrometheus("rendezvous_pairing_latency_milliseconds", elapsedMilliseconds, string.Empty);
|
||||
}
|
||||
|
||||
public void RecordMetricsScrape(string result) => IncrementPrometheus(
|
||||
"rendezvous_metrics_scrapes_total",
|
||||
Labels(("result", result)));
|
||||
|
||||
public string RenderPrometheus()
|
||||
{
|
||||
StringBuilder output = new(16 * 1024);
|
||||
foreach (IGrouping<string, KeyValuePair<MetricSeriesKey, long>> family in _prometheusCounters
|
||||
.OrderBy(static item => item.Key.Name, StringComparer.Ordinal)
|
||||
.ThenBy(static item => item.Key.Labels, StringComparer.Ordinal)
|
||||
.GroupBy(static item => item.Key.Name, StringComparer.Ordinal))
|
||||
{
|
||||
output.Append("# TYPE ").Append(family.Key).Append(" counter\n");
|
||||
foreach (KeyValuePair<MetricSeriesKey, long> series in family)
|
||||
{
|
||||
AppendValue(output, series.Key.Name, series.Key.Labels, series.Value);
|
||||
}
|
||||
}
|
||||
foreach (IGrouping<string, KeyValuePair<MetricSeriesKey, PrometheusHistogram>> family in
|
||||
_prometheusHistograms
|
||||
.OrderBy(static item => item.Key.Name, StringComparer.Ordinal)
|
||||
.ThenBy(static item => item.Key.Labels, StringComparer.Ordinal)
|
||||
.GroupBy(static item => item.Key.Name, StringComparer.Ordinal))
|
||||
{
|
||||
output.Append("# TYPE ").Append(family.Key).Append(" histogram\n");
|
||||
foreach (KeyValuePair<MetricSeriesKey, PrometheusHistogram> series in family)
|
||||
{
|
||||
series.Value.Append(output, series.Key.Name, series.Key.Labels);
|
||||
}
|
||||
}
|
||||
|
||||
EphemeralStoreSnapshot store = _store.GetMetricsSnapshot();
|
||||
AppendGauge(output, "rendezvous_store_active_listings", store.ActiveListings);
|
||||
AppendGauge(output, "rendezvous_store_fresh_presence_bindings", store.FreshPresenceBindings);
|
||||
AppendGauge(
|
||||
output,
|
||||
"rendezvous_store_awaiting_presence_listings",
|
||||
Math.Max(0, store.ActiveListings - store.FreshPresenceBindings));
|
||||
AppendGauge(output, "rendezvous_store_active_leases", store.ActiveListings);
|
||||
AppendGauge(output, "rendezvous_store_active_attempts", store.ActiveJoinAttempts);
|
||||
AppendGauge(output, "rendezvous_queue_depth", store.ActiveJoinAttempts);
|
||||
AppendGauge(output, "rendezvous_store_replay_markers", store.ReplayMarkers);
|
||||
AppendGauge(output, "rendezvous_store_available", store.IsAvailable ? 1 : 0);
|
||||
AppendGauge(output, "rendezvous_store_draining", store.IsDraining ? 1 : 0);
|
||||
AppendCounter(output, "rendezvous_store_expiry_churn_total", store.ExpiryChurn);
|
||||
if (_sessionChanges is not null)
|
||||
{
|
||||
AppendGauge(output, "rendezvous_browser_sse_subscribers", _sessionChanges.SubscriberCount);
|
||||
AppendGauge(output, "rendezvous_browser_sse_tenants", _sessionChanges.SubscribedTenantCount);
|
||||
AppendGauge(output, "rendezvous_browser_replay_entries", _sessionChanges.ReplayCount);
|
||||
}
|
||||
AppendProcessMetrics(output);
|
||||
AppendSigningKeyMetrics(output);
|
||||
return output.ToString();
|
||||
}
|
||||
|
||||
private void AppendSigningKeyMetrics(StringBuilder output)
|
||||
{
|
||||
if (_provisioning is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
DateTimeOffset now = DateTimeOffset.UtcNow;
|
||||
SigningKeyStatus[] statuses = _provisioning.SigningKeys.GetStatuses(now).ToArray();
|
||||
output.Append("# TYPE rendezvous_signing_keys gauge\n");
|
||||
foreach (IGrouping<string, SigningKeyStatus> state in statuses.GroupBy(
|
||||
static status => status.Status,
|
||||
StringComparer.Ordinal))
|
||||
{
|
||||
AppendValue(
|
||||
output,
|
||||
"rendezvous_signing_keys",
|
||||
Labels(("state", state.Key)),
|
||||
state.Count());
|
||||
}
|
||||
double seconds = statuses
|
||||
.Where(static status => status.Status == "signing")
|
||||
.Select(status => Math.Max(0, (status.SignUntil - now).TotalSeconds))
|
||||
.DefaultIfEmpty(0)
|
||||
.Min();
|
||||
AppendGauge(output, "rendezvous_signing_key_sign_seconds_remaining", seconds);
|
||||
}
|
||||
|
||||
private static void AppendProcessMetrics(StringBuilder output)
|
||||
{
|
||||
using Process process = Process.GetCurrentProcess();
|
||||
process.Refresh();
|
||||
AppendCounter(output, "process_cpu_seconds_total", process.TotalProcessorTime.TotalSeconds);
|
||||
AppendGauge(output, "process_resident_memory_bytes", process.WorkingSet64);
|
||||
AppendGauge(output, "process_virtual_memory_bytes", process.VirtualMemorySize64);
|
||||
AppendGauge(output, "process_threads", process.Threads.Count);
|
||||
try
|
||||
{
|
||||
AppendGauge(
|
||||
output,
|
||||
"process_open_file_descriptors",
|
||||
Directory.EnumerateFileSystemEntries("/proc/self/fd").Count());
|
||||
}
|
||||
catch (Exception exception) when (exception is IOException
|
||||
or UnauthorizedAccessException)
|
||||
{
|
||||
}
|
||||
AppendGauge(output, "dotnet_gc_heap_size_bytes", GC.GetTotalMemory(forceFullCollection: false));
|
||||
output.Append("# TYPE dotnet_gc_collections_total counter\n");
|
||||
AppendValue(output, "dotnet_gc_collections_total", Labels(("generation", "0")), GC.CollectionCount(0));
|
||||
AppendValue(output, "dotnet_gc_collections_total", Labels(("generation", "1")), GC.CollectionCount(1));
|
||||
AppendValue(output, "dotnet_gc_collections_total", Labels(("generation", "2")), GC.CollectionCount(2));
|
||||
AppendGauge(output, "dotnet_thread_pool_threads", ThreadPool.ThreadCount);
|
||||
ThreadPool.GetAvailableThreads(out int workerThreads, out int completionThreads);
|
||||
AppendGauge(output, "dotnet_thread_pool_available_worker_threads", workerThreads);
|
||||
AppendGauge(output, "dotnet_thread_pool_available_completion_threads", completionThreads);
|
||||
}
|
||||
|
||||
private void IncrementPrometheus(string name, string labels) =>
|
||||
_prometheusCounters.AddOrUpdate(new(name, labels), 1, static (_, current) => current + 1);
|
||||
|
||||
private void AddPrometheus(string name, long value, string labels)
|
||||
{
|
||||
if (value <= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
_prometheusCounters.AddOrUpdate(new(name, labels), value, (_, current) => current + value);
|
||||
}
|
||||
|
||||
private void ObservePrometheus(string name, double value, string labels) =>
|
||||
_prometheusHistograms.GetOrAdd(new(name, labels), static _ => new()).Observe(value);
|
||||
|
||||
private static string Labels(params (string Name, string Value)[] labels)
|
||||
{
|
||||
if (labels.Length == 0)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
return "{" + string.Join(',', labels.Select(static label =>
|
||||
$"{label.Name}=\"{EscapeLabel(NormalizeLabel(label.Value))}\"")) + "}";
|
||||
}
|
||||
|
||||
private static string NormalizeLabel(string value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value) || value.Length > 64)
|
||||
{
|
||||
return "other";
|
||||
}
|
||||
return value.All(static character => char.IsAsciiLetterOrDigit(character)
|
||||
|| character is '-' or '_' or '.')
|
||||
? value
|
||||
: "other";
|
||||
}
|
||||
|
||||
private static string EscapeLabel(string value) => value
|
||||
.Replace("\\", "\\\\", StringComparison.Ordinal)
|
||||
.Replace("\"", "\\\"", StringComparison.Ordinal)
|
||||
.Replace("\n", "\\n", StringComparison.Ordinal);
|
||||
|
||||
private static void AppendCounter(
|
||||
StringBuilder output,
|
||||
string name,
|
||||
double value,
|
||||
string labels = "")
|
||||
{
|
||||
output.Append("# TYPE ").Append(name).Append(" counter\n");
|
||||
AppendValue(output, name, labels, value);
|
||||
}
|
||||
|
||||
private static void AppendGauge(
|
||||
StringBuilder output,
|
||||
string name,
|
||||
double value,
|
||||
string labels = "")
|
||||
{
|
||||
output.Append("# TYPE ").Append(name).Append(" gauge\n");
|
||||
AppendValue(output, name, labels, value);
|
||||
}
|
||||
|
||||
private static void AppendValue(StringBuilder output, string name, string labels, double value) =>
|
||||
output.Append(name)
|
||||
.Append(labels)
|
||||
.Append(' ')
|
||||
.Append(value.ToString("R", CultureInfo.InvariantCulture))
|
||||
.Append('\n');
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_activities.Dispose();
|
||||
_meter.Dispose();
|
||||
}
|
||||
|
||||
private readonly record struct MetricSeriesKey(string Name, string Labels);
|
||||
|
||||
private sealed class PrometheusHistogram
|
||||
{
|
||||
private static readonly double[] Bounds = [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000];
|
||||
private readonly object _gate = new();
|
||||
private readonly long[] _buckets = new long[Bounds.Length];
|
||||
private long _count;
|
||||
private double _sum;
|
||||
|
||||
public void Observe(double value)
|
||||
{
|
||||
if (!double.IsFinite(value) || value < 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
lock (_gate)
|
||||
{
|
||||
_count++;
|
||||
_sum += value;
|
||||
for (int index = 0; index < Bounds.Length; index++)
|
||||
{
|
||||
if (value <= Bounds[index])
|
||||
{
|
||||
_buckets[index]++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Append(StringBuilder output, string name, string labels)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
for (int index = 0; index < Bounds.Length; index++)
|
||||
{
|
||||
AppendValue(
|
||||
output,
|
||||
name + "_bucket",
|
||||
AddLabel(labels, "le", Bounds[index].ToString("R", CultureInfo.InvariantCulture)),
|
||||
_buckets[index]);
|
||||
}
|
||||
AppendValue(output, name + "_bucket", AddLabel(labels, "le", "+Inf"), _count);
|
||||
AppendValue(output, name + "_sum", labels, _sum);
|
||||
AppendValue(output, name + "_count", labels, _count);
|
||||
}
|
||||
}
|
||||
|
||||
private static string AddLabel(string labels, string name, string value) =>
|
||||
string.IsNullOrEmpty(labels)
|
||||
? $"{{{name}=\"{value}\"}}"
|
||||
: labels[..^1] + $",{name}=\"{value}\"}}";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ using FinalFactory.Rendezvous.Server.Abuse;
|
||||
using FinalFactory.Rendezvous.Server.Browser;
|
||||
using FinalFactory.Rendezvous.Server.ConnectionOutcomes;
|
||||
using FinalFactory.Rendezvous.Server.Deployment;
|
||||
using FinalFactory.Rendezvous.Server.Diagnostics;
|
||||
using FinalFactory.Rendezvous.Server.Http;
|
||||
using FinalFactory.Rendezvous.Server.JoinAttempts;
|
||||
using FinalFactory.Rendezvous.Server.Observability;
|
||||
@@ -251,6 +252,37 @@ if (!builder.Environment.IsDevelopment() && !isOpenApiGeneration)
|
||||
}
|
||||
|
||||
builder.Services.AddSingleton(Microsoft.Extensions.Options.Options.Create(deploymentOptions));
|
||||
DiagnosticDashboardOptions diagnosticDashboardOptions = builder.Configuration
|
||||
.GetSection(DiagnosticDashboardOptions.SectionName)
|
||||
.Get<DiagnosticDashboardOptions>() ?? new DiagnosticDashboardOptions();
|
||||
IReadOnlyList<string> diagnosticErrors = diagnosticDashboardOptions.Validate();
|
||||
if (diagnosticErrors.Count > 0)
|
||||
{
|
||||
throw new DeploymentConfigurationException(diagnosticErrors);
|
||||
}
|
||||
builder.Services.AddSingleton(
|
||||
Microsoft.Extensions.Options.Options.Create(diagnosticDashboardOptions));
|
||||
PrometheusMetricsOptions metricsOptions = builder.Configuration
|
||||
.GetSection(PrometheusMetricsOptions.SectionName)
|
||||
.Get<PrometheusMetricsOptions>() ?? new PrometheusMetricsOptions();
|
||||
IReadOnlyList<string> metricsErrors = metricsOptions.Validate();
|
||||
if (metricsErrors.Count > 0)
|
||||
{
|
||||
throw new DeploymentConfigurationException(metricsErrors);
|
||||
}
|
||||
builder.Services.AddSingleton(Microsoft.Extensions.Options.Options.Create(metricsOptions));
|
||||
MetricsAccessCredential? metricsCredential = null;
|
||||
if (metricsOptions.Enabled && !isOpenApiGeneration)
|
||||
{
|
||||
EnvironmentSecretProvider metricsSecrets = new();
|
||||
if (!MetricsAccessCredential.TryCreate(metricsOptions, metricsSecrets, out metricsCredential)
|
||||
|| metricsCredential is null)
|
||||
{
|
||||
throw new DeploymentConfigurationException(
|
||||
[$"{PrometheusMetricsOptions.SectionName}:BearerTokenSecretReference did not resolve to 32-128 visible ASCII bytes."]);
|
||||
}
|
||||
builder.Services.AddSingleton(metricsCredential);
|
||||
}
|
||||
builder.Services.Configure<HostOptions>(options =>
|
||||
options.ShutdownTimeout = TimeSpan.FromSeconds(deploymentOptions.DrainDeadlineSeconds + 10));
|
||||
|
||||
@@ -349,6 +381,8 @@ app.MapOpenApi();
|
||||
app.MapRendezvousContractEndpoints();
|
||||
app.MapOperatorEndpoints();
|
||||
app.MapRendezvousHealthEndpoints();
|
||||
app.MapDiagnosticDashboardEndpoints();
|
||||
app.MapPrometheusMetricsEndpoint(metricsOptions, metricsCredential);
|
||||
|
||||
await app.RunAsync();
|
||||
|
||||
|
||||
@@ -82,7 +82,9 @@ internal sealed class NatMediationProcessor(
|
||||
telemetry?.RecordUdp(
|
||||
"frozen",
|
||||
result.ToString(),
|
||||
Stopwatch.GetElapsedTime(started).TotalMilliseconds);
|
||||
Stopwatch.GetElapsedTime(started).TotalMilliseconds,
|
||||
encoded.Length,
|
||||
result == NatMediationResult.Introduced ? 2 * ContractLimits.UdpDatagramMaxBytes : 0);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -157,7 +159,7 @@ internal sealed class NatMediationProcessor(
|
||||
observedPublicEndpoint,
|
||||
token,
|
||||
introductionSink,
|
||||
cancellationToken);
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
internal NatMediationResult ProcessRequestAfterIngress(
|
||||
@@ -165,6 +167,7 @@ internal sealed class NatMediationProcessor(
|
||||
IPEndPoint observedPublicEndpoint,
|
||||
string token,
|
||||
INatIntroductionSink introductionSink,
|
||||
int receivedBytes = 0,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
long started = Stopwatch.GetTimestamp();
|
||||
@@ -178,7 +181,9 @@ internal sealed class NatMediationProcessor(
|
||||
telemetry?.RecordUdp(
|
||||
"litenet",
|
||||
result.ToString(),
|
||||
Stopwatch.GetElapsedTime(started).TotalMilliseconds);
|
||||
Stopwatch.GetElapsedTime(started).TotalMilliseconds,
|
||||
receivedBytes,
|
||||
result == NatMediationResult.Introduced ? 2 * ContractLimits.UdpDatagramMaxBytes : 0);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
@@ -201,7 +201,7 @@ internal sealed class UdpMediatorService : BackgroundService
|
||||
&& token is not null)
|
||||
{
|
||||
_ = processor.ProcessRequestAfterIngress(
|
||||
claimedLocalEndpoint, endPoint, token, sink);
|
||||
claimedLocalEndpoint, endPoint, token, sink, length);
|
||||
}
|
||||
|
||||
// Every inbound packet is consumed here. NatPunchModule is used only for outbound introductions.
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import test from "node:test";
|
||||
import {
|
||||
SessionProjection,
|
||||
buildBrowseUrl,
|
||||
buildStreamUrl,
|
||||
chooseSnapshotTransport,
|
||||
safeText,
|
||||
} from "../../src/FinalFactory.Rendezvous.Server/Diagnostics/Assets/app.mjs";
|
||||
|
||||
const cursor = "rvs1.test-cursor";
|
||||
|
||||
function session(id, name = `Host ${id}`) {
|
||||
return {
|
||||
contractVersion: 1,
|
||||
listingId: `00000000-0000-0000-0000-${String(id).padStart(12, "0")}`,
|
||||
gameId: "space-game",
|
||||
environmentId: "smoke",
|
||||
regionId: "local",
|
||||
protocolVersion: 1,
|
||||
buildVersion: "1.0.0",
|
||||
displayName: name,
|
||||
visibility: "public",
|
||||
publisherTrustMode: "managedDedicated",
|
||||
capacity: { currentPlayers: 1, maximumPlayers: 8 },
|
||||
metadata: { mode: "online-coop" },
|
||||
};
|
||||
}
|
||||
|
||||
function event(kind, values = {}) {
|
||||
return {
|
||||
contractVersion: 1,
|
||||
kind,
|
||||
cursor: values.cursor ?? cursor,
|
||||
session: values.session ?? null,
|
||||
listingId: values.listingId ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
test("snapshot and ordered deltas match the final public projection", () => {
|
||||
const projection = new SessionProjection(10);
|
||||
assert.equal(projection.replace([session(1)], cursor), "applied");
|
||||
assert.equal(projection.apply(event("sessionUpsert", {
|
||||
cursor: `${cursor}-2`,
|
||||
session: session(2),
|
||||
})), "applied");
|
||||
assert.equal(projection.apply(event("sessionUpsert", {
|
||||
cursor: `${cursor}-3`,
|
||||
session: session(1, "Updated host"),
|
||||
})), "applied");
|
||||
assert.equal(projection.apply(event("sessionRemove", {
|
||||
cursor: `${cursor}-4`,
|
||||
listingId: session(2).listingId,
|
||||
})), "applied");
|
||||
|
||||
assert.deepEqual(
|
||||
projection.sessions().map((entry) => entry.session.displayName),
|
||||
["Updated host"],
|
||||
);
|
||||
assert.equal(projection.cursor, `${cursor}-4`);
|
||||
});
|
||||
|
||||
test("reset, malformed events, and a partial snapshot fail closed", () => {
|
||||
const projection = new SessionProjection(2);
|
||||
assert.equal(projection.replace([session(1)], cursor, true), "applied");
|
||||
assert.equal(projection.apply(event("sessionRemove", {
|
||||
listingId: session(1).listingId,
|
||||
})), "refresh");
|
||||
assert.equal(projection.apply(event("reset")), "reset");
|
||||
assert.equal(projection.apply({ kind: "sessionUpsert" }), "invalid");
|
||||
assert.equal(projection.replace([session(1), session(2), session(3)], cursor), "overflow");
|
||||
});
|
||||
|
||||
test("bursts coalesce by listing identity and memory stays bounded", () => {
|
||||
const projection = new SessionProjection(2);
|
||||
assert.equal(projection.replace([], cursor), "applied");
|
||||
for (let index = 0; index < 1_000; index += 1) {
|
||||
assert.equal(projection.apply(event("sessionUpsert", {
|
||||
cursor: `${cursor}-${index}`,
|
||||
session: session(1, `Host ${index}`),
|
||||
})), "applied");
|
||||
}
|
||||
assert.equal(projection.sessions().length, 1);
|
||||
assert.equal(projection.sessions()[0].session.displayName, "Host 999");
|
||||
assert.equal(projection.apply(event("sessionUpsert", { session: session(2) })), "applied");
|
||||
assert.equal(projection.apply(event("sessionUpsert", { session: session(3) })), "overflow");
|
||||
assert.equal(projection.sessions().length, 2);
|
||||
});
|
||||
|
||||
test("requests are fixed same-origin paths with an exact configured filter", () => {
|
||||
const filter = {
|
||||
gameId: "space-game",
|
||||
environmentId: "smoke",
|
||||
protocolVersion: 1,
|
||||
regionId: "local",
|
||||
excludeFull: true,
|
||||
};
|
||||
const browse = buildBrowseUrl(filter);
|
||||
const stream = buildStreamUrl(filter, cursor);
|
||||
assert.match(browse, /^\/v1\/sessions\?/u);
|
||||
assert.match(stream, /^\/v1\/sessions\/stream\?/u);
|
||||
assert.match(browse, /gameId=space-game/u);
|
||||
assert.match(browse, /environmentId=smoke/u);
|
||||
assert.match(stream, /streamCursor=rvs1.test-cursor/u);
|
||||
assert.doesNotMatch(browse, /https?:/u);
|
||||
assert.doesNotMatch(stream, /https?:/u);
|
||||
});
|
||||
|
||||
test("snapshot transport preserves deliberate polling and bounds partial snapshots", () => {
|
||||
assert.equal(chooseSnapshotTransport(false, false), "stream");
|
||||
assert.equal(chooseSnapshotTransport(true, false), "boundedPolling");
|
||||
assert.equal(chooseSnapshotTransport(false, true), "pollingOnly");
|
||||
assert.equal(chooseSnapshotTransport(true, true), "pollingOnly");
|
||||
});
|
||||
|
||||
test("hostile display data remains literal text and no unsafe DOM sink exists", async () => {
|
||||
const payload = `<img src=x onerror=alert(1)><style>body{display:none}</style><a href=//evil>go</a>`;
|
||||
assert.equal(safeText(payload, 200), payload);
|
||||
const source = await readFile(new URL(
|
||||
"../../src/FinalFactory.Rendezvous.Server/Diagnostics/Assets/app.mjs",
|
||||
import.meta.url,
|
||||
), "utf8");
|
||||
for (const forbidden of [
|
||||
"innerHTML",
|
||||
"outerHTML",
|
||||
"insertAdjacentHTML",
|
||||
"document.write",
|
||||
"eval(",
|
||||
"new Function",
|
||||
"window.location",
|
||||
]) {
|
||||
assert.equal(source.includes(forbidden), false, `unsafe browser sink: ${forbidden}`);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,205 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text.Json;
|
||||
using FinalFactory.Rendezvous.Server.Diagnostics;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace FinalFactory.Rendezvous.Tests.Diagnostics;
|
||||
|
||||
public sealed class DiagnosticDashboardTests
|
||||
{
|
||||
[Fact]
|
||||
public void ConfigurationRequiresBoundedUniqueAllowListedScopes()
|
||||
{
|
||||
Assert.Empty(ValidOptions().Validate());
|
||||
Assert.Empty(new DiagnosticDashboardOptions().Validate());
|
||||
|
||||
DiagnosticDashboardOptions invalid = ValidOptions() with
|
||||
{
|
||||
PollIntervalSeconds = 1,
|
||||
MaximumRenderedSessions = 501,
|
||||
Scopes =
|
||||
[
|
||||
new DiagnosticDashboardScope
|
||||
{
|
||||
GameId = "INVALID",
|
||||
EnvironmentId = "smoke",
|
||||
ProtocolVersions = [0, 0],
|
||||
Regions = ["INVALID", "INVALID"],
|
||||
},
|
||||
new DiagnosticDashboardScope
|
||||
{
|
||||
GameId = "INVALID",
|
||||
EnvironmentId = "smoke",
|
||||
ProtocolVersions = [1],
|
||||
Regions = ["local"],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
IReadOnlyList<string> errors = invalid.Validate();
|
||||
Assert.Contains(errors, error => error.Contains("PollIntervalSeconds", StringComparison.Ordinal));
|
||||
Assert.Contains(errors, error => error.Contains("MaximumRenderedSessions", StringComparison.Ordinal));
|
||||
Assert.Contains(errors, error => error.Contains("invalid game", StringComparison.Ordinal));
|
||||
Assert.Contains(errors, error => error.Contains("protocol versions", StringComparison.Ordinal));
|
||||
Assert.Contains(errors, error => error.Contains("regions", StringComparison.Ordinal));
|
||||
Assert.Contains(errors, error => error.Contains("duplicate", StringComparison.Ordinal));
|
||||
|
||||
DiagnosticDashboardOptions nullBound = ValidOptions() with
|
||||
{
|
||||
Scopes = null!,
|
||||
};
|
||||
Assert.Contains(
|
||||
nullBound.Validate(),
|
||||
error => error.Contains("Scopes must contain", StringComparison.Ordinal));
|
||||
DiagnosticDashboardOptions nullScope = ValidOptions() with
|
||||
{
|
||||
Scopes = [null!],
|
||||
};
|
||||
Assert.Contains(
|
||||
nullScope.Validate(),
|
||||
error => error.Contains("null entries", StringComparison.Ordinal));
|
||||
DiagnosticDashboardOptions nullCollections = ValidOptions() with
|
||||
{
|
||||
Scopes =
|
||||
[
|
||||
new DiagnosticDashboardScope
|
||||
{
|
||||
GameId = null!,
|
||||
EnvironmentId = null!,
|
||||
ProtocolVersions = null!,
|
||||
Regions = null!,
|
||||
},
|
||||
],
|
||||
};
|
||||
Assert.True(nullCollections.Validate().Count >= 3);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DisabledDashboardHasNoPublicAssetOrConfigurationSurface()
|
||||
{
|
||||
await using DashboardHost host = await DashboardHost.StartAsync(new());
|
||||
|
||||
Assert.Equal(HttpStatusCode.NotFound, (await host.Client.GetAsync("diagnostics/")).StatusCode);
|
||||
Assert.Equal(HttpStatusCode.NotFound, (await host.Client.GetAsync("diagnostics/app.mjs")).StatusCode);
|
||||
Assert.Equal(HttpStatusCode.NotFound, (await host.Client.GetAsync("diagnostics/config.json")).StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task EnabledDashboardServesOnlyHardenedSameOriginReadOnlyAssets()
|
||||
{
|
||||
await using DashboardHost host = await DashboardHost.StartAsync(ValidOptions());
|
||||
using HttpRequestMessage request = new(HttpMethod.Get, "diagnostics/");
|
||||
request.Headers.Add("Origin", "https://hostile.example");
|
||||
using HttpResponseMessage response = await host.Client.SendAsync(request);
|
||||
string html = await response.Content.ReadAsStringAsync();
|
||||
|
||||
Assert.True(response.IsSuccessStatusCode, html);
|
||||
Assert.Equal("text/html", response.Content.Headers.ContentType?.MediaType);
|
||||
Assert.Equal("DENY", Header(response, "X-Frame-Options"));
|
||||
Assert.Equal("nosniff", Header(response, "X-Content-Type-Options"));
|
||||
Assert.Equal("no-referrer", Header(response, "Referrer-Policy"));
|
||||
Assert.Contains("default-src 'none'", Header(response, "Content-Security-Policy"), StringComparison.Ordinal);
|
||||
Assert.Contains("connect-src 'self'", Header(response, "Content-Security-Policy"), StringComparison.Ordinal);
|
||||
Assert.Contains("frame-ancestors 'none'", Header(response, "Content-Security-Policy"), StringComparison.Ordinal);
|
||||
Assert.False(response.Headers.Contains("Access-Control-Allow-Origin"));
|
||||
Assert.Contains("<html lang=\"en\">", html, StringComparison.Ordinal);
|
||||
Assert.Contains("href=\"#main-content\"", html, StringComparison.Ordinal);
|
||||
Assert.Contains("aria-live=\"polite\"", html, StringComparison.Ordinal);
|
||||
Assert.Contains("type=\"module\" src=\"/diagnostics/app.mjs\"", html, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain("<script src=\"http", html, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("/v1/operator", html, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
using HttpResponseMessage scriptResponse = await host.Client.GetAsync("diagnostics/app.mjs");
|
||||
string script = await scriptResponse.Content.ReadAsStringAsync();
|
||||
Assert.Equal("text/javascript", scriptResponse.Content.Headers.ContentType?.MediaType);
|
||||
Assert.DoesNotContain("/v1/operator", script, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("Authorization", script, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("innerHTML", script, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BrowserConfigurationContainsOnlyTheExplicitPublicAllowList()
|
||||
{
|
||||
await using DashboardHost host = await DashboardHost.StartAsync(ValidOptions());
|
||||
using HttpResponseMessage response = await host.Client.GetAsync("diagnostics/config.json");
|
||||
using JsonDocument document = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
|
||||
JsonElement root = document.RootElement;
|
||||
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
Assert.Equal(1, root.GetProperty("contractVersion").GetInt32());
|
||||
Assert.Equal(10, root.GetProperty("pollIntervalSeconds").GetInt32());
|
||||
JsonElement scope = Assert.Single(root.GetProperty("scopes").EnumerateArray());
|
||||
Assert.Equal("space-game", scope.GetProperty("gameId").GetString());
|
||||
Assert.Equal("smoke", scope.GetProperty("environmentId").GetString());
|
||||
Assert.Equal("public", scope.GetProperty("visibility").GetString());
|
||||
string json = root.GetRawText();
|
||||
Assert.DoesNotContain("http", json, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("credential", json, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("token", json, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain("endpoint", json, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static string Header(HttpResponseMessage response, string name) =>
|
||||
Assert.Single(response.Headers.GetValues(name));
|
||||
|
||||
private static DiagnosticDashboardOptions ValidOptions() => new()
|
||||
{
|
||||
Enabled = true,
|
||||
PollIntervalSeconds = 10,
|
||||
MaximumRenderedSessions = 100,
|
||||
Scopes =
|
||||
[
|
||||
new DiagnosticDashboardScope
|
||||
{
|
||||
GameId = "space-game",
|
||||
EnvironmentId = "smoke",
|
||||
ProtocolVersions = [1, 2],
|
||||
Regions = ["local"],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
private sealed class DashboardHost : IAsyncDisposable
|
||||
{
|
||||
private readonly WebApplication _application;
|
||||
|
||||
private DashboardHost(WebApplication application, HttpClient client)
|
||||
{
|
||||
_application = application;
|
||||
Client = client;
|
||||
}
|
||||
|
||||
public HttpClient Client { get; }
|
||||
|
||||
public static async Task<DashboardHost> StartAsync(DiagnosticDashboardOptions options)
|
||||
{
|
||||
int port = ReserveTcpPort();
|
||||
string address = $"http://127.0.0.1:{port}";
|
||||
WebApplicationBuilder builder = WebApplication.CreateBuilder();
|
||||
builder.WebHost.UseUrls(address);
|
||||
builder.Services.AddSingleton(Options.Create(options));
|
||||
WebApplication application = builder.Build();
|
||||
application.MapDiagnosticDashboardEndpoints();
|
||||
await application.StartAsync();
|
||||
return new(application, new HttpClient { BaseAddress = new Uri(address) });
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
Client.Dispose();
|
||||
await _application.StopAsync();
|
||||
await _application.DisposeAsync();
|
||||
}
|
||||
|
||||
private static int ReserveTcpPort()
|
||||
{
|
||||
using TcpListener listener = new(IPAddress.Loopback, 0);
|
||||
listener.Start();
|
||||
return ((IPEndPoint)listener.LocalEndpoint).Port;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using FinalFactory.Rendezvous.Server.Browser;
|
||||
using FinalFactory.Rendezvous.Server.Observability;
|
||||
using FinalFactory.Rendezvous.Server.Provisioning;
|
||||
using FinalFactory.Rendezvous.Server.State;
|
||||
using FinalFactory.Rendezvous.Tests.State;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
namespace FinalFactory.Rendezvous.Tests.Observability;
|
||||
|
||||
[Collection(RendezvousTelemetryIsolation.Name)]
|
||||
public sealed class PrometheusMetricsTests
|
||||
{
|
||||
private const string Token = "0123456789abcdef0123456789abcdef";
|
||||
|
||||
[Fact]
|
||||
public void EnabledExporterRequiresAnExternalBoundedSecretReference()
|
||||
{
|
||||
Assert.Empty(new PrometheusMetricsOptions().Validate());
|
||||
Assert.Empty(new PrometheusMetricsOptions
|
||||
{
|
||||
Enabled = true,
|
||||
BearerTokenSecretReference = "file:/run/secrets/rendezvous-metrics-token",
|
||||
}.Validate());
|
||||
Assert.NotEmpty(new PrometheusMetricsOptions { Enabled = true }.Validate());
|
||||
Assert.NotEmpty(new PrometheusMetricsOptions
|
||||
{
|
||||
Enabled = true,
|
||||
BearerTokenSecretReference = "literal-secret",
|
||||
}.Validate());
|
||||
Assert.NotEmpty(new PrometheusMetricsOptions
|
||||
{
|
||||
Enabled = true,
|
||||
BearerTokenSecretReference = "file:relative-token",
|
||||
}.Validate());
|
||||
Assert.NotEmpty(new PrometheusMetricsOptions
|
||||
{
|
||||
Enabled = true,
|
||||
BearerTokenSecretReference = null!,
|
||||
}.Validate());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExporterIsDisabledByDefaultAndConcealsRejectedAuthentication()
|
||||
{
|
||||
await using MetricsHost disabled = await MetricsHost.StartAsync(
|
||||
new PrometheusMetricsOptions(),
|
||||
credential: null);
|
||||
Assert.Equal(HttpStatusCode.NotFound, (await disabled.Client.GetAsync("metrics")).StatusCode);
|
||||
|
||||
PrometheusMetricsOptions options = EnabledOptions();
|
||||
using MetricsAccessCredential credential = Credential(options);
|
||||
await using MetricsHost enabled = await MetricsHost.StartAsync(options, credential);
|
||||
Assert.Equal(HttpStatusCode.NotFound, (await enabled.Client.GetAsync("metrics")).StatusCode);
|
||||
using HttpRequestMessage wrong = new(HttpMethod.Get, "metrics");
|
||||
wrong.Headers.Authorization = new("Bearer", new string('x', 32));
|
||||
Assert.Equal(HttpStatusCode.NotFound, (await enabled.Client.SendAsync(wrong)).StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AuthenticatedExporterReturnsBoundedPrivacySafePrometheusFamilies()
|
||||
{
|
||||
PrometheusMetricsOptions options = EnabledOptions();
|
||||
using MetricsAccessCredential credential = Credential(options);
|
||||
await using MetricsHost host = await MetricsHost.StartAsync(options, credential);
|
||||
host.Telemetry.RecordHttp("BrowseSessions", 200, 3.5);
|
||||
host.Telemetry.RecordHttp("listing-id/canary", 500, 4.5);
|
||||
host.Telemetry.RecordUdp("frozen", "Introduced", 1.25, 128, 2400);
|
||||
host.Telemetry.RecordLimiterDrop("udp", "rate-or-concurrency");
|
||||
host.Telemetry.RecordConnectionOutcome("Connected", "UnderOneSecond");
|
||||
host.Telemetry.RecordPairingLatency(12.5);
|
||||
using HttpRequestMessage request = new(HttpMethod.Get, "metrics");
|
||||
request.Headers.Authorization = new("Bearer", Token);
|
||||
using HttpResponseMessage response = await host.Client.SendAsync(request);
|
||||
string body = await response.Content.ReadAsStringAsync();
|
||||
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
Assert.Equal("text/plain", response.Content.Headers.ContentType?.MediaType);
|
||||
Assert.Equal("no-store", response.Headers.CacheControl?.ToString());
|
||||
Assert.Contains("rendezvous_http_requests_total{operation=\"BrowseSessions\",status_code=\"200\"} 1", body, StringComparison.Ordinal);
|
||||
Assert.Contains("rendezvous_http_duration_milliseconds_bucket", body, StringComparison.Ordinal);
|
||||
Assert.Contains("rendezvous_udp_received_bytes_total{operation=\"frozen\"} 128", body, StringComparison.Ordinal);
|
||||
Assert.Contains("rendezvous_udp_response_budget_bytes_total{operation=\"frozen\"} 2400", body, StringComparison.Ordinal);
|
||||
Assert.Contains("rendezvous_store_active_listings", body, StringComparison.Ordinal);
|
||||
Assert.Contains("rendezvous_browser_sse_subscribers", body, StringComparison.Ordinal);
|
||||
Assert.Contains("process_resident_memory_bytes", body, StringComparison.Ordinal);
|
||||
Assert.Contains("dotnet_gc_heap_size_bytes", body, StringComparison.Ordinal);
|
||||
Assert.Contains("operation=\"other\"", body, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain("listing-id", body, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.DoesNotContain(Token, body, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
private static PrometheusMetricsOptions EnabledOptions() => new()
|
||||
{
|
||||
Enabled = true,
|
||||
BearerTokenSecretReference = "file:/metrics-token",
|
||||
};
|
||||
|
||||
private static MetricsAccessCredential Credential(PrometheusMetricsOptions options)
|
||||
{
|
||||
using DictionarySecretProvider secrets = new(new Dictionary<string, byte[]>
|
||||
{
|
||||
[options.BearerTokenSecretReference] = Encoding.ASCII.GetBytes(Token + "\n"),
|
||||
});
|
||||
Assert.True(MetricsAccessCredential.TryCreate(options, secrets, out MetricsAccessCredential? credential));
|
||||
return Assert.IsType<MetricsAccessCredential>(credential);
|
||||
}
|
||||
|
||||
private sealed class MetricsHost : IAsyncDisposable
|
||||
{
|
||||
private readonly WebApplication _application;
|
||||
|
||||
private MetricsHost(
|
||||
WebApplication application,
|
||||
HttpClient client,
|
||||
RendezvousTelemetry telemetry)
|
||||
{
|
||||
_application = application;
|
||||
Client = client;
|
||||
Telemetry = telemetry;
|
||||
}
|
||||
|
||||
public HttpClient Client { get; }
|
||||
public RendezvousTelemetry Telemetry { get; }
|
||||
|
||||
public static async Task<MetricsHost> StartAsync(
|
||||
PrometheusMetricsOptions options,
|
||||
MetricsAccessCredential? credential)
|
||||
{
|
||||
ManualRendezvousClock clock = new();
|
||||
SessionChangeJournal changes = new(new SessionChangeJournalOptions());
|
||||
InMemoryEphemeralRendezvousStore store = new(new(), clock, clock, changes);
|
||||
RendezvousTelemetry telemetry = new(store, changes);
|
||||
int port = ReserveTcpPort();
|
||||
string address = $"http://127.0.0.1:{port}";
|
||||
WebApplicationBuilder builder = WebApplication.CreateBuilder();
|
||||
builder.WebHost.UseUrls(address);
|
||||
builder.Services.AddSingleton(telemetry);
|
||||
WebApplication application = builder.Build();
|
||||
application.MapPrometheusMetricsEndpoint(options, credential);
|
||||
await application.StartAsync();
|
||||
return new(application, new HttpClient { BaseAddress = new Uri(address) }, telemetry);
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
Client.Dispose();
|
||||
await _application.StopAsync();
|
||||
await _application.DisposeAsync();
|
||||
// This test supplies an existing singleton instance, so the host does not own it.
|
||||
Telemetry.Dispose();
|
||||
}
|
||||
|
||||
private static int ReserveTcpPort()
|
||||
{
|
||||
using TcpListener listener = new(IPAddress.Loopback, 0);
|
||||
listener.Start();
|
||||
return ((IPEndPoint)listener.LocalEndpoint).Port;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import test from "node:test";
|
||||
|
||||
const root = new URL("../../", import.meta.url);
|
||||
const dashboardPath = new URL(
|
||||
"deploy/observability/grafana/dashboards/rendezvous-overview.json",
|
||||
root,
|
||||
);
|
||||
|
||||
test("dashboard is provisionable, broad, and uses privacy-safe bounded series", async () => {
|
||||
const dashboard = JSON.parse(await readFile(dashboardPath, "utf8"));
|
||||
assert.equal(dashboard.uid, "rendezvous-overview");
|
||||
assert.equal(dashboard.editable, false);
|
||||
assert.ok(dashboard.panels.length >= 20);
|
||||
assert.equal(new Set(dashboard.panels.map((panel) => panel.id)).size, dashboard.panels.length);
|
||||
assert.equal(new Set(dashboard.panels.map((panel) => panel.title)).size, dashboard.panels.length);
|
||||
|
||||
const expressions = dashboard.panels
|
||||
.flatMap((panel) => panel.targets ?? [])
|
||||
.map((target) => target.expr ?? "")
|
||||
.join("\n");
|
||||
for (const metric of [
|
||||
"rendezvous_http_requests_total",
|
||||
"rendezvous_http_duration_milliseconds_bucket",
|
||||
"rendezvous_udp_results_total",
|
||||
"rendezvous_udp_received_bytes_total",
|
||||
"rendezvous_limiter_drops_total",
|
||||
"rendezvous_connection_outcomes_total",
|
||||
"rendezvous_browser_sse_subscribers",
|
||||
"rendezvous_store_active_listings",
|
||||
"rendezvous_store_active_attempts",
|
||||
"rendezvous_signing_keys",
|
||||
"process_resident_memory_bytes",
|
||||
"process_open_file_descriptors",
|
||||
"dotnet_gc_heap_size_bytes",
|
||||
]) {
|
||||
assert.match(expressions, new RegExp(`\\b${metric}\\b`));
|
||||
}
|
||||
assert.doesNotMatch(
|
||||
expressions,
|
||||
/listing_?id|session_?id|player|subject|token|capability|endpoint|address|metadata|credential/i,
|
||||
);
|
||||
|
||||
for (const panel of dashboard.panels) {
|
||||
assert.ok(panel.title?.trim());
|
||||
assert.ok(panel.gridPos?.w > 0 && panel.gridPos?.h > 0);
|
||||
for (const target of panel.targets ?? []) {
|
||||
assert.equal(target.editorMode, "code");
|
||||
assert.ok(target.expr?.trim());
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("Compose overlay pins hardened services and keeps telemetry private", async () => {
|
||||
const compose = await readFile(new URL("deploy/observability/compose.yaml", root), "utf8");
|
||||
assert.match(compose, /prom\/prometheus:v3\.13\.1@sha256:[a-f0-9]{64}/);
|
||||
assert.match(compose, /grafana\/grafana:13\.1\.0@sha256:[a-f0-9]{64}/);
|
||||
assert.match(compose, /Rendezvous__Metrics__Enabled: "true"/);
|
||||
assert.match(compose, /rendezvous-metrics-token:\/run\/secrets\/rendezvous-metrics-token:ro/);
|
||||
assert.match(compose, /GF_AUTH_ANONYMOUS_ENABLED: "false"/);
|
||||
assert.match(compose, /GF_PLUGINS_PREINSTALL_DISABLED: "true"/);
|
||||
assert.match(compose, /"127\.0\.0\.1:3000:3000\/tcp"/);
|
||||
assert.doesNotMatch(compose, /9090:9090/);
|
||||
assert.ok((compose.match(/read_only: true/g) ?? []).length >= 2);
|
||||
assert.ok((compose.match(/no-new-privileges:true/g) ?? []).length >= 2);
|
||||
});
|
||||
|
||||
test("Prometheus and Grafana provisioning use server-side authenticated access", async () => {
|
||||
const prometheus = await readFile(
|
||||
new URL("deploy/observability/prometheus/prometheus.yml", root),
|
||||
"utf8",
|
||||
);
|
||||
const datasource = await readFile(
|
||||
new URL("deploy/observability/grafana/provisioning/datasources/prometheus.yaml", root),
|
||||
"utf8",
|
||||
);
|
||||
const provider = await readFile(
|
||||
new URL("deploy/observability/grafana/provisioning/dashboards/rendezvous.yaml", root),
|
||||
"utf8",
|
||||
);
|
||||
assert.match(prometheus, /bearer_token_file: \/run\/secrets\/rendezvous-metrics-token/);
|
||||
assert.match(prometheus, /- rendezvous:8080/);
|
||||
assert.match(datasource, /uid: rendezvous-prometheus/);
|
||||
assert.match(datasource, /access: proxy/);
|
||||
assert.match(datasource, /url: http:\/\/prometheus:9090/);
|
||||
assert.match(provider, /allowUiUpdates: false/);
|
||||
assert.match(provider, /path: \/var\/lib\/grafana\/dashboards/);
|
||||
});
|
||||
Reference in New Issue
Block a user