feat(observability): add diagnostic dashboards (#27)

2026-07-17 00:22:46 +02:00
parent 06c4ecf8f3
commit 99885f8c8c
30 changed files with 3173 additions and 18 deletions
@@ -0,0 +1,135 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import test from "node:test";
+import {
+  SessionProjection,
+  buildBrowseUrl,
+  buildStreamUrl,
+  chooseSnapshotTransport,
+  safeText,
+} from "../../src/FinalFactory.Rendezvous.Server/Diagnostics/Assets/app.mjs";
+
+const cursor = "rvs1.test-cursor";
+
+function session(id, name = `Host ${id}`) {
+  return {
+    contractVersion: 1,
+    listingId: `00000000-0000-0000-0000-${String(id).padStart(12, "0")}`,
+    gameId: "space-game",
+    environmentId: "smoke",
+    regionId: "local",
+    protocolVersion: 1,
+    buildVersion: "1.0.0",
+    displayName: name,
+    visibility: "public",
+    publisherTrustMode: "managedDedicated",
+    capacity: { currentPlayers: 1, maximumPlayers: 8 },
+    metadata: { mode: "online-coop" },
+  };
+}
+
+function event(kind, values = {}) {
+  return {
+    contractVersion: 1,
+    kind,
+    cursor: values.cursor ?? cursor,
+    session: values.session ?? null,
+    listingId: values.listingId ?? null,
+  };
+}
+
+test("snapshot and ordered deltas match the final public projection", () => {
+  const projection = new SessionProjection(10);
+  assert.equal(projection.replace([session(1)], cursor), "applied");
+  assert.equal(projection.apply(event("sessionUpsert", {
+    cursor: `${cursor}-2`,
+    session: session(2),
+  })), "applied");
+  assert.equal(projection.apply(event("sessionUpsert", {
+    cursor: `${cursor}-3`,
+    session: session(1, "Updated host"),
+  })), "applied");
+  assert.equal(projection.apply(event("sessionRemove", {
+    cursor: `${cursor}-4`,
+    listingId: session(2).listingId,
+  })), "applied");
+
+  assert.deepEqual(
+    projection.sessions().map((entry) => entry.session.displayName),
+    ["Updated host"],
+  );
+  assert.equal(projection.cursor, `${cursor}-4`);
+});
+
+test("reset, malformed events, and a partial snapshot fail closed", () => {
+  const projection = new SessionProjection(2);
+  assert.equal(projection.replace([session(1)], cursor, true), "applied");
+  assert.equal(projection.apply(event("sessionRemove", {
+    listingId: session(1).listingId,
+  })), "refresh");
+  assert.equal(projection.apply(event("reset")), "reset");
+  assert.equal(projection.apply({ kind: "sessionUpsert" }), "invalid");
+  assert.equal(projection.replace([session(1), session(2), session(3)], cursor), "overflow");
+});
+
+test("bursts coalesce by listing identity and memory stays bounded", () => {
+  const projection = new SessionProjection(2);
+  assert.equal(projection.replace([], cursor), "applied");
+  for (let index = 0; index < 1_000; index += 1) {
+    assert.equal(projection.apply(event("sessionUpsert", {
+      cursor: `${cursor}-${index}`,
+      session: session(1, `Host ${index}`),
+    })), "applied");
+  }
+  assert.equal(projection.sessions().length, 1);
+  assert.equal(projection.sessions()[0].session.displayName, "Host 999");
+  assert.equal(projection.apply(event("sessionUpsert", { session: session(2) })), "applied");
+  assert.equal(projection.apply(event("sessionUpsert", { session: session(3) })), "overflow");
+  assert.equal(projection.sessions().length, 2);
+});
+
+test("requests are fixed same-origin paths with an exact configured filter", () => {
+  const filter = {
+    gameId: "space-game",
+    environmentId: "smoke",
+    protocolVersion: 1,
+    regionId: "local",
+    excludeFull: true,
+  };
+  const browse = buildBrowseUrl(filter);
+  const stream = buildStreamUrl(filter, cursor);
+  assert.match(browse, /^\/v1\/sessions\?/u);
+  assert.match(stream, /^\/v1\/sessions\/stream\?/u);
+  assert.match(browse, /gameId=space-game/u);
+  assert.match(browse, /environmentId=smoke/u);
+  assert.match(stream, /streamCursor=rvs1.test-cursor/u);
+  assert.doesNotMatch(browse, /https?:/u);
+  assert.doesNotMatch(stream, /https?:/u);
+});
+
+test("snapshot transport preserves deliberate polling and bounds partial snapshots", () => {
+  assert.equal(chooseSnapshotTransport(false, false), "stream");
+  assert.equal(chooseSnapshotTransport(true, false), "boundedPolling");
+  assert.equal(chooseSnapshotTransport(false, true), "pollingOnly");
+  assert.equal(chooseSnapshotTransport(true, true), "pollingOnly");
+});
+
+test("hostile display data remains literal text and no unsafe DOM sink exists", async () => {
+  const payload = `<img src=x onerror=alert(1)><style>body{display:none}</style><a href=//evil>go</a>`;
+  assert.equal(safeText(payload, 200), payload);
+  const source = await readFile(new URL(
+    "../../src/FinalFactory.Rendezvous.Server/Diagnostics/Assets/app.mjs",
+    import.meta.url,
+  ), "utf8");
+  for (const forbidden of [
+    "innerHTML",
+    "outerHTML",
+    "insertAdjacentHTML",
+    "document.write",
+    "eval(",
+    "new Function",
+    "window.location",
+  ]) {
+    assert.equal(source.includes(forbidden), false, `unsafe browser sink: ${forbidden}`);
+  }
+});
@@ -0,0 +1,205 @@
+using System.Net;
+using System.Net.Sockets;
+using System.Text.Json;
+using FinalFactory.Rendezvous.Server.Diagnostics;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
+
+namespace FinalFactory.Rendezvous.Tests.Diagnostics;
+
+public sealed class DiagnosticDashboardTests
+{
+    [Fact]
+    public void ConfigurationRequiresBoundedUniqueAllowListedScopes()
+    {
+        Assert.Empty(ValidOptions().Validate());
+        Assert.Empty(new DiagnosticDashboardOptions().Validate());
+
+        DiagnosticDashboardOptions invalid = ValidOptions() with
+        {
+            PollIntervalSeconds = 1,
+            MaximumRenderedSessions = 501,
+            Scopes =
+            [
+                new DiagnosticDashboardScope
+                {
+                    GameId = "INVALID",
+                    EnvironmentId = "smoke",
+                    ProtocolVersions = [0, 0],
+                    Regions = ["INVALID", "INVALID"],
+                },
+                new DiagnosticDashboardScope
+                {
+                    GameId = "INVALID",
+                    EnvironmentId = "smoke",
+                    ProtocolVersions = [1],
+                    Regions = ["local"],
+                },
+            ],
+        };
+
+        IReadOnlyList<string> errors = invalid.Validate();
+        Assert.Contains(errors, error => error.Contains("PollIntervalSeconds", StringComparison.Ordinal));
+        Assert.Contains(errors, error => error.Contains("MaximumRenderedSessions", StringComparison.Ordinal));
+        Assert.Contains(errors, error => error.Contains("invalid game", StringComparison.Ordinal));
+        Assert.Contains(errors, error => error.Contains("protocol versions", StringComparison.Ordinal));
+        Assert.Contains(errors, error => error.Contains("regions", StringComparison.Ordinal));
+        Assert.Contains(errors, error => error.Contains("duplicate", StringComparison.Ordinal));
+
+        DiagnosticDashboardOptions nullBound = ValidOptions() with
+        {
+            Scopes = null!,
+        };
+        Assert.Contains(
+            nullBound.Validate(),
+            error => error.Contains("Scopes must contain", StringComparison.Ordinal));
+        DiagnosticDashboardOptions nullScope = ValidOptions() with
+        {
+            Scopes = [null!],
+        };
+        Assert.Contains(
+            nullScope.Validate(),
+            error => error.Contains("null entries", StringComparison.Ordinal));
+        DiagnosticDashboardOptions nullCollections = ValidOptions() with
+        {
+            Scopes =
+            [
+                new DiagnosticDashboardScope
+                {
+                    GameId = null!,
+                    EnvironmentId = null!,
+                    ProtocolVersions = null!,
+                    Regions = null!,
+                },
+            ],
+        };
+        Assert.True(nullCollections.Validate().Count >= 3);
+    }
+
+    [Fact]
+    public async Task DisabledDashboardHasNoPublicAssetOrConfigurationSurface()
+    {
+        await using DashboardHost host = await DashboardHost.StartAsync(new());
+
+        Assert.Equal(HttpStatusCode.NotFound, (await host.Client.GetAsync("diagnostics/")).StatusCode);
+        Assert.Equal(HttpStatusCode.NotFound, (await host.Client.GetAsync("diagnostics/app.mjs")).StatusCode);
+        Assert.Equal(HttpStatusCode.NotFound, (await host.Client.GetAsync("diagnostics/config.json")).StatusCode);
+    }
+
+    [Fact]
+    public async Task EnabledDashboardServesOnlyHardenedSameOriginReadOnlyAssets()
+    {
+        await using DashboardHost host = await DashboardHost.StartAsync(ValidOptions());
+        using HttpRequestMessage request = new(HttpMethod.Get, "diagnostics/");
+        request.Headers.Add("Origin", "https://hostile.example");
+        using HttpResponseMessage response = await host.Client.SendAsync(request);
+        string html = await response.Content.ReadAsStringAsync();
+
+        Assert.True(response.IsSuccessStatusCode, html);
+        Assert.Equal("text/html", response.Content.Headers.ContentType?.MediaType);
+        Assert.Equal("DENY", Header(response, "X-Frame-Options"));
+        Assert.Equal("nosniff", Header(response, "X-Content-Type-Options"));
+        Assert.Equal("no-referrer", Header(response, "Referrer-Policy"));
+        Assert.Contains("default-src 'none'", Header(response, "Content-Security-Policy"), StringComparison.Ordinal);
+        Assert.Contains("connect-src 'self'", Header(response, "Content-Security-Policy"), StringComparison.Ordinal);
+        Assert.Contains("frame-ancestors 'none'", Header(response, "Content-Security-Policy"), StringComparison.Ordinal);
+        Assert.False(response.Headers.Contains("Access-Control-Allow-Origin"));
+        Assert.Contains("<html lang=\"en\">", html, StringComparison.Ordinal);
+        Assert.Contains("href=\"#main-content\"", html, StringComparison.Ordinal);
+        Assert.Contains("aria-live=\"polite\"", html, StringComparison.Ordinal);
+        Assert.Contains("type=\"module\" src=\"/diagnostics/app.mjs\"", html, StringComparison.Ordinal);
+        Assert.DoesNotContain("<script src=\"http", html, StringComparison.OrdinalIgnoreCase);
+        Assert.DoesNotContain("/v1/operator", html, StringComparison.OrdinalIgnoreCase);
+
+        using HttpResponseMessage scriptResponse = await host.Client.GetAsync("diagnostics/app.mjs");
+        string script = await scriptResponse.Content.ReadAsStringAsync();
+        Assert.Equal("text/javascript", scriptResponse.Content.Headers.ContentType?.MediaType);
+        Assert.DoesNotContain("/v1/operator", script, StringComparison.OrdinalIgnoreCase);
+        Assert.DoesNotContain("Authorization", script, StringComparison.OrdinalIgnoreCase);
+        Assert.DoesNotContain("innerHTML", script, StringComparison.Ordinal);
+    }
+
+    [Fact]
+    public async Task BrowserConfigurationContainsOnlyTheExplicitPublicAllowList()
+    {
+        await using DashboardHost host = await DashboardHost.StartAsync(ValidOptions());
+        using HttpResponseMessage response = await host.Client.GetAsync("diagnostics/config.json");
+        using JsonDocument document = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
+        JsonElement root = document.RootElement;
+
+        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+        Assert.Equal(1, root.GetProperty("contractVersion").GetInt32());
+        Assert.Equal(10, root.GetProperty("pollIntervalSeconds").GetInt32());
+        JsonElement scope = Assert.Single(root.GetProperty("scopes").EnumerateArray());
+        Assert.Equal("space-game", scope.GetProperty("gameId").GetString());
+        Assert.Equal("smoke", scope.GetProperty("environmentId").GetString());
+        Assert.Equal("public", scope.GetProperty("visibility").GetString());
+        string json = root.GetRawText();
+        Assert.DoesNotContain("http", json, StringComparison.OrdinalIgnoreCase);
+        Assert.DoesNotContain("credential", json, StringComparison.OrdinalIgnoreCase);
+        Assert.DoesNotContain("token", json, StringComparison.OrdinalIgnoreCase);
+        Assert.DoesNotContain("endpoint", json, StringComparison.OrdinalIgnoreCase);
+    }
+
+    private static string Header(HttpResponseMessage response, string name) =>
+        Assert.Single(response.Headers.GetValues(name));
+
+    private static DiagnosticDashboardOptions ValidOptions() => new()
+    {
+        Enabled = true,
+        PollIntervalSeconds = 10,
+        MaximumRenderedSessions = 100,
+        Scopes =
+        [
+            new DiagnosticDashboardScope
+            {
+                GameId = "space-game",
+                EnvironmentId = "smoke",
+                ProtocolVersions = [1, 2],
+                Regions = ["local"],
+            },
+        ],
+    };
+
+    private sealed class DashboardHost : IAsyncDisposable
+    {
+        private readonly WebApplication _application;
+
+        private DashboardHost(WebApplication application, HttpClient client)
+        {
+            _application = application;
+            Client = client;
+        }
+
+        public HttpClient Client { get; }
+
+        public static async Task<DashboardHost> StartAsync(DiagnosticDashboardOptions options)
+        {
+            int port = ReserveTcpPort();
+            string address = $"http://127.0.0.1:{port}";
+            WebApplicationBuilder builder = WebApplication.CreateBuilder();
+            builder.WebHost.UseUrls(address);
+            builder.Services.AddSingleton(Options.Create(options));
+            WebApplication application = builder.Build();
+            application.MapDiagnosticDashboardEndpoints();
+            await application.StartAsync();
+            return new(application, new HttpClient { BaseAddress = new Uri(address) });
+        }
+
+        public async ValueTask DisposeAsync()
+        {
+            Client.Dispose();
+            await _application.StopAsync();
+            await _application.DisposeAsync();
+        }
+
+        private static int ReserveTcpPort()
+        {
+            using TcpListener listener = new(IPAddress.Loopback, 0);
+            listener.Start();
+            return ((IPEndPoint)listener.LocalEndpoint).Port;
+        }
+    }
+}
@@ -0,0 +1,165 @@
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using FinalFactory.Rendezvous.Server.Browser;
+using FinalFactory.Rendezvous.Server.Observability;
+using FinalFactory.Rendezvous.Server.Provisioning;
+using FinalFactory.Rendezvous.Server.State;
+using FinalFactory.Rendezvous.Tests.State;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace FinalFactory.Rendezvous.Tests.Observability;
+
+[Collection(RendezvousTelemetryIsolation.Name)]
+public sealed class PrometheusMetricsTests
+{
+    private const string Token = "0123456789abcdef0123456789abcdef";
+
+    [Fact]
+    public void EnabledExporterRequiresAnExternalBoundedSecretReference()
+    {
+        Assert.Empty(new PrometheusMetricsOptions().Validate());
+        Assert.Empty(new PrometheusMetricsOptions
+        {
+            Enabled = true,
+            BearerTokenSecretReference = "file:/run/secrets/rendezvous-metrics-token",
+        }.Validate());
+        Assert.NotEmpty(new PrometheusMetricsOptions { Enabled = true }.Validate());
+        Assert.NotEmpty(new PrometheusMetricsOptions
+        {
+            Enabled = true,
+            BearerTokenSecretReference = "literal-secret",
+        }.Validate());
+        Assert.NotEmpty(new PrometheusMetricsOptions
+        {
+            Enabled = true,
+            BearerTokenSecretReference = "file:relative-token",
+        }.Validate());
+        Assert.NotEmpty(new PrometheusMetricsOptions
+        {
+            Enabled = true,
+            BearerTokenSecretReference = null!,
+        }.Validate());
+    }
+
+    [Fact]
+    public async Task ExporterIsDisabledByDefaultAndConcealsRejectedAuthentication()
+    {
+        await using MetricsHost disabled = await MetricsHost.StartAsync(
+            new PrometheusMetricsOptions(),
+            credential: null);
+        Assert.Equal(HttpStatusCode.NotFound, (await disabled.Client.GetAsync("metrics")).StatusCode);
+
+        PrometheusMetricsOptions options = EnabledOptions();
+        using MetricsAccessCredential credential = Credential(options);
+        await using MetricsHost enabled = await MetricsHost.StartAsync(options, credential);
+        Assert.Equal(HttpStatusCode.NotFound, (await enabled.Client.GetAsync("metrics")).StatusCode);
+        using HttpRequestMessage wrong = new(HttpMethod.Get, "metrics");
+        wrong.Headers.Authorization = new("Bearer", new string('x', 32));
+        Assert.Equal(HttpStatusCode.NotFound, (await enabled.Client.SendAsync(wrong)).StatusCode);
+    }
+
+    [Fact]
+    public async Task AuthenticatedExporterReturnsBoundedPrivacySafePrometheusFamilies()
+    {
+        PrometheusMetricsOptions options = EnabledOptions();
+        using MetricsAccessCredential credential = Credential(options);
+        await using MetricsHost host = await MetricsHost.StartAsync(options, credential);
+        host.Telemetry.RecordHttp("BrowseSessions", 200, 3.5);
+        host.Telemetry.RecordHttp("listing-id/canary", 500, 4.5);
+        host.Telemetry.RecordUdp("frozen", "Introduced", 1.25, 128, 2400);
+        host.Telemetry.RecordLimiterDrop("udp", "rate-or-concurrency");
+        host.Telemetry.RecordConnectionOutcome("Connected", "UnderOneSecond");
+        host.Telemetry.RecordPairingLatency(12.5);
+        using HttpRequestMessage request = new(HttpMethod.Get, "metrics");
+        request.Headers.Authorization = new("Bearer", Token);
+        using HttpResponseMessage response = await host.Client.SendAsync(request);
+        string body = await response.Content.ReadAsStringAsync();
+
+        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+        Assert.Equal("text/plain", response.Content.Headers.ContentType?.MediaType);
+        Assert.Equal("no-store", response.Headers.CacheControl?.ToString());
+        Assert.Contains("rendezvous_http_requests_total{operation=\"BrowseSessions\",status_code=\"200\"} 1", body, StringComparison.Ordinal);
+        Assert.Contains("rendezvous_http_duration_milliseconds_bucket", body, StringComparison.Ordinal);
+        Assert.Contains("rendezvous_udp_received_bytes_total{operation=\"frozen\"} 128", body, StringComparison.Ordinal);
+        Assert.Contains("rendezvous_udp_response_budget_bytes_total{operation=\"frozen\"} 2400", body, StringComparison.Ordinal);
+        Assert.Contains("rendezvous_store_active_listings", body, StringComparison.Ordinal);
+        Assert.Contains("rendezvous_browser_sse_subscribers", body, StringComparison.Ordinal);
+        Assert.Contains("process_resident_memory_bytes", body, StringComparison.Ordinal);
+        Assert.Contains("dotnet_gc_heap_size_bytes", body, StringComparison.Ordinal);
+        Assert.Contains("operation=\"other\"", body, StringComparison.Ordinal);
+        Assert.DoesNotContain("listing-id", body, StringComparison.OrdinalIgnoreCase);
+        Assert.DoesNotContain(Token, body, StringComparison.Ordinal);
+    }
+
+    private static PrometheusMetricsOptions EnabledOptions() => new()
+    {
+        Enabled = true,
+        BearerTokenSecretReference = "file:/metrics-token",
+    };
+
+    private static MetricsAccessCredential Credential(PrometheusMetricsOptions options)
+    {
+        using DictionarySecretProvider secrets = new(new Dictionary<string, byte[]>
+        {
+            [options.BearerTokenSecretReference] = Encoding.ASCII.GetBytes(Token + "\n"),
+        });
+        Assert.True(MetricsAccessCredential.TryCreate(options, secrets, out MetricsAccessCredential? credential));
+        return Assert.IsType<MetricsAccessCredential>(credential);
+    }
+
+    private sealed class MetricsHost : IAsyncDisposable
+    {
+        private readonly WebApplication _application;
+
+        private MetricsHost(
+            WebApplication application,
+            HttpClient client,
+            RendezvousTelemetry telemetry)
+        {
+            _application = application;
+            Client = client;
+            Telemetry = telemetry;
+        }
+
+        public HttpClient Client { get; }
+        public RendezvousTelemetry Telemetry { get; }
+
+        public static async Task<MetricsHost> StartAsync(
+            PrometheusMetricsOptions options,
+            MetricsAccessCredential? credential)
+        {
+            ManualRendezvousClock clock = new();
+            SessionChangeJournal changes = new(new SessionChangeJournalOptions());
+            InMemoryEphemeralRendezvousStore store = new(new(), clock, clock, changes);
+            RendezvousTelemetry telemetry = new(store, changes);
+            int port = ReserveTcpPort();
+            string address = $"http://127.0.0.1:{port}";
+            WebApplicationBuilder builder = WebApplication.CreateBuilder();
+            builder.WebHost.UseUrls(address);
+            builder.Services.AddSingleton(telemetry);
+            WebApplication application = builder.Build();
+            application.MapPrometheusMetricsEndpoint(options, credential);
+            await application.StartAsync();
+            return new(application, new HttpClient { BaseAddress = new Uri(address) }, telemetry);
+        }
+
+        public async ValueTask DisposeAsync()
+        {
+            Client.Dispose();
+            await _application.StopAsync();
+            await _application.DisposeAsync();
+            // This test supplies an existing singleton instance, so the host does not own it.
+            Telemetry.Dispose();
+        }
+
+        private static int ReserveTcpPort()
+        {
+            using TcpListener listener = new(IPAddress.Loopback, 0);
+            listener.Start();
+            return ((IPEndPoint)listener.LocalEndpoint).Port;
+        }
+    }
+}
@@ -0,0 +1,89 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import test from "node:test";
+
+const root = new URL("../../", import.meta.url);
+const dashboardPath = new URL(
+  "deploy/observability/grafana/dashboards/rendezvous-overview.json",
+  root,
+);
+
+test("dashboard is provisionable, broad, and uses privacy-safe bounded series", async () => {
+  const dashboard = JSON.parse(await readFile(dashboardPath, "utf8"));
+  assert.equal(dashboard.uid, "rendezvous-overview");
+  assert.equal(dashboard.editable, false);
+  assert.ok(dashboard.panels.length >= 20);
+  assert.equal(new Set(dashboard.panels.map((panel) => panel.id)).size, dashboard.panels.length);
+  assert.equal(new Set(dashboard.panels.map((panel) => panel.title)).size, dashboard.panels.length);
+
+  const expressions = dashboard.panels
+    .flatMap((panel) => panel.targets ?? [])
+    .map((target) => target.expr ?? "")
+    .join("\n");
+  for (const metric of [
+    "rendezvous_http_requests_total",
+    "rendezvous_http_duration_milliseconds_bucket",
+    "rendezvous_udp_results_total",
+    "rendezvous_udp_received_bytes_total",
+    "rendezvous_limiter_drops_total",
+    "rendezvous_connection_outcomes_total",
+    "rendezvous_browser_sse_subscribers",
+    "rendezvous_store_active_listings",
+    "rendezvous_store_active_attempts",
+    "rendezvous_signing_keys",
+    "process_resident_memory_bytes",
+    "process_open_file_descriptors",
+    "dotnet_gc_heap_size_bytes",
+  ]) {
+    assert.match(expressions, new RegExp(`\\b${metric}\\b`));
+  }
+  assert.doesNotMatch(
+    expressions,
+    /listing_?id|session_?id|player|subject|token|capability|endpoint|address|metadata|credential/i,
+  );
+
+  for (const panel of dashboard.panels) {
+    assert.ok(panel.title?.trim());
+    assert.ok(panel.gridPos?.w > 0 && panel.gridPos?.h > 0);
+    for (const target of panel.targets ?? []) {
+      assert.equal(target.editorMode, "code");
+      assert.ok(target.expr?.trim());
+    }
+  }
+});
+
+test("Compose overlay pins hardened services and keeps telemetry private", async () => {
+  const compose = await readFile(new URL("deploy/observability/compose.yaml", root), "utf8");
+  assert.match(compose, /prom\/prometheus:v3\.13\.1@sha256:[a-f0-9]{64}/);
+  assert.match(compose, /grafana\/grafana:13\.1\.0@sha256:[a-f0-9]{64}/);
+  assert.match(compose, /Rendezvous__Metrics__Enabled: "true"/);
+  assert.match(compose, /rendezvous-metrics-token:\/run\/secrets\/rendezvous-metrics-token:ro/);
+  assert.match(compose, /GF_AUTH_ANONYMOUS_ENABLED: "false"/);
+  assert.match(compose, /GF_PLUGINS_PREINSTALL_DISABLED: "true"/);
+  assert.match(compose, /"127\.0\.0\.1:3000:3000\/tcp"/);
+  assert.doesNotMatch(compose, /9090:9090/);
+  assert.ok((compose.match(/read_only: true/g) ?? []).length >= 2);
+  assert.ok((compose.match(/no-new-privileges:true/g) ?? []).length >= 2);
+});
+
+test("Prometheus and Grafana provisioning use server-side authenticated access", async () => {
+  const prometheus = await readFile(
+    new URL("deploy/observability/prometheus/prometheus.yml", root),
+    "utf8",
+  );
+  const datasource = await readFile(
+    new URL("deploy/observability/grafana/provisioning/datasources/prometheus.yaml", root),
+    "utf8",
+  );
+  const provider = await readFile(
+    new URL("deploy/observability/grafana/provisioning/dashboards/rendezvous.yaml", root),
+    "utf8",
+  );
+  assert.match(prometheus, /bearer_token_file: \/run\/secrets\/rendezvous-metrics-token/);
+  assert.match(prometheus, /- rendezvous:8080/);
+  assert.match(datasource, /uid: rendezvous-prometheus/);
+  assert.match(datasource, /access: proxy/);
+  assert.match(datasource, /url: http:\/\/prometheus:9090/);
+  assert.match(provider, /allowUiUpdates: false/);
+  assert.match(provider, /path: \/var\/lib\/grafana\/dashboards/);
+});