feat(observability): add diagnostic dashboards (#27)

2026-07-17 00:22:46 +02:00
parent 06c4ecf8f3
commit 99885f8c8c
30 changed files with 3173 additions and 18 deletions
@@ -0,0 +1,205 @@
+using System.Net;
+using System.Net.Sockets;
+using System.Text.Json;
+using FinalFactory.Rendezvous.Server.Diagnostics;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Options;
+
+namespace FinalFactory.Rendezvous.Tests.Diagnostics;
+
+public sealed class DiagnosticDashboardTests
+{
+    [Fact]
+    public void ConfigurationRequiresBoundedUniqueAllowListedScopes()
+    {
+        Assert.Empty(ValidOptions().Validate());
+        Assert.Empty(new DiagnosticDashboardOptions().Validate());
+
+        DiagnosticDashboardOptions invalid = ValidOptions() with
+        {
+            PollIntervalSeconds = 1,
+            MaximumRenderedSessions = 501,
+            Scopes =
+            [
+                new DiagnosticDashboardScope
+                {
+                    GameId = "INVALID",
+                    EnvironmentId = "smoke",
+                    ProtocolVersions = [0, 0],
+                    Regions = ["INVALID", "INVALID"],
+                },
+                new DiagnosticDashboardScope
+                {
+                    GameId = "INVALID",
+                    EnvironmentId = "smoke",
+                    ProtocolVersions = [1],
+                    Regions = ["local"],
+                },
+            ],
+        };
+
+        IReadOnlyList<string> errors = invalid.Validate();
+        Assert.Contains(errors, error => error.Contains("PollIntervalSeconds", StringComparison.Ordinal));
+        Assert.Contains(errors, error => error.Contains("MaximumRenderedSessions", StringComparison.Ordinal));
+        Assert.Contains(errors, error => error.Contains("invalid game", StringComparison.Ordinal));
+        Assert.Contains(errors, error => error.Contains("protocol versions", StringComparison.Ordinal));
+        Assert.Contains(errors, error => error.Contains("regions", StringComparison.Ordinal));
+        Assert.Contains(errors, error => error.Contains("duplicate", StringComparison.Ordinal));
+
+        DiagnosticDashboardOptions nullBound = ValidOptions() with
+        {
+            Scopes = null!,
+        };
+        Assert.Contains(
+            nullBound.Validate(),
+            error => error.Contains("Scopes must contain", StringComparison.Ordinal));
+        DiagnosticDashboardOptions nullScope = ValidOptions() with
+        {
+            Scopes = [null!],
+        };
+        Assert.Contains(
+            nullScope.Validate(),
+            error => error.Contains("null entries", StringComparison.Ordinal));
+        DiagnosticDashboardOptions nullCollections = ValidOptions() with
+        {
+            Scopes =
+            [
+                new DiagnosticDashboardScope
+                {
+                    GameId = null!,
+                    EnvironmentId = null!,
+                    ProtocolVersions = null!,
+                    Regions = null!,
+                },
+            ],
+        };
+        Assert.True(nullCollections.Validate().Count >= 3);
+    }
+
+    [Fact]
+    public async Task DisabledDashboardHasNoPublicAssetOrConfigurationSurface()
+    {
+        await using DashboardHost host = await DashboardHost.StartAsync(new());
+
+        Assert.Equal(HttpStatusCode.NotFound, (await host.Client.GetAsync("diagnostics/")).StatusCode);
+        Assert.Equal(HttpStatusCode.NotFound, (await host.Client.GetAsync("diagnostics/app.mjs")).StatusCode);
+        Assert.Equal(HttpStatusCode.NotFound, (await host.Client.GetAsync("diagnostics/config.json")).StatusCode);
+    }
+
+    [Fact]
+    public async Task EnabledDashboardServesOnlyHardenedSameOriginReadOnlyAssets()
+    {
+        await using DashboardHost host = await DashboardHost.StartAsync(ValidOptions());
+        using HttpRequestMessage request = new(HttpMethod.Get, "diagnostics/");
+        request.Headers.Add("Origin", "https://hostile.example");
+        using HttpResponseMessage response = await host.Client.SendAsync(request);
+        string html = await response.Content.ReadAsStringAsync();
+
+        Assert.True(response.IsSuccessStatusCode, html);
+        Assert.Equal("text/html", response.Content.Headers.ContentType?.MediaType);
+        Assert.Equal("DENY", Header(response, "X-Frame-Options"));
+        Assert.Equal("nosniff", Header(response, "X-Content-Type-Options"));
+        Assert.Equal("no-referrer", Header(response, "Referrer-Policy"));
+        Assert.Contains("default-src 'none'", Header(response, "Content-Security-Policy"), StringComparison.Ordinal);
+        Assert.Contains("connect-src 'self'", Header(response, "Content-Security-Policy"), StringComparison.Ordinal);
+        Assert.Contains("frame-ancestors 'none'", Header(response, "Content-Security-Policy"), StringComparison.Ordinal);
+        Assert.False(response.Headers.Contains("Access-Control-Allow-Origin"));
+        Assert.Contains("<html lang=\"en\">", html, StringComparison.Ordinal);
+        Assert.Contains("href=\"#main-content\"", html, StringComparison.Ordinal);
+        Assert.Contains("aria-live=\"polite\"", html, StringComparison.Ordinal);
+        Assert.Contains("type=\"module\" src=\"/diagnostics/app.mjs\"", html, StringComparison.Ordinal);
+        Assert.DoesNotContain("<script src=\"http", html, StringComparison.OrdinalIgnoreCase);
+        Assert.DoesNotContain("/v1/operator", html, StringComparison.OrdinalIgnoreCase);
+
+        using HttpResponseMessage scriptResponse = await host.Client.GetAsync("diagnostics/app.mjs");
+        string script = await scriptResponse.Content.ReadAsStringAsync();
+        Assert.Equal("text/javascript", scriptResponse.Content.Headers.ContentType?.MediaType);
+        Assert.DoesNotContain("/v1/operator", script, StringComparison.OrdinalIgnoreCase);
+        Assert.DoesNotContain("Authorization", script, StringComparison.OrdinalIgnoreCase);
+        Assert.DoesNotContain("innerHTML", script, StringComparison.Ordinal);
+    }
+
+    [Fact]
+    public async Task BrowserConfigurationContainsOnlyTheExplicitPublicAllowList()
+    {
+        await using DashboardHost host = await DashboardHost.StartAsync(ValidOptions());
+        using HttpResponseMessage response = await host.Client.GetAsync("diagnostics/config.json");
+        using JsonDocument document = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
+        JsonElement root = document.RootElement;
+
+        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+        Assert.Equal(1, root.GetProperty("contractVersion").GetInt32());
+        Assert.Equal(10, root.GetProperty("pollIntervalSeconds").GetInt32());
+        JsonElement scope = Assert.Single(root.GetProperty("scopes").EnumerateArray());
+        Assert.Equal("space-game", scope.GetProperty("gameId").GetString());
+        Assert.Equal("smoke", scope.GetProperty("environmentId").GetString());
+        Assert.Equal("public", scope.GetProperty("visibility").GetString());
+        string json = root.GetRawText();
+        Assert.DoesNotContain("http", json, StringComparison.OrdinalIgnoreCase);
+        Assert.DoesNotContain("credential", json, StringComparison.OrdinalIgnoreCase);
+        Assert.DoesNotContain("token", json, StringComparison.OrdinalIgnoreCase);
+        Assert.DoesNotContain("endpoint", json, StringComparison.OrdinalIgnoreCase);
+    }
+
+    private static string Header(HttpResponseMessage response, string name) =>
+        Assert.Single(response.Headers.GetValues(name));
+
+    private static DiagnosticDashboardOptions ValidOptions() => new()
+    {
+        Enabled = true,
+        PollIntervalSeconds = 10,
+        MaximumRenderedSessions = 100,
+        Scopes =
+        [
+            new DiagnosticDashboardScope
+            {
+                GameId = "space-game",
+                EnvironmentId = "smoke",
+                ProtocolVersions = [1, 2],
+                Regions = ["local"],
+            },
+        ],
+    };
+
+    private sealed class DashboardHost : IAsyncDisposable
+    {
+        private readonly WebApplication _application;
+
+        private DashboardHost(WebApplication application, HttpClient client)
+        {
+            _application = application;
+            Client = client;
+        }
+
+        public HttpClient Client { get; }
+
+        public static async Task<DashboardHost> StartAsync(DiagnosticDashboardOptions options)
+        {
+            int port = ReserveTcpPort();
+            string address = $"http://127.0.0.1:{port}";
+            WebApplicationBuilder builder = WebApplication.CreateBuilder();
+            builder.WebHost.UseUrls(address);
+            builder.Services.AddSingleton(Options.Create(options));
+            WebApplication application = builder.Build();
+            application.MapDiagnosticDashboardEndpoints();
+            await application.StartAsync();
+            return new(application, new HttpClient { BaseAddress = new Uri(address) });
+        }
+
+        public async ValueTask DisposeAsync()
+        {
+            Client.Dispose();
+            await _application.StopAsync();
+            await _application.DisposeAsync();
+        }
+
+        private static int ReserveTcpPort()
+        {
+            using TcpListener listener = new(IPAddress.Loopback, 0);
+            listener.Start();
+            return ((IPEndPoint)listener.LocalEndpoint).Port;
+        }
+    }
+}
@@ -0,0 +1,165 @@
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using FinalFactory.Rendezvous.Server.Browser;
+using FinalFactory.Rendezvous.Server.Observability;
+using FinalFactory.Rendezvous.Server.Provisioning;
+using FinalFactory.Rendezvous.Server.State;
+using FinalFactory.Rendezvous.Tests.State;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace FinalFactory.Rendezvous.Tests.Observability;
+
+[Collection(RendezvousTelemetryIsolation.Name)]
+public sealed class PrometheusMetricsTests
+{
+    private const string Token = "0123456789abcdef0123456789abcdef";
+
+    [Fact]
+    public void EnabledExporterRequiresAnExternalBoundedSecretReference()
+    {
+        Assert.Empty(new PrometheusMetricsOptions().Validate());
+        Assert.Empty(new PrometheusMetricsOptions
+        {
+            Enabled = true,
+            BearerTokenSecretReference = "file:/run/secrets/rendezvous-metrics-token",
+        }.Validate());
+        Assert.NotEmpty(new PrometheusMetricsOptions { Enabled = true }.Validate());
+        Assert.NotEmpty(new PrometheusMetricsOptions
+        {
+            Enabled = true,
+            BearerTokenSecretReference = "literal-secret",
+        }.Validate());
+        Assert.NotEmpty(new PrometheusMetricsOptions
+        {
+            Enabled = true,
+            BearerTokenSecretReference = "file:relative-token",
+        }.Validate());
+        Assert.NotEmpty(new PrometheusMetricsOptions
+        {
+            Enabled = true,
+            BearerTokenSecretReference = null!,
+        }.Validate());
+    }
+
+    [Fact]
+    public async Task ExporterIsDisabledByDefaultAndConcealsRejectedAuthentication()
+    {
+        await using MetricsHost disabled = await MetricsHost.StartAsync(
+            new PrometheusMetricsOptions(),
+            credential: null);
+        Assert.Equal(HttpStatusCode.NotFound, (await disabled.Client.GetAsync("metrics")).StatusCode);
+
+        PrometheusMetricsOptions options = EnabledOptions();
+        using MetricsAccessCredential credential = Credential(options);
+        await using MetricsHost enabled = await MetricsHost.StartAsync(options, credential);
+        Assert.Equal(HttpStatusCode.NotFound, (await enabled.Client.GetAsync("metrics")).StatusCode);
+        using HttpRequestMessage wrong = new(HttpMethod.Get, "metrics");
+        wrong.Headers.Authorization = new("Bearer", new string('x', 32));
+        Assert.Equal(HttpStatusCode.NotFound, (await enabled.Client.SendAsync(wrong)).StatusCode);
+    }
+
+    [Fact]
+    public async Task AuthenticatedExporterReturnsBoundedPrivacySafePrometheusFamilies()
+    {
+        PrometheusMetricsOptions options = EnabledOptions();
+        using MetricsAccessCredential credential = Credential(options);
+        await using MetricsHost host = await MetricsHost.StartAsync(options, credential);
+        host.Telemetry.RecordHttp("BrowseSessions", 200, 3.5);
+        host.Telemetry.RecordHttp("listing-id/canary", 500, 4.5);
+        host.Telemetry.RecordUdp("frozen", "Introduced", 1.25, 128, 2400);
+        host.Telemetry.RecordLimiterDrop("udp", "rate-or-concurrency");
+        host.Telemetry.RecordConnectionOutcome("Connected", "UnderOneSecond");
+        host.Telemetry.RecordPairingLatency(12.5);
+        using HttpRequestMessage request = new(HttpMethod.Get, "metrics");
+        request.Headers.Authorization = new("Bearer", Token);
+        using HttpResponseMessage response = await host.Client.SendAsync(request);
+        string body = await response.Content.ReadAsStringAsync();
+
+        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+        Assert.Equal("text/plain", response.Content.Headers.ContentType?.MediaType);
+        Assert.Equal("no-store", response.Headers.CacheControl?.ToString());
+        Assert.Contains("rendezvous_http_requests_total{operation=\"BrowseSessions\",status_code=\"200\"} 1", body, StringComparison.Ordinal);
+        Assert.Contains("rendezvous_http_duration_milliseconds_bucket", body, StringComparison.Ordinal);
+        Assert.Contains("rendezvous_udp_received_bytes_total{operation=\"frozen\"} 128", body, StringComparison.Ordinal);
+        Assert.Contains("rendezvous_udp_response_budget_bytes_total{operation=\"frozen\"} 2400", body, StringComparison.Ordinal);
+        Assert.Contains("rendezvous_store_active_listings", body, StringComparison.Ordinal);
+        Assert.Contains("rendezvous_browser_sse_subscribers", body, StringComparison.Ordinal);
+        Assert.Contains("process_resident_memory_bytes", body, StringComparison.Ordinal);
+        Assert.Contains("dotnet_gc_heap_size_bytes", body, StringComparison.Ordinal);
+        Assert.Contains("operation=\"other\"", body, StringComparison.Ordinal);
+        Assert.DoesNotContain("listing-id", body, StringComparison.OrdinalIgnoreCase);
+        Assert.DoesNotContain(Token, body, StringComparison.Ordinal);
+    }
+
+    private static PrometheusMetricsOptions EnabledOptions() => new()
+    {
+        Enabled = true,
+        BearerTokenSecretReference = "file:/metrics-token",
+    };
+
+    private static MetricsAccessCredential Credential(PrometheusMetricsOptions options)
+    {
+        using DictionarySecretProvider secrets = new(new Dictionary<string, byte[]>
+        {
+            [options.BearerTokenSecretReference] = Encoding.ASCII.GetBytes(Token + "\n"),
+        });
+        Assert.True(MetricsAccessCredential.TryCreate(options, secrets, out MetricsAccessCredential? credential));
+        return Assert.IsType<MetricsAccessCredential>(credential);
+    }
+
+    private sealed class MetricsHost : IAsyncDisposable
+    {
+        private readonly WebApplication _application;
+
+        private MetricsHost(
+            WebApplication application,
+            HttpClient client,
+            RendezvousTelemetry telemetry)
+        {
+            _application = application;
+            Client = client;
+            Telemetry = telemetry;
+        }
+
+        public HttpClient Client { get; }
+        public RendezvousTelemetry Telemetry { get; }
+
+        public static async Task<MetricsHost> StartAsync(
+            PrometheusMetricsOptions options,
+            MetricsAccessCredential? credential)
+        {
+            ManualRendezvousClock clock = new();
+            SessionChangeJournal changes = new(new SessionChangeJournalOptions());
+            InMemoryEphemeralRendezvousStore store = new(new(), clock, clock, changes);
+            RendezvousTelemetry telemetry = new(store, changes);
+            int port = ReserveTcpPort();
+            string address = $"http://127.0.0.1:{port}";
+            WebApplicationBuilder builder = WebApplication.CreateBuilder();
+            builder.WebHost.UseUrls(address);
+            builder.Services.AddSingleton(telemetry);
+            WebApplication application = builder.Build();
+            application.MapPrometheusMetricsEndpoint(options, credential);
+            await application.StartAsync();
+            return new(application, new HttpClient { BaseAddress = new Uri(address) }, telemetry);
+        }
+
+        public async ValueTask DisposeAsync()
+        {
+            Client.Dispose();
+            await _application.StopAsync();
+            await _application.DisposeAsync();
+            // This test supplies an existing singleton instance, so the host does not own it.
+            Telemetry.Dispose();
+        }
+
+        private static int ReserveTcpPort()
+        {
+            using TcpListener listener = new(IPAddress.Loopback, 0);
+            listener.Start();
+            return ((IPEndPoint)listener.LocalEndpoint).Port;
+        }
+    }
+}