feat(observability): add diagnostic dashboards (#27)

tests/FinalFactory.Rendezvous.Tests/Observability/PrometheusMetricsTests.cs

@@ -0,0 +1,165 @@
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using FinalFactory.Rendezvous.Server.Browser;
+using FinalFactory.Rendezvous.Server.Observability;
+using FinalFactory.Rendezvous.Server.Provisioning;
+using FinalFactory.Rendezvous.Server.State;
+using FinalFactory.Rendezvous.Tests.State;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace FinalFactory.Rendezvous.Tests.Observability;
+
+[Collection(RendezvousTelemetryIsolation.Name)]
+public sealed class PrometheusMetricsTests
+{
+    private const string Token = "0123456789abcdef0123456789abcdef";
+
+    [Fact]
+    public void EnabledExporterRequiresAnExternalBoundedSecretReference()
+    {
+        Assert.Empty(new PrometheusMetricsOptions().Validate());
+        Assert.Empty(new PrometheusMetricsOptions
+        {
+            Enabled = true,
+            BearerTokenSecretReference = "file:/run/secrets/rendezvous-metrics-token",
+        }.Validate());
+        Assert.NotEmpty(new PrometheusMetricsOptions { Enabled = true }.Validate());
+        Assert.NotEmpty(new PrometheusMetricsOptions
+        {
+            Enabled = true,
+            BearerTokenSecretReference = "literal-secret",
+        }.Validate());
+        Assert.NotEmpty(new PrometheusMetricsOptions
+        {
+            Enabled = true,
+            BearerTokenSecretReference = "file:relative-token",
+        }.Validate());
+        Assert.NotEmpty(new PrometheusMetricsOptions
+        {
+            Enabled = true,
+            BearerTokenSecretReference = null!,
+        }.Validate());
+    }
+
+    [Fact]
+    public async Task ExporterIsDisabledByDefaultAndConcealsRejectedAuthentication()
+    {
+        await using MetricsHost disabled = await MetricsHost.StartAsync(
+            new PrometheusMetricsOptions(),
+            credential: null);
+        Assert.Equal(HttpStatusCode.NotFound, (await disabled.Client.GetAsync("metrics")).StatusCode);
+
+        PrometheusMetricsOptions options = EnabledOptions();
+        using MetricsAccessCredential credential = Credential(options);
+        await using MetricsHost enabled = await MetricsHost.StartAsync(options, credential);
+        Assert.Equal(HttpStatusCode.NotFound, (await enabled.Client.GetAsync("metrics")).StatusCode);
+        using HttpRequestMessage wrong = new(HttpMethod.Get, "metrics");
+        wrong.Headers.Authorization = new("Bearer", new string('x', 32));
+        Assert.Equal(HttpStatusCode.NotFound, (await enabled.Client.SendAsync(wrong)).StatusCode);
+    }
+
+    [Fact]
+    public async Task AuthenticatedExporterReturnsBoundedPrivacySafePrometheusFamilies()
+    {
+        PrometheusMetricsOptions options = EnabledOptions();
+        using MetricsAccessCredential credential = Credential(options);
+        await using MetricsHost host = await MetricsHost.StartAsync(options, credential);
+        host.Telemetry.RecordHttp("BrowseSessions", 200, 3.5);
+        host.Telemetry.RecordHttp("listing-id/canary", 500, 4.5);
+        host.Telemetry.RecordUdp("frozen", "Introduced", 1.25, 128, 2400);
+        host.Telemetry.RecordLimiterDrop("udp", "rate-or-concurrency");
+        host.Telemetry.RecordConnectionOutcome("Connected", "UnderOneSecond");
+        host.Telemetry.RecordPairingLatency(12.5);
+        using HttpRequestMessage request = new(HttpMethod.Get, "metrics");
+        request.Headers.Authorization = new("Bearer", Token);
+        using HttpResponseMessage response = await host.Client.SendAsync(request);
+        string body = await response.Content.ReadAsStringAsync();
+
+        Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+        Assert.Equal("text/plain", response.Content.Headers.ContentType?.MediaType);
+        Assert.Equal("no-store", response.Headers.CacheControl?.ToString());
+        Assert.Contains("rendezvous_http_requests_total{operation=\"BrowseSessions\",status_code=\"200\"} 1", body, StringComparison.Ordinal);
+        Assert.Contains("rendezvous_http_duration_milliseconds_bucket", body, StringComparison.Ordinal);
+        Assert.Contains("rendezvous_udp_received_bytes_total{operation=\"frozen\"} 128", body, StringComparison.Ordinal);
+        Assert.Contains("rendezvous_udp_response_budget_bytes_total{operation=\"frozen\"} 2400", body, StringComparison.Ordinal);
+        Assert.Contains("rendezvous_store_active_listings", body, StringComparison.Ordinal);
+        Assert.Contains("rendezvous_browser_sse_subscribers", body, StringComparison.Ordinal);
+        Assert.Contains("process_resident_memory_bytes", body, StringComparison.Ordinal);
+        Assert.Contains("dotnet_gc_heap_size_bytes", body, StringComparison.Ordinal);
+        Assert.Contains("operation=\"other\"", body, StringComparison.Ordinal);
+        Assert.DoesNotContain("listing-id", body, StringComparison.OrdinalIgnoreCase);
+        Assert.DoesNotContain(Token, body, StringComparison.Ordinal);
+    }
+
+    private static PrometheusMetricsOptions EnabledOptions() => new()
+    {
+        Enabled = true,
+        BearerTokenSecretReference = "file:/metrics-token",
+    };
+
+    private static MetricsAccessCredential Credential(PrometheusMetricsOptions options)
+    {
+        using DictionarySecretProvider secrets = new(new Dictionary<string, byte[]>
+        {
+            [options.BearerTokenSecretReference] = Encoding.ASCII.GetBytes(Token + "\n"),
+        });
+        Assert.True(MetricsAccessCredential.TryCreate(options, secrets, out MetricsAccessCredential? credential));
+        return Assert.IsType<MetricsAccessCredential>(credential);
+    }
+
+    private sealed class MetricsHost : IAsyncDisposable
+    {
+        private readonly WebApplication _application;
+
+        private MetricsHost(
+            WebApplication application,
+            HttpClient client,
+            RendezvousTelemetry telemetry)
+        {
+            _application = application;
+            Client = client;
+            Telemetry = telemetry;
+        }
+
+        public HttpClient Client { get; }
+        public RendezvousTelemetry Telemetry { get; }
+
+        public static async Task<MetricsHost> StartAsync(
+            PrometheusMetricsOptions options,
+            MetricsAccessCredential? credential)
+        {
+            ManualRendezvousClock clock = new();
+            SessionChangeJournal changes = new(new SessionChangeJournalOptions());
+            InMemoryEphemeralRendezvousStore store = new(new(), clock, clock, changes);
+            RendezvousTelemetry telemetry = new(store, changes);
+            int port = ReserveTcpPort();
+            string address = $"http://127.0.0.1:{port}";
+            WebApplicationBuilder builder = WebApplication.CreateBuilder();
+            builder.WebHost.UseUrls(address);
+            builder.Services.AddSingleton(telemetry);
+            WebApplication application = builder.Build();
+            application.MapPrometheusMetricsEndpoint(options, credential);
+            await application.StartAsync();
+            return new(application, new HttpClient { BaseAddress = new Uri(address) }, telemetry);
+        }
+
+        public async ValueTask DisposeAsync()
+        {
+            Client.Dispose();
+            await _application.StopAsync();
+            await _application.DisposeAsync();
+            // This test supplies an existing singleton instance, so the host does not own it.
+            Telemetry.Dispose();
+        }
+
+        private static int ReserveTcpPort()
+        {
+            using TcpListener listener = new(IPAddress.Loopback, 0);
+            listener.Start();
+            return ((IPEndPoint)listener.LocalEndpoint).Port;
+        }
+    }
+}