feat(server): add observability and operator controls (#16)

tests/FinalFactory.Rendezvous.Tests/Observability/ObservabilityTests.cs

@@ -0,0 +1,253 @@
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Diagnostics.Metrics;
+using System.Net;
+using FinalFactory.Rendezvous.Server.Observability;
+using FinalFactory.Rendezvous.Server.State;
+using FinalFactory.Rendezvous.Server.Transport;
+using FinalFactory.Rendezvous.Tests.State;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace FinalFactory.Rendezvous.Tests.Observability;
+
+[CollectionDefinition(RendezvousTelemetryIsolation.Name, DisableParallelization = true)]
+public sealed class RendezvousTelemetryIsolation
+{
+    public const string Name = "Rendezvous telemetry";
+}
+
+[Collection(RendezvousTelemetryIsolation.Name)]
+public sealed class ObservabilityTests
+{
+    private static readonly HashSet<string> AllowedTagKeys =
+    [
+        "operation",
+        "status_code",
+        "result",
+        "transport",
+        "partition",
+        "action",
+        "outcome",
+        "elapsed_bucket",
+    ];
+
+    [Fact]
+    public void MetricsAndTracesUseBoundedDimensionsWithoutSensitiveValues()
+    {
+        EphemeralStateFixture fixture = new();
+        using RendezvousTelemetry telemetry = new(fixture.Store);
+        List<Measurement> measurements = [];
+        using MeterListener meterListener = new();
+        meterListener.InstrumentPublished = (instrument, listener) =>
+        {
+            if (instrument.Meter.Name == RendezvousTelemetry.MeterName)
+            {
+                listener.EnableMeasurementEvents(instrument);
+            }
+        };
+        meterListener.SetMeasurementEventCallback<long>((instrument, value, tags, _) =>
+            measurements.Add(new(instrument.Name, value, Tags(tags))));
+        meterListener.SetMeasurementEventCallback<int>((instrument, value, tags, _) =>
+            measurements.Add(new(instrument.Name, value, Tags(tags))));
+        meterListener.SetMeasurementEventCallback<double>((instrument, value, tags, _) =>
+            measurements.Add(new(instrument.Name, value, Tags(tags))));
+        meterListener.Start();
+
+        Activity? observed = null;
+        List<string> activityData = [];
+        using ActivityListener activityListener = new()
+        {
+            ShouldListenTo = source => source.Name == RendezvousTelemetry.ActivitySourceName,
+            Sample = static (ref ActivityCreationOptions<ActivityContext> _) =>
+                ActivitySamplingResult.AllData,
+            ActivityStopped = activity =>
+            {
+                observed = activity;
+                activityData.Add(activity.DisplayName);
+                activityData.AddRange(activity.TagObjects.Select(static tag => $"{tag.Key}={tag.Value}"));
+            },
+        };
+        ActivitySource.AddActivityListener(activityListener);
+
+        const string secret = "secret-player-token-canary";
+        using (telemetry.StartActivity("HTTP GetOperatorStatus", ActivityKind.Server))
+        {
+            telemetry.RecordHttp("GetOperatorStatus", 200, 3.5);
+            telemetry.RecordUdp("frozen", "Introduced", 1.25);
+            telemetry.RecordLimiterDrop("udp", "rate-or-concurrency");
+            telemetry.RecordAudit("revoke-listing", "succeeded");
+            telemetry.RecordConnectionOutcome("Connected", "UnderOneSecond");
+            telemetry.RecordOperatorAuthentication("accepted");
+            telemetry.RecordPairingLatency(12.5);
+        }
+        NatMediationProcessor processor = new(null!, null!, null!, telemetry: telemetry);
+        Assert.Equal(
+            NatMediationResult.Dropped,
+            processor.ProcessRequest(
+                new IPEndPoint(IPAddress.Parse("10.0.0.8"), 9000),
+                new IPEndPoint(IPAddress.Parse("203.0.113.8"), 50000),
+                secret,
+                NoopIntroductionSink.Instance));
+
+        meterListener.RecordObservableInstruments();
+        Assert.NotNull(observed);
+        string flattened = string.Join('|', measurements.Select(static item => item.ToString()));
+        Assert.DoesNotContain(secret, flattened, StringComparison.Ordinal);
+        Assert.DoesNotContain(secret, string.Join('|', activityData), StringComparison.Ordinal);
+        Assert.Contains(measurements, static item => item.Name == "rendezvous.http.requests");
+        Assert.Contains(measurements, static item => item.Name == "rendezvous.udp.results");
+        Assert.Contains(measurements, static item => item.Name == "rendezvous.limiter.drops");
+        Assert.Contains(measurements, static item => item.Name == "rendezvous.queue.depth");
+        Assert.Contains(measurements, static item => item.Name == "rendezvous.store.active_leases");
+        Assert.Contains(measurements, static item => item.Name == "rendezvous.store.expiry_churn");
+        Assert.Contains(measurements, static item => item.Name == "rendezvous.pairing.latency");
+        Assert.All(measurements.SelectMany(static item => item.Tags), static tag =>
+            Assert.Contains(tag.Key, AllowedTagKeys));
+    }
+
+    [Fact]
+    public void MetricScrapeExpiresIdleStateAndReportsExpiryChurn()
+    {
+        EphemeralStateFixture fixture = new();
+        Assert.True(fixture.Store.CreateListing(fixture.ListingCommand()).Succeeded);
+        using RendezvousTelemetry telemetry = new(fixture.Store);
+        List<Measurement> measurements = [];
+        using MeterListener listener = new();
+        listener.InstrumentPublished = (instrument, meterListener) =>
+        {
+            if (instrument.Meter.Name == RendezvousTelemetry.MeterName)
+            {
+                meterListener.EnableMeasurementEvents(instrument);
+            }
+        };
+        listener.SetMeasurementEventCallback<long>((instrument, value, tags, _) =>
+            measurements.Add(new(instrument.Name, value, Tags(tags))));
+        listener.SetMeasurementEventCallback<int>((instrument, value, tags, _) =>
+            measurements.Add(new(instrument.Name, value, Tags(tags))));
+        listener.Start();
+
+        listener.RecordObservableInstruments();
+        Assert.Equal(
+            1,
+            Assert.Single(measurements, static item => item.Name == "rendezvous.store.active_leases").Value);
+
+        fixture.Clock.Advance(TimeSpan.FromSeconds(61));
+        measurements.Clear();
+        listener.RecordObservableInstruments();
+        Assert.Equal(
+            0,
+            Assert.Single(measurements, static item => item.Name == "rendezvous.store.active_leases").Value);
+        Assert.True(Assert.Single(
+            measurements,
+            static item => item.Name == "rendezvous.store.expiry_churn").Value >= 1);
+    }
+
+    [Fact]
+    public void AuditTrailFingerprintsIdentifiersEnforcesRetentionAndBoundsCapacity()
+    {
+        EphemeralStateFixture fixture = new();
+        using RendezvousTelemetry telemetry = new(fixture.Store);
+        CapturingLogger<AuditTrail> logger = new();
+        ManualTimeProvider time = new(new DateTimeOffset(2026, 7, 16, 0, 0, 0, TimeSpan.Zero));
+        AuditTrail audit = new(
+            Options.Create(new AuditOptions { MaxEntries = 100, RetentionDays = 1 }),
+            logger,
+            telemetry,
+            time);
+        const string actor = "operator-secret-subject";
+        const string target = "player-secret-subject";
+
+        for (int index = 0; index < 101; index++)
+        {
+            audit.Record(actor, "revoke-principal", "succeeded", "principal", target, "safe-correlation");
+        }
+
+        IReadOnlyList<AuditEntry> bounded = audit.GetEntriesForTests();
+        Assert.Equal(100, bounded.Count);
+        Assert.All(bounded, entry =>
+        {
+            Assert.DoesNotContain(actor, entry.ToString(), StringComparison.Ordinal);
+            Assert.DoesNotContain(target, entry.ToString(), StringComparison.Ordinal);
+            Assert.NotEqual(actor, entry.ActorFingerprint);
+            Assert.NotEqual(target, entry.TargetFingerprint);
+        });
+        Assert.DoesNotContain(actor, string.Join('|', logger.Messages), StringComparison.Ordinal);
+        Assert.DoesNotContain(target, string.Join('|', logger.Messages), StringComparison.Ordinal);
+
+        time.Advance(TimeSpan.FromDays(2));
+        Assert.Empty(audit.GetEntriesForTests());
+        Assert.Empty(audit.GetAggregateCounts());
+        audit.Record(actor, "inspect-status", "succeeded", "service", "rendezvous", "safe-correlation");
+        Assert.Single(audit.GetEntriesForTests());
+        Assert.Equal(1, audit.GetAggregateCounts()["inspect-status:succeeded"]);
+    }
+
+    private static KeyValuePair<string, object?>[] Tags(
+        ReadOnlySpan<KeyValuePair<string, object?>> tags) => tags.ToArray();
+
+    private sealed record Measurement(
+        string Name,
+        double Value,
+        KeyValuePair<string, object?>[] Tags);
+
+    private sealed class ManualTimeProvider(DateTimeOffset now) : TimeProvider
+    {
+        private DateTimeOffset _now = now;
+        public override DateTimeOffset GetUtcNow() => _now;
+        public void Advance(TimeSpan duration) => _now += duration;
+    }
+
+    private sealed class NoopIntroductionSink : INatIntroductionSink
+    {
+        public static NoopIntroductionSink Instance { get; } = new();
+        public void Introduce(NatIntroductionPlan plan)
+        {
+        }
+    }
+}
+
+internal sealed class CapturingLogger<T> : ILogger<T>
+{
+    public List<string> Messages { get; } = [];
+
+    public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
+    public bool IsEnabled(LogLevel logLevel) => true;
+
+    public void Log<TState>(
+        LogLevel logLevel,
+        EventId eventId,
+        TState state,
+        Exception? exception,
+        Func<TState, Exception?, string> formatter) => Messages.Add(formatter(state, exception));
+}
+
+internal sealed class CapturingLoggerProvider : ILoggerProvider
+{
+    public ConcurrentQueue<string> Messages { get; } = new();
+
+    public ILogger CreateLogger(string categoryName) => new Sink(Messages);
+
+    public void Dispose() => GC.SuppressFinalize(this);
+
+    private sealed class Sink(ConcurrentQueue<string> messages) : ILogger
+    {
+        public IDisposable BeginScope<TState>(TState state) where TState : notnull => Scope.Instance;
+        public bool IsEnabled(LogLevel logLevel) => true;
+
+        public void Log<TState>(
+            LogLevel logLevel,
+            EventId eventId,
+            TState state,
+            Exception? exception,
+            Func<TState, Exception?, string> formatter) => messages.Enqueue(formatter(state, exception));
+    }
+
+    private sealed class Scope : IDisposable
+    {
+        public static Scope Instance { get; } = new();
+        public void Dispose()
+        {
+        }
+    }
+}