feat(server): add observability and operator controls (#16)

src/FinalFactory.Rendezvous.Server/Observability/AuditOptions.cs

@@ -0,0 +1,14 @@
+using System.ComponentModel.DataAnnotations;
+
+namespace FinalFactory.Rendezvous.Server.Observability;
+
+internal sealed class AuditOptions
+{
+    public const string SectionName = "Rendezvous:Audit";
+
+    [Range(100, 100_000)]
+    public int MaxEntries { get; set; } = 10_000;
+
+    [Range(1, 30)]
+    public int RetentionDays { get; set; } = 30;
+}

src/FinalFactory.Rendezvous.Server/Observability/AuditTrail.cs

@@ -0,0 +1,134 @@
+using System.Security.Cryptography;
+using System.Text;
+using Microsoft.Extensions.Options;
+
+namespace FinalFactory.Rendezvous.Server.Observability;
+
+internal sealed partial class AuditTrail
+{
+    private readonly object _gate = new();
+    private readonly LinkedList<AuditEntry> _entries = [];
+    private readonly AuditOptions _options;
+    private readonly TimeProvider _timeProvider;
+    private readonly ILogger<AuditTrail> _logger;
+    private readonly RendezvousTelemetry _telemetry;
+
+    public AuditTrail(
+        IOptions<AuditOptions> options,
+        ILogger<AuditTrail> logger,
+        RendezvousTelemetry telemetry,
+        TimeProvider? timeProvider = null)
+    {
+        _options = options.Value;
+        _logger = logger;
+        _telemetry = telemetry;
+        _timeProvider = timeProvider ?? TimeProvider.System;
+    }
+
+    public void Record(
+        string actorSubject,
+        string action,
+        string result,
+        string targetKind,
+        string targetIdentifier,
+        string correlationId)
+    {
+        DateTimeOffset now = _timeProvider.GetUtcNow();
+        AuditEntry entry = new(
+            now,
+            Fingerprint(actorSubject),
+            action,
+            result,
+            targetKind,
+            Fingerprint(targetIdentifier),
+            correlationId);
+        lock (_gate)
+        {
+            PurgeExpired(now);
+
+            while (_entries.Count >= _options.MaxEntries)
+            {
+                _entries.RemoveFirst();
+            }
+
+            _entries.AddLast(entry);
+        }
+
+        _telemetry.RecordAudit(action, result);
+        LogOperatorAction(
+            _logger,
+            entry.Timestamp,
+            entry.ActorFingerprint,
+            action,
+            result,
+            targetKind,
+            entry.TargetFingerprint,
+            correlationId);
+    }
+
+    public IReadOnlyDictionary<string, long> GetAggregateCounts()
+    {
+        lock (_gate)
+        {
+            PurgeExpired(_timeProvider.GetUtcNow());
+            return _entries
+                .GroupBy(static entry => $"{entry.Action}:{entry.Result}", StringComparer.Ordinal)
+                .ToDictionary(
+                    static group => group.Key,
+                    static group => (long)group.Count(),
+                    StringComparer.Ordinal);
+        }
+    }
+
+    internal IReadOnlyList<AuditEntry> GetEntriesForTests()
+    {
+        lock (_gate)
+        {
+            PurgeExpired(_timeProvider.GetUtcNow());
+            return _entries.ToArray();
+        }
+    }
+
+    private void PurgeExpired(DateTimeOffset now)
+    {
+        DateTimeOffset oldest = now.AddDays(-_options.RetentionDays);
+        while (_entries.First is { Value.Timestamp: var timestamp }
+               && timestamp < oldest)
+        {
+            _entries.RemoveFirst();
+        }
+    }
+
+    private static string Fingerprint(string value)
+    {
+        byte[] digest = SHA256.HashData(Encoding.UTF8.GetBytes(value));
+        return Convert.ToHexString(digest.AsSpan(0, 12));
+    }
+
+    [LoggerMessage(
+        EventId = 100,
+        Level = LogLevel.Information,
+        Message = "Operator audit at {Timestamp}: actor {ActorFingerprint} action {Action} completed with {Result} for {TargetKind} target {TargetFingerprint}; correlation {CorrelationId}")]
+    private static partial void LogOperatorAction(
+        ILogger logger,
+        DateTimeOffset timestamp,
+        string actorFingerprint,
+        string action,
+        string result,
+        string targetKind,
+        string targetFingerprint,
+        string correlationId);
+}
+
+internal sealed record AuditEntry(
+    DateTimeOffset Timestamp,
+    string ActorFingerprint,
+    string Action,
+    string Result,
+    string TargetKind,
+    string TargetFingerprint,
+    string CorrelationId)
+{
+    public override string ToString() =>
+        $"[AuditEntry {Action}/{Result}; actor and target fingerprinted]";
+}

src/FinalFactory.Rendezvous.Server/Observability/HealthEndpoints.cs

@@ -0,0 +1,30 @@
+using FinalFactory.Rendezvous.Contracts;
+
+namespace FinalFactory.Rendezvous.Server.Observability;
+
+internal static class HealthEndpoints
+{
+    public static IEndpointRouteBuilder MapRendezvousHealthEndpoints(
+        this IEndpointRouteBuilder endpoints)
+    {
+        endpoints.MapGet(
+                "/health/live",
+                static () => Results.Ok(new HealthResponse { Status = "live" }))
+            .Produces<HealthResponse>()
+            .Produces<ApiError>(StatusCodes.Status429TooManyRequests)
+            .WithName("GetLiveness")
+            .WithTags("Health");
+        endpoints.MapGet(
+                "/health/ready",
+                static (RendezvousReadiness readiness) =>
+                    !readiness.GetSnapshot().IsReady
+                        ? Results.StatusCode(StatusCodes.Status503ServiceUnavailable)
+                        : Results.Ok(new HealthResponse { Status = "ready" }))
+            .Produces<HealthResponse>()
+            .Produces<ApiError>(StatusCodes.Status429TooManyRequests)
+            .Produces(StatusCodes.Status503ServiceUnavailable)
+            .WithName("GetReadiness")
+            .WithTags("Health");
+        return endpoints;
+    }
+}

src/FinalFactory.Rendezvous.Server/Observability/RendezvousReadiness.cs

@@ -0,0 +1,41 @@
+using FinalFactory.Rendezvous.Server.Provisioning;
+using FinalFactory.Rendezvous.Server.State;
+using FinalFactory.Rendezvous.Server.Transport;
+using Microsoft.Extensions.Options;
+
+namespace FinalFactory.Rendezvous.Server.Observability;
+
+internal sealed class RendezvousReadiness(
+    UdpMediatorService mediator,
+    ProvisioningReadiness provisioning,
+    IEphemeralRendezvousStore state,
+    IOptions<UdpMediatorOptions> udpOptions)
+{
+    public ReadinessSnapshot GetSnapshot()
+    {
+        bool ipv6Required = !string.IsNullOrWhiteSpace(udpOptions.Value.Ipv6ListenAddress);
+        return new ReadinessSnapshot(
+            HttpListenerReady: true,
+            UdpIpv4ListenerReady: mediator.LocalEndpoint is not null,
+            UdpIpv6ListenerReady: !ipv6Required || mediator.LocalIpv6Endpoint is not null,
+            ProvisioningReady: provisioning.IsReady,
+            StoreAvailable: state.IsAvailable,
+            Draining: state.IsDraining);
+    }
+}
+
+internal sealed record ReadinessSnapshot(
+    bool HttpListenerReady,
+    bool UdpIpv4ListenerReady,
+    bool UdpIpv6ListenerReady,
+    bool ProvisioningReady,
+    bool StoreAvailable,
+    bool Draining)
+{
+    public bool IsReady => HttpListenerReady
+        && UdpIpv4ListenerReady
+        && UdpIpv6ListenerReady
+        && ProvisioningReady
+        && StoreAvailable
+        && !Draining;
+}

src/FinalFactory.Rendezvous.Server/Observability/RendezvousTelemetry.cs

@@ -0,0 +1,126 @@
+using System.Diagnostics;
+using System.Diagnostics.Metrics;
+using FinalFactory.Rendezvous.Server.State;
+
+namespace FinalFactory.Rendezvous.Server.Observability;
+
+internal sealed class RendezvousTelemetry : IDisposable
+{
+    public const string MeterName = "FinalFactory.Rendezvous";
+    public const string ActivitySourceName = "FinalFactory.Rendezvous.Server";
+
+    private readonly InMemoryEphemeralRendezvousStore _store;
+    private readonly Meter _meter = new(MeterName, "1.0.0");
+    private readonly ActivitySource _activities = new(ActivitySourceName, "1.0.0");
+    private readonly Counter<long> _httpRequests;
+    private readonly Histogram<double> _httpDuration;
+    private readonly Counter<long> _udpResults;
+    private readonly Histogram<double> _udpDuration;
+    private readonly Counter<long> _limiterDrops;
+    private readonly Counter<long> _auditEvents;
+    private readonly Counter<long> _connectionOutcomes;
+    private readonly Counter<long> _operatorAuthentication;
+    private readonly Histogram<double> _pairingLatency;
+
+    public RendezvousTelemetry(InMemoryEphemeralRendezvousStore store)
+    {
+        _store = store;
+        _httpRequests = _meter.CreateCounter<long>("rendezvous.http.requests");
+        _httpDuration = _meter.CreateHistogram<double>(
+            "rendezvous.http.duration",
+            "ms");
+        _udpResults = _meter.CreateCounter<long>("rendezvous.udp.results");
+        _udpDuration = _meter.CreateHistogram<double>(
+            "rendezvous.udp.duration",
+            "ms");
+        _limiterDrops = _meter.CreateCounter<long>("rendezvous.limiter.drops");
+        _auditEvents = _meter.CreateCounter<long>("rendezvous.audit.events");
+        _connectionOutcomes = _meter.CreateCounter<long>("rendezvous.connection.outcomes");
+        _operatorAuthentication = _meter.CreateCounter<long>("rendezvous.operator.authentication");
+        _pairingLatency = _meter.CreateHistogram<double>(
+            "rendezvous.pairing.latency",
+            "ms");
+        _meter.CreateObservableGauge(
+            "rendezvous.store.active_listings",
+            () => _store.GetMetricsSnapshot().ActiveListings);
+        _meter.CreateObservableGauge(
+            "rendezvous.store.active_leases",
+            () => _store.GetMetricsSnapshot().ActiveListings);
+        _meter.CreateObservableGauge(
+            "rendezvous.store.active_attempts",
+            () => _store.GetMetricsSnapshot().ActiveJoinAttempts);
+        _meter.CreateObservableGauge(
+            "rendezvous.queue.depth",
+            () => _store.GetMetricsSnapshot().ActiveJoinAttempts);
+        _meter.CreateObservableGauge(
+            "rendezvous.store.replay_markers",
+            () => _store.GetMetricsSnapshot().ReplayMarkers);
+        _meter.CreateObservableGauge(
+            "rendezvous.store.available",
+            () => _store.GetMetricsSnapshot().IsAvailable ? 1 : 0);
+        _meter.CreateObservableCounter(
+            "rendezvous.store.expiry_churn",
+            () => _store.GetMetricsSnapshot().ExpiryChurn);
+    }
+
+    public Activity? StartActivity(string name, ActivityKind kind = ActivityKind.Internal) =>
+        _activities.StartActivity(name, kind);
+
+    public void RecordHttp(string operation, int statusCode, double elapsedMilliseconds)
+    {
+        TagList tags = new()
+        {
+            { "operation", operation },
+            { "status_code", statusCode },
+        };
+        _httpRequests.Add(1, tags);
+        _httpDuration.Record(elapsedMilliseconds, tags);
+    }
+
+    public void RecordUdp(string operation, string result, double elapsedMilliseconds)
+    {
+        TagList tags = new()
+        {
+            { "operation", operation },
+            { "result", result },
+        };
+        _udpResults.Add(1, tags);
+        _udpDuration.Record(elapsedMilliseconds, tags);
+    }
+
+    public void RecordLimiterDrop(string transport, string partition) =>
+        _limiterDrops.Add(1, new TagList
+        {
+            { "transport", transport },
+            { "partition", partition },
+        });
+
+    public void RecordAudit(string action, string result) =>
+        _auditEvents.Add(1, new TagList
+        {
+            { "action", action },
+            { "result", result },
+        });
+
+    public void RecordConnectionOutcome(string outcome, string elapsedBucket) =>
+        _connectionOutcomes.Add(1, new TagList
+        {
+            { "outcome", outcome },
+            { "elapsed_bucket", elapsedBucket },
+        });
+
+    public void RecordOperatorAuthentication(string result) =>
+        _operatorAuthentication.Add(1, new TagList
+        {
+            { "result", result },
+        });
+
+    public void RecordPairingLatency(double elapsedMilliseconds) =>
+        _pairingLatency.Record(elapsedMilliseconds);
+
+    public void Dispose()
+    {
+        _activities.Dispose();
+        _meter.Dispose();
+    }
+}

src/FinalFactory.Rendezvous.Server/Observability/TelemetryMiddleware.cs

@@ -0,0 +1,32 @@
+using System.Diagnostics;
+
+namespace FinalFactory.Rendezvous.Server.Observability;
+
+internal sealed class TelemetryMiddleware(
+    RequestDelegate next,
+    RendezvousTelemetry telemetry)
+{
+    public async Task InvokeAsync(HttpContext context)
+    {
+        string operation = context.GetEndpoint()?.Metadata.GetMetadata<IEndpointNameMetadata>()
+            ?.EndpointName ?? "Unmatched";
+        long started = Stopwatch.GetTimestamp();
+        using Activity? activity = telemetry.StartActivity(
+            $"HTTP {operation}",
+            ActivityKind.Server);
+        string correlationId = activity?.TraceId.ToString() ?? Guid.NewGuid().ToString("N");
+        context.Response.Headers["X-Rendezvous-Correlation-ID"] = correlationId;
+        activity?.SetTag("rendezvous.operation", operation);
+        try
+        {
+            await next(context).ConfigureAwait(false);
+        }
+        finally
+        {
+            telemetry.RecordHttp(
+                operation,
+                context.Response.StatusCode,
+                Stopwatch.GetElapsedTime(started).TotalMilliseconds);
+        }
+    }
+}