feat(server): add observability and operator controls (#16)

src/FinalFactory.Rendezvous.Server/Abuse/AbuseProtectionOptions.cs

@@ -20,6 +20,8 @@ internal sealed class AbuseProtectionOptions
 
     public string[] TrustedProxyAddresses { get; set; } = [];
 
+    public string[] OperatorAllowedAddresses { get; set; } = [];
+
     [Range(1, 100_000)]
     public int HealthGlobalRequestsPerWindow { get; set; } = 1_000;
 
@@ -32,6 +34,18 @@ internal sealed class AbuseProtectionOptions
     [Range(1, 1_000)]
     public int HealthIpPrefixConcurrency { get; set; } = 8;
 
+    [Range(1, 100_000)]
+    public int OperatorGlobalRequestsPerWindow { get; set; } = 1_000;
+
+    [Range(1, 10_000)]
+    public int OperatorGlobalConcurrency { get; set; } = 32;
+
+    [Range(1, 100_000)]
+    public int OperatorIpPrefixRequestsPerWindow { get; set; } = 120;
+
+    [Range(1, 1_000)]
+    public int OperatorIpPrefixConcurrency { get; set; } = 8;
+
     [Range(1, 1_000_000)]
     public int HttpGlobalRequestsPerWindow { get; set; } = 20_000;
 

src/FinalFactory.Rendezvous.Server/Abuse/AbuseProtectionService.cs

@@ -2,6 +2,7 @@ using System.Buffers;
 using System.Net;
 using System.Security.Cryptography;
 using System.Text;
+using FinalFactory.Rendezvous.Server.Observability;
 using Microsoft.Extensions.Options;
 
 namespace FinalFactory.Rendezvous.Server.Abuse;
@@ -12,13 +13,23 @@ internal sealed class AbuseProtectionService
     private readonly TimeProvider _timeProvider;
     private readonly TrackerState _httpTracker;
     private readonly TrackerState _udpTracker;
+    private readonly RendezvousTelemetry? _telemetry;
+    private readonly HashSet<string> _operatorAllowedAddresses;
 
     public AbuseProtectionService(
         IOptions<AbuseProtectionOptions> options,
-        TimeProvider? timeProvider = null)
+        TimeProvider? timeProvider = null,
+        RendezvousTelemetry? telemetry = null)
     {
         _options = options.Value;
         _timeProvider = timeProvider ?? TimeProvider.System;
+        _telemetry = telemetry;
+        _operatorAllowedAddresses = options.Value.OperatorAllowedAddresses
+            .Select(static value => IPAddress.TryParse(value, out IPAddress? address)
+                ? NormalizeAddress(address).ToString()
+                : string.Empty)
+            .Where(static value => value.Length > 0)
+            .ToHashSet(StringComparer.Ordinal);
         DateTimeOffset now = _timeProvider.GetUtcNow();
         _httpTracker = new(now);
         _udpTracker = new(now);
@@ -87,6 +98,35 @@ internal sealed class AbuseProtectionService
             out retryAfterSeconds);
     }
 
+    public bool IsOperatorSourceAllowed(IPAddress? remoteAddress) =>
+        remoteAddress is not null
+        && _operatorAllowedAddresses.Contains(NormalizeAddress(remoteAddress).ToString());
+
+    public bool TryAcquireOperatorIngress(
+        IPAddress? remoteAddress,
+        out AbuseLease? lease,
+        out int retryAfterSeconds)
+    {
+        string prefix = GetNetworkPrefix(remoteAddress);
+        RateDimension[] rates =
+        [
+            new("operator:rate:global", _options.OperatorGlobalRequestsPerWindow),
+            new($"operator:rate:ip:{prefix}", _options.OperatorIpPrefixRequestsPerWindow),
+        ];
+        RateDimension[] concurrency =
+        [
+            new("operator:concurrency:global", _options.OperatorGlobalConcurrency),
+            new($"operator:concurrency:ip:{prefix}", _options.OperatorIpPrefixConcurrency),
+        ];
+        return TryAcquire(
+            rates,
+            concurrency,
+            TrackerDomain.Http,
+            true,
+            out lease,
+            out retryAfterSeconds);
+    }
+
     public bool TryAcquireHttpIdentity(
         string operation,
         string? tenant,
@@ -266,67 +306,96 @@ internal sealed class AbuseProtectionService
         out int retryAfterSeconds)
     {
         TrackerState tracker = domain == TrackerDomain.Udp ? _udpTracker : _httpTracker;
+        bool accepted;
         lock (tracker.Gate)
         {
-            DateTimeOffset now = _timeProvider.GetUtcNow();
-            TimeSpan window = TimeSpan.FromSeconds(_options.WindowSeconds);
-            if (now - tracker.WindowStartedAt >= window || now < tracker.WindowStartedAt)
-            {
-                tracker.WindowCounts.Clear();
-                tracker.WindowStartedAt = now;
-            }
+            accepted = TryAcquireLocked(
+                tracker,
+                rates,
+                concurrency,
+                domain,
+                canUseCriticalReserve,
+                out lease,
+                out retryAfterSeconds);
+        }
 
-            retryAfterSeconds = Math.Max(
-                1,
-                (int)Math.Ceiling((window - (now - tracker.WindowStartedAt)).TotalSeconds));
-            int stagedNewKeys = 0;
-            int partitionLimit = domain == TrackerDomain.Udp
-                ? _options.UdpTrackedKeyLimit
-                : _options.MaxTrackedKeys - _options.UdpTrackedKeyLimit;
-            int maxTrackedKeys = domain == TrackerDomain.Udp || canUseCriticalReserve
-                ? partitionLimit
-                : partitionLimit - _options.CriticalTrackedKeyReserve;
-            if (!CanAcquireAll(
-                    tracker,
-                    tracker.WindowCounts,
-                    rates,
-                    maxTrackedKeys,
-                    ref stagedNewKeys)
-                || !CanAcquireAll(
-                    tracker,
-                    tracker.ConcurrencyCounts,
-                    concurrency,
-                    maxTrackedKeys,
-                    ref stagedNewKeys))
-            {
-                lease = null;
-                return false;
-            }
+        if (!accepted)
+        {
+            _telemetry?.RecordLimiterDrop(
+                domain == TrackerDomain.Udp ? "udp" : "http",
+                "rate-or-concurrency");
+        }
 
-            foreach (RateDimension dimension in rates)
-            {
-                tracker.WindowCounts[dimension.Key] =
-                    tracker.WindowCounts.GetValueOrDefault(dimension.Key) + 1;
-            }
+        return accepted;
+    }
 
-            if (concurrency.IsEmpty)
-            {
-                lease = null;
-                return true;
-            }
+    private bool TryAcquireLocked(
+        TrackerState tracker,
+        ReadOnlySpan<RateDimension> rates,
+        ReadOnlySpan<RateDimension> concurrency,
+        TrackerDomain domain,
+        bool canUseCriticalReserve,
+        out AbuseLease? lease,
+        out int retryAfterSeconds)
+    {
+        DateTimeOffset now = _timeProvider.GetUtcNow();
+        TimeSpan window = TimeSpan.FromSeconds(_options.WindowSeconds);
+        if (now - tracker.WindowStartedAt >= window || now < tracker.WindowStartedAt)
+        {
+            tracker.WindowCounts.Clear();
+            tracker.WindowStartedAt = now;
+        }
 
-            string[] acquiredConcurrency = new string[concurrency.Length];
-            for (int index = 0; index < concurrency.Length; index++)
-            {
-                RateDimension dimension = concurrency[index];
-                tracker.ConcurrencyCounts[dimension.Key] =
-                    tracker.ConcurrencyCounts.GetValueOrDefault(dimension.Key) + 1;
-                acquiredConcurrency[index] = dimension.Key;
-            }
+        retryAfterSeconds = Math.Max(
+            1,
+            (int)Math.Ceiling((window - (now - tracker.WindowStartedAt)).TotalSeconds));
+        int stagedNewKeys = 0;
+        int partitionLimit = domain == TrackerDomain.Udp
+            ? _options.UdpTrackedKeyLimit
+            : _options.MaxTrackedKeys - _options.UdpTrackedKeyLimit;
+        int maxTrackedKeys = domain == TrackerDomain.Udp || canUseCriticalReserve
+            ? partitionLimit
+            : partitionLimit - _options.CriticalTrackedKeyReserve;
+        if (!CanAcquireAll(
+                tracker,
+                tracker.WindowCounts,
+                rates,
+                maxTrackedKeys,
+                ref stagedNewKeys)
+            || !CanAcquireAll(
+                tracker,
+                tracker.ConcurrencyCounts,
+                concurrency,
+                maxTrackedKeys,
+                ref stagedNewKeys))
+        {
+            lease = null;
+            return false;
+        }
 
-            lease = new AbuseLease(this, tracker, acquiredConcurrency);
+        foreach (RateDimension dimension in rates)
+        {
+            tracker.WindowCounts[dimension.Key] =
+                tracker.WindowCounts.GetValueOrDefault(dimension.Key) + 1;
+        }
+
+        if (concurrency.IsEmpty)
+        {
+            lease = null;
             return true;
         }
+
+        string[] acquiredConcurrency = new string[concurrency.Length];
+        for (int index = 0; index < concurrency.Length; index++)
+        {
+            RateDimension dimension = concurrency[index];
+            tracker.ConcurrencyCounts[dimension.Key] =
+                tracker.ConcurrencyCounts.GetValueOrDefault(dimension.Key) + 1;
+            acquiredConcurrency[index] = dimension.Key;
+        }
+
+        lease = new AbuseLease(this, tracker, acquiredConcurrency);
+        return true;
     }
 
     private static bool CanAcquireAll(
@@ -415,6 +484,9 @@ internal sealed class AbuseProtectionService
         return "unknown";
     }
 
+    private static IPAddress NormalizeAddress(IPAddress address) =>
+        address.IsIPv4MappedToIPv6 ? address.MapToIPv4() : address;
+
     private readonly record struct RateDimension(string Key, int Limit);
 
     private enum TrackerDomain

src/FinalFactory.Rendezvous.Server/Abuse/HttpAbuseProtectionMiddleware.cs

@@ -19,16 +19,71 @@ internal sealed class HttpAbuseProtectionMiddleware(
         string operation = context.GetEndpoint()?.Metadata.GetMetadata<IEndpointNameMetadata>()
             ?.EndpointName ?? "Unmatched";
         bool healthEndpoint = operation is "GetLiveness" or "GetReadiness";
-        bool acquired = healthEndpoint
-            ? protection.TryAcquireHealthIngress(
+        bool operatorEndpoint = operation is
+            "GetOperatorStatus"
+            or "RevokeOperatorListing"
+            or "RevokeOperatorPrincipal"
+            or "RevokeOperatorSigningKey"
+            or "BeginOperatorDrain";
+        if (operatorEndpoint
+            && !protection.IsOperatorSourceAllowed(context.Connection.RemoteIpAddress))
+        {
+            bool deniedSourceAdmitted = protection.TryAcquireHttpIngress(
                 context.Connection.RemoteIpAddress,
-                out AbuseProtectionService.AbuseLease? lease,
-                out int retryAfterSeconds)
-            : protection.TryAcquireHttpIngress(
+                "Unmatched",
+                out AbuseProtectionService.AbuseLease? deniedSourceLease,
+                out int deniedRetryAfterSeconds);
+            using (deniedSourceLease)
+            {
+                if (!deniedSourceAdmitted)
+                {
+                    context.Response.Headers.RetryAfter = deniedRetryAfterSeconds.ToString(
+                        System.Globalization.CultureInfo.InvariantCulture);
+                    await WriteErrorAsync(
+                        context,
+                        StatusCodes.Status429TooManyRequests,
+                        RendezvousErrorCode.RateLimited,
+                        "The request rate limit was exceeded.",
+                        deniedRetryAfterSeconds).ConfigureAwait(false);
+                    return;
+                }
+
+                await WriteErrorAsync(
+                    context,
+                    StatusCodes.Status404NotFound,
+                    RendezvousErrorCode.NotFound,
+                    "The requested resource was not found.").ConfigureAwait(false);
+            }
+
+            return;
+        }
+
+        AbuseProtectionService.AbuseLease? lease;
+        int retryAfterSeconds;
+        bool acquired;
+        if (healthEndpoint)
+        {
+            acquired = protection.TryAcquireHealthIngress(
+                context.Connection.RemoteIpAddress,
+                out lease,
+                out retryAfterSeconds);
+        }
+        else if (operatorEndpoint)
+        {
+            acquired = protection.TryAcquireOperatorIngress(
+                context.Connection.RemoteIpAddress,
+                out lease,
+                out retryAfterSeconds);
+        }
+        else
+        {
+            acquired = protection.TryAcquireHttpIngress(
                 context.Connection.RemoteIpAddress,
                 operation,
                 out lease,
                 out retryAfterSeconds);
+        }
+
         if (!acquired)
         {
             context.Response.Headers.RetryAfter = retryAfterSeconds.ToString(

src/FinalFactory.Rendezvous.Server/ConnectionOutcomes/ConnectionOutcomeService.cs

@@ -1,4 +1,5 @@
 using FinalFactory.Rendezvous.Contracts;
+using FinalFactory.Rendezvous.Server.Observability;
 using FinalFactory.Rendezvous.Server.Sessions;
 using FinalFactory.Rendezvous.Server.State;
 
@@ -15,6 +16,10 @@ internal sealed class ConnectionOutcomeMetrics
 {
     private readonly object _gate = new();
     private readonly Dictionary<(ConnectionOutcomeKind, ConnectionElapsedBucket), long> _counts = [];
+    private readonly RendezvousTelemetry? _telemetry;
+
+    public ConnectionOutcomeMetrics(RendezvousTelemetry? telemetry = null) =>
+        _telemetry = telemetry;
 
     internal void Record(ConnectionOutcomeKind outcome, ConnectionElapsedBucket elapsedBucket)
     {
@@ -24,6 +29,8 @@ internal sealed class ConnectionOutcomeMetrics
             _counts.TryGetValue(key, out long count);
             _counts[key] = count + 1;
         }
+
+        _telemetry?.RecordConnectionOutcome(outcome.ToString(), elapsedBucket.ToString());
     }
 
     internal long GetCount(ConnectionOutcomeKind outcome, ConnectionElapsedBucket elapsedBucket)

src/FinalFactory.Rendezvous.Server/Http/RendezvousExceptionHandler.cs

@@ -4,7 +4,8 @@ using Microsoft.AspNetCore.Diagnostics;
 
 namespace FinalFactory.Rendezvous.Server.Http;
 
-internal sealed class RendezvousExceptionHandler : IExceptionHandler
+internal sealed partial class RendezvousExceptionHandler(
+    ILogger<RendezvousExceptionHandler> logger) : IExceptionHandler
 {
     public async ValueTask<bool> TryHandleAsync(
         HttpContext httpContext,
@@ -26,6 +27,13 @@ internal sealed class RendezvousExceptionHandler : IExceptionHandler
             : invalidRequest
                 ? StatusCodes.Status400BadRequest
                 : StatusCodes.Status500InternalServerError;
+        LogRequestFailure(
+            logger,
+            payloadTooLarge ? "payload-too-large" : invalidRequest ? "invalid-request" : "internal-error",
+            httpContext.Response.StatusCode,
+            httpContext.Response.Headers["X-Rendezvous-Correlation-ID"].ToString() is { Length: > 0 } value
+                ? value
+                : "unavailable");
         await httpContext.Response.WriteAsJsonAsync(
             new ApiError
             {
@@ -42,4 +50,14 @@ internal sealed class RendezvousExceptionHandler : IExceptionHandler
             cancellationToken).ConfigureAwait(false);
         return true;
     }
+
+    [LoggerMessage(
+        EventId = 200,
+        Level = LogLevel.Warning,
+        Message = "Request failed with {FailureKind} and HTTP status {StatusCode}; correlation {CorrelationId}")]
+    private static partial void LogRequestFailure(
+        ILogger logger,
+        string failureKind,
+        int statusCode,
+        string correlationId);
 }

src/FinalFactory.Rendezvous.Server/Observability/AuditOptions.cs

@@ -0,0 +1,14 @@
+using System.ComponentModel.DataAnnotations;
+
+namespace FinalFactory.Rendezvous.Server.Observability;
+
+internal sealed class AuditOptions
+{
+    public const string SectionName = "Rendezvous:Audit";
+
+    [Range(100, 100_000)]
+    public int MaxEntries { get; set; } = 10_000;
+
+    [Range(1, 30)]
+    public int RetentionDays { get; set; } = 30;
+}

src/FinalFactory.Rendezvous.Server/Observability/AuditTrail.cs

@@ -0,0 +1,134 @@
+using System.Security.Cryptography;
+using System.Text;
+using Microsoft.Extensions.Options;
+
+namespace FinalFactory.Rendezvous.Server.Observability;
+
+internal sealed partial class AuditTrail
+{
+    private readonly object _gate = new();
+    private readonly LinkedList<AuditEntry> _entries = [];
+    private readonly AuditOptions _options;
+    private readonly TimeProvider _timeProvider;
+    private readonly ILogger<AuditTrail> _logger;
+    private readonly RendezvousTelemetry _telemetry;
+
+    public AuditTrail(
+        IOptions<AuditOptions> options,
+        ILogger<AuditTrail> logger,
+        RendezvousTelemetry telemetry,
+        TimeProvider? timeProvider = null)
+    {
+        _options = options.Value;
+        _logger = logger;
+        _telemetry = telemetry;
+        _timeProvider = timeProvider ?? TimeProvider.System;
+    }
+
+    public void Record(
+        string actorSubject,
+        string action,
+        string result,
+        string targetKind,
+        string targetIdentifier,
+        string correlationId)
+    {
+        DateTimeOffset now = _timeProvider.GetUtcNow();
+        AuditEntry entry = new(
+            now,
+            Fingerprint(actorSubject),
+            action,
+            result,
+            targetKind,
+            Fingerprint(targetIdentifier),
+            correlationId);
+        lock (_gate)
+        {
+            PurgeExpired(now);
+
+            while (_entries.Count >= _options.MaxEntries)
+            {
+                _entries.RemoveFirst();
+            }
+
+            _entries.AddLast(entry);
+        }
+
+        _telemetry.RecordAudit(action, result);
+        LogOperatorAction(
+            _logger,
+            entry.Timestamp,
+            entry.ActorFingerprint,
+            action,
+            result,
+            targetKind,
+            entry.TargetFingerprint,
+            correlationId);
+    }
+
+    public IReadOnlyDictionary<string, long> GetAggregateCounts()
+    {
+        lock (_gate)
+        {
+            PurgeExpired(_timeProvider.GetUtcNow());
+            return _entries
+                .GroupBy(static entry => $"{entry.Action}:{entry.Result}", StringComparer.Ordinal)
+                .ToDictionary(
+                    static group => group.Key,
+                    static group => (long)group.Count(),
+                    StringComparer.Ordinal);
+        }
+    }
+
+    internal IReadOnlyList<AuditEntry> GetEntriesForTests()
+    {
+        lock (_gate)
+        {
+            PurgeExpired(_timeProvider.GetUtcNow());
+            return _entries.ToArray();
+        }
+    }
+
+    private void PurgeExpired(DateTimeOffset now)
+    {
+        DateTimeOffset oldest = now.AddDays(-_options.RetentionDays);
+        while (_entries.First is { Value.Timestamp: var timestamp }
+               && timestamp < oldest)
+        {
+            _entries.RemoveFirst();
+        }
+    }
+
+    private static string Fingerprint(string value)
+    {
+        byte[] digest = SHA256.HashData(Encoding.UTF8.GetBytes(value));
+        return Convert.ToHexString(digest.AsSpan(0, 12));
+    }
+
+    [LoggerMessage(
+        EventId = 100,
+        Level = LogLevel.Information,
+        Message = "Operator audit at {Timestamp}: actor {ActorFingerprint} action {Action} completed with {Result} for {TargetKind} target {TargetFingerprint}; correlation {CorrelationId}")]
+    private static partial void LogOperatorAction(
+        ILogger logger,
+        DateTimeOffset timestamp,
+        string actorFingerprint,
+        string action,
+        string result,
+        string targetKind,
+        string targetFingerprint,
+        string correlationId);
+}
+
+internal sealed record AuditEntry(
+    DateTimeOffset Timestamp,
+    string ActorFingerprint,
+    string Action,
+    string Result,
+    string TargetKind,
+    string TargetFingerprint,
+    string CorrelationId)
+{
+    public override string ToString() =>
+        $"[AuditEntry {Action}/{Result}; actor and target fingerprinted]";
+}

src/FinalFactory.Rendezvous.Server/Observability/HealthEndpoints.cs

@@ -0,0 +1,30 @@
+using FinalFactory.Rendezvous.Contracts;
+
+namespace FinalFactory.Rendezvous.Server.Observability;
+
+internal static class HealthEndpoints
+{
+    public static IEndpointRouteBuilder MapRendezvousHealthEndpoints(
+        this IEndpointRouteBuilder endpoints)
+    {
+        endpoints.MapGet(
+                "/health/live",
+                static () => Results.Ok(new HealthResponse { Status = "live" }))
+            .Produces<HealthResponse>()
+            .Produces<ApiError>(StatusCodes.Status429TooManyRequests)
+            .WithName("GetLiveness")
+            .WithTags("Health");
+        endpoints.MapGet(
+                "/health/ready",
+                static (RendezvousReadiness readiness) =>
+                    !readiness.GetSnapshot().IsReady
+                        ? Results.StatusCode(StatusCodes.Status503ServiceUnavailable)
+                        : Results.Ok(new HealthResponse { Status = "ready" }))
+            .Produces<HealthResponse>()
+            .Produces<ApiError>(StatusCodes.Status429TooManyRequests)
+            .Produces(StatusCodes.Status503ServiceUnavailable)
+            .WithName("GetReadiness")
+            .WithTags("Health");
+        return endpoints;
+    }
+}

src/FinalFactory.Rendezvous.Server/Observability/RendezvousReadiness.cs

@@ -0,0 +1,41 @@
+using FinalFactory.Rendezvous.Server.Provisioning;
+using FinalFactory.Rendezvous.Server.State;
+using FinalFactory.Rendezvous.Server.Transport;
+using Microsoft.Extensions.Options;
+
+namespace FinalFactory.Rendezvous.Server.Observability;
+
+internal sealed class RendezvousReadiness(
+    UdpMediatorService mediator,
+    ProvisioningReadiness provisioning,
+    IEphemeralRendezvousStore state,
+    IOptions<UdpMediatorOptions> udpOptions)
+{
+    public ReadinessSnapshot GetSnapshot()
+    {
+        bool ipv6Required = !string.IsNullOrWhiteSpace(udpOptions.Value.Ipv6ListenAddress);
+        return new ReadinessSnapshot(
+            HttpListenerReady: true,
+            UdpIpv4ListenerReady: mediator.LocalEndpoint is not null,
+            UdpIpv6ListenerReady: !ipv6Required || mediator.LocalIpv6Endpoint is not null,
+            ProvisioningReady: provisioning.IsReady,
+            StoreAvailable: state.IsAvailable,
+            Draining: state.IsDraining);
+    }
+}
+
+internal sealed record ReadinessSnapshot(
+    bool HttpListenerReady,
+    bool UdpIpv4ListenerReady,
+    bool UdpIpv6ListenerReady,
+    bool ProvisioningReady,
+    bool StoreAvailable,
+    bool Draining)
+{
+    public bool IsReady => HttpListenerReady
+        && UdpIpv4ListenerReady
+        && UdpIpv6ListenerReady
+        && ProvisioningReady
+        && StoreAvailable
+        && !Draining;
+}

src/FinalFactory.Rendezvous.Server/Observability/RendezvousTelemetry.cs

@@ -0,0 +1,126 @@
+using System.Diagnostics;
+using System.Diagnostics.Metrics;
+using FinalFactory.Rendezvous.Server.State;
+
+namespace FinalFactory.Rendezvous.Server.Observability;
+
+internal sealed class RendezvousTelemetry : IDisposable
+{
+    public const string MeterName = "FinalFactory.Rendezvous";
+    public const string ActivitySourceName = "FinalFactory.Rendezvous.Server";
+
+    private readonly InMemoryEphemeralRendezvousStore _store;
+    private readonly Meter _meter = new(MeterName, "1.0.0");
+    private readonly ActivitySource _activities = new(ActivitySourceName, "1.0.0");
+    private readonly Counter<long> _httpRequests;
+    private readonly Histogram<double> _httpDuration;
+    private readonly Counter<long> _udpResults;
+    private readonly Histogram<double> _udpDuration;
+    private readonly Counter<long> _limiterDrops;
+    private readonly Counter<long> _auditEvents;
+    private readonly Counter<long> _connectionOutcomes;
+    private readonly Counter<long> _operatorAuthentication;
+    private readonly Histogram<double> _pairingLatency;
+
+    public RendezvousTelemetry(InMemoryEphemeralRendezvousStore store)
+    {
+        _store = store;
+        _httpRequests = _meter.CreateCounter<long>("rendezvous.http.requests");
+        _httpDuration = _meter.CreateHistogram<double>(
+            "rendezvous.http.duration",
+            "ms");
+        _udpResults = _meter.CreateCounter<long>("rendezvous.udp.results");
+        _udpDuration = _meter.CreateHistogram<double>(
+            "rendezvous.udp.duration",
+            "ms");
+        _limiterDrops = _meter.CreateCounter<long>("rendezvous.limiter.drops");
+        _auditEvents = _meter.CreateCounter<long>("rendezvous.audit.events");
+        _connectionOutcomes = _meter.CreateCounter<long>("rendezvous.connection.outcomes");
+        _operatorAuthentication = _meter.CreateCounter<long>("rendezvous.operator.authentication");
+        _pairingLatency = _meter.CreateHistogram<double>(
+            "rendezvous.pairing.latency",
+            "ms");
+        _meter.CreateObservableGauge(
+            "rendezvous.store.active_listings",
+            () => _store.GetMetricsSnapshot().ActiveListings);
+        _meter.CreateObservableGauge(
+            "rendezvous.store.active_leases",
+            () => _store.GetMetricsSnapshot().ActiveListings);
+        _meter.CreateObservableGauge(
+            "rendezvous.store.active_attempts",
+            () => _store.GetMetricsSnapshot().ActiveJoinAttempts);
+        _meter.CreateObservableGauge(
+            "rendezvous.queue.depth",
+            () => _store.GetMetricsSnapshot().ActiveJoinAttempts);
+        _meter.CreateObservableGauge(
+            "rendezvous.store.replay_markers",
+            () => _store.GetMetricsSnapshot().ReplayMarkers);
+        _meter.CreateObservableGauge(
+            "rendezvous.store.available",
+            () => _store.GetMetricsSnapshot().IsAvailable ? 1 : 0);
+        _meter.CreateObservableCounter(
+            "rendezvous.store.expiry_churn",
+            () => _store.GetMetricsSnapshot().ExpiryChurn);
+    }
+
+    public Activity? StartActivity(string name, ActivityKind kind = ActivityKind.Internal) =>
+        _activities.StartActivity(name, kind);
+
+    public void RecordHttp(string operation, int statusCode, double elapsedMilliseconds)
+    {
+        TagList tags = new()
+        {
+            { "operation", operation },
+            { "status_code", statusCode },
+        };
+        _httpRequests.Add(1, tags);
+        _httpDuration.Record(elapsedMilliseconds, tags);
+    }
+
+    public void RecordUdp(string operation, string result, double elapsedMilliseconds)
+    {
+        TagList tags = new()
+        {
+            { "operation", operation },
+            { "result", result },
+        };
+        _udpResults.Add(1, tags);
+        _udpDuration.Record(elapsedMilliseconds, tags);
+    }
+
+    public void RecordLimiterDrop(string transport, string partition) =>
+        _limiterDrops.Add(1, new TagList
+        {
+            { "transport", transport },
+            { "partition", partition },
+        });
+
+    public void RecordAudit(string action, string result) =>
+        _auditEvents.Add(1, new TagList
+        {
+            { "action", action },
+            { "result", result },
+        });
+
+    public void RecordConnectionOutcome(string outcome, string elapsedBucket) =>
+        _connectionOutcomes.Add(1, new TagList
+        {
+            { "outcome", outcome },
+            { "elapsed_bucket", elapsedBucket },
+        });
+
+    public void RecordOperatorAuthentication(string result) =>
+        _operatorAuthentication.Add(1, new TagList
+        {
+            { "result", result },
+        });
+
+    public void RecordPairingLatency(double elapsedMilliseconds) =>
+        _pairingLatency.Record(elapsedMilliseconds);
+
+    public void Dispose()
+    {
+        _activities.Dispose();
+        _meter.Dispose();
+    }
+}

src/FinalFactory.Rendezvous.Server/Observability/TelemetryMiddleware.cs

@@ -0,0 +1,32 @@
+using System.Diagnostics;
+
+namespace FinalFactory.Rendezvous.Server.Observability;
+
+internal sealed class TelemetryMiddleware(
+    RequestDelegate next,
+    RendezvousTelemetry telemetry)
+{
+    public async Task InvokeAsync(HttpContext context)
+    {
+        string operation = context.GetEndpoint()?.Metadata.GetMetadata<IEndpointNameMetadata>()
+            ?.EndpointName ?? "Unmatched";
+        long started = Stopwatch.GetTimestamp();
+        using Activity? activity = telemetry.StartActivity(
+            $"HTTP {operation}",
+            ActivityKind.Server);
+        string correlationId = activity?.TraceId.ToString() ?? Guid.NewGuid().ToString("N");
+        context.Response.Headers["X-Rendezvous-Correlation-ID"] = correlationId;
+        activity?.SetTag("rendezvous.operation", operation);
+        try
+        {
+            await next(context).ConfigureAwait(false);
+        }
+        finally
+        {
+            telemetry.RecordHttp(
+                operation,
+                context.Response.StatusCode,
+                Stopwatch.GetElapsedTime(started).TotalMilliseconds);
+        }
+    }
+}

src/FinalFactory.Rendezvous.Server/Operations/OperatorEndpoints.cs

@@ -0,0 +1,428 @@
+using FinalFactory.Rendezvous.Contracts;
+using FinalFactory.Rendezvous.Server.Observability;
+using FinalFactory.Rendezvous.Server.Provisioning;
+using FinalFactory.Rendezvous.Server.State;
+using Microsoft.AspNetCore.Mvc;
+
+namespace FinalFactory.Rendezvous.Server.Operations;
+
+internal static class OperatorEndpoints
+{
+    private const string CorrelationHeader = "X-Rendezvous-Correlation-ID";
+
+    public static IEndpointRouteBuilder MapOperatorEndpoints(this IEndpointRouteBuilder endpoints)
+    {
+        RouteGroupBuilder group = endpoints.MapGroup("/v1/operator").WithTags("Operator");
+        group.MapGet("/status", GetStatus)
+            .Produces<OperatorStatusResponse>()
+            .Produces<ApiError>(StatusCodes.Status401Unauthorized)
+            .Produces<ApiError>(StatusCodes.Status403Forbidden)
+            .Produces<ApiError>(StatusCodes.Status404NotFound)
+            .Produces<ApiError>(StatusCodes.Status429TooManyRequests)
+            .WithName("GetOperatorStatus");
+        group.MapPost("/listings/revoke", RevokeListing)
+            .Accepts<RevokeListingRequest>("application/json")
+            .Produces<OperatorActionResponse>()
+            .Produces<ApiError>(StatusCodes.Status400BadRequest)
+            .Produces<ApiError>(StatusCodes.Status413PayloadTooLarge)
+            .Produces<ApiError>(StatusCodes.Status401Unauthorized)
+            .Produces<ApiError>(StatusCodes.Status403Forbidden)
+            .Produces<ApiError>(StatusCodes.Status404NotFound)
+            .Produces<ApiError>(StatusCodes.Status429TooManyRequests)
+            .Produces<ApiError>(StatusCodes.Status503ServiceUnavailable)
+            .WithName("RevokeOperatorListing");
+        group.MapPost("/principals/revoke", RevokePrincipal)
+            .Accepts<RevokePrincipalRequest>("application/json")
+            .Produces<OperatorActionResponse>()
+            .Produces<ApiError>(StatusCodes.Status400BadRequest)
+            .Produces<ApiError>(StatusCodes.Status413PayloadTooLarge)
+            .Produces<ApiError>(StatusCodes.Status401Unauthorized)
+            .Produces<ApiError>(StatusCodes.Status403Forbidden)
+            .Produces<ApiError>(StatusCodes.Status404NotFound)
+            .Produces<ApiError>(StatusCodes.Status429TooManyRequests)
+            .Produces<ApiError>(StatusCodes.Status503ServiceUnavailable)
+            .WithName("RevokeOperatorPrincipal");
+        group.MapPost("/keys/revoke", RevokeSigningKey)
+            .Accepts<RevokeSigningKeyRequest>("application/json")
+            .Produces<OperatorActionResponse>()
+            .Produces<ApiError>(StatusCodes.Status400BadRequest)
+            .Produces<ApiError>(StatusCodes.Status413PayloadTooLarge)
+            .Produces<ApiError>(StatusCodes.Status401Unauthorized)
+            .Produces<ApiError>(StatusCodes.Status403Forbidden)
+            .Produces<ApiError>(StatusCodes.Status404NotFound)
+            .Produces<ApiError>(StatusCodes.Status429TooManyRequests)
+            .WithName("RevokeOperatorSigningKey");
+        group.MapPost("/drain", BeginDrain)
+            .Accepts<BeginDrainRequest>("application/json")
+            .Produces<OperatorActionResponse>()
+            .Produces<ApiError>(StatusCodes.Status400BadRequest)
+            .Produces<ApiError>(StatusCodes.Status413PayloadTooLarge)
+            .Produces<ApiError>(StatusCodes.Status401Unauthorized)
+            .Produces<ApiError>(StatusCodes.Status403Forbidden)
+            .Produces<ApiError>(StatusCodes.Status404NotFound)
+            .Produces<ApiError>(StatusCodes.Status429TooManyRequests)
+            .WithName("BeginOperatorDrain");
+        return endpoints;
+    }
+
+    private static IResult GetStatus(
+        [FromHeader(Name = "Authorization")] string? authorization,
+        [FromServices] PrincipalCredentialService credentials,
+        [FromServices] IWallClock clock,
+        [FromServices] OperatorService service,
+        [FromServices] AuditTrail audit,
+        [FromServices] RendezvousTelemetry telemetry,
+        HttpContext context)
+    {
+        if (!TryAuthorize(
+                authorization,
+                OperatorPermission.ReadPolicy,
+                "inspect-status",
+                credentials,
+                clock,
+                audit,
+                telemetry,
+                context,
+                out OperatorPrincipal? principal,
+                out IResult? failure))
+        {
+            return failure!;
+        }
+
+        OperatorStatusResponse response = service.GetStatus();
+        audit.Record(
+            principal!.Subject,
+            "inspect-status",
+            "succeeded",
+            "service",
+            "rendezvous",
+            Correlation(context));
+        return Results.Ok(response);
+    }
+
+    private static IResult RevokeListing(
+        [FromBody] RevokeListingRequest request,
+        [FromHeader(Name = "Authorization")] string? authorization,
+        [FromServices] PrincipalCredentialService credentials,
+        [FromServices] IWallClock clock,
+        [FromServices] OperatorService service,
+        [FromServices] AuditTrail audit,
+        [FromServices] RendezvousTelemetry telemetry,
+        HttpContext context,
+        CancellationToken cancellationToken)
+    {
+        if (!TryAuthorize(
+                authorization,
+                OperatorPermission.RevokePublisher,
+                "revoke-listing",
+                credentials,
+                clock,
+                audit,
+                telemetry,
+                context,
+                out OperatorPrincipal? principal,
+                out IResult? failure))
+        {
+            return failure!;
+        }
+
+        bool valid = SessionListingId.TryParse(request.ListingId, out SessionListingId listingId);
+        if (!valid || !string.Equals(request.ListingId, request.ConfirmListingId, StringComparison.Ordinal))
+        {
+            AuditRejected(audit, principal!, "revoke-listing", "listing", request.ListingId, context);
+            return BadRequest("A valid listing ID and an exact repeated confirmation are required.");
+        }
+
+        StoreResult<bool> result = service.RevokeListing(listingId, cancellationToken);
+        return StoreActionResult(
+            result.Code,
+            audit,
+            principal!,
+            "revoke-listing",
+            "listing",
+            request.ListingId,
+            context,
+            affectedResources: result.Succeeded ? 1 : null);
+    }
+
+    private static IResult RevokePrincipal(
+        [FromBody] RevokePrincipalRequest request,
+        [FromHeader(Name = "Authorization")] string? authorization,
+        [FromServices] PrincipalCredentialService credentials,
+        [FromServices] IWallClock clock,
+        [FromServices] OperatorService service,
+        [FromServices] AuditTrail audit,
+        [FromServices] RendezvousTelemetry telemetry,
+        HttpContext context,
+        CancellationToken cancellationToken)
+    {
+        if (!TryAuthorize(
+                authorization,
+                OperatorPermission.RevokePublisher,
+                "revoke-principal",
+                credentials,
+                clock,
+                audit,
+                telemetry,
+                context,
+                out OperatorPrincipal? principal,
+                out IResult? failure))
+        {
+            return failure!;
+        }
+
+        bool safeSubject = request.Subject is { Length: > 0 and <= 128 }
+            && request.Subject.All(static character => character is >= '!' and <= '~');
+        if (!safeSubject
+            || !string.Equals(request.Subject, request.ConfirmSubject, StringComparison.Ordinal)
+            || request.LifetimeSeconds is < 1 or > 600)
+        {
+            AuditRejected(audit, principal!, "revoke-principal", "principal", request.Subject, context);
+            return BadRequest("A valid subject, exact repeated confirmation, and 1-600 second lifetime are required.");
+        }
+
+        StoreResult<int> result = service.RevokePrincipal(
+            request.Subject,
+            TimeSpan.FromSeconds(request.LifetimeSeconds),
+            cancellationToken);
+        return StoreActionResult(
+            result.Code,
+            audit,
+            principal!,
+            "revoke-principal",
+            "principal",
+            request.Subject,
+            context,
+            result.Value);
+    }
+
+    private static IResult RevokeSigningKey(
+        [FromBody] RevokeSigningKeyRequest request,
+        [FromHeader(Name = "Authorization")] string? authorization,
+        [FromServices] PrincipalCredentialService credentials,
+        [FromServices] IWallClock clock,
+        [FromServices] OperatorService service,
+        [FromServices] AuditTrail audit,
+        [FromServices] RendezvousTelemetry telemetry,
+        HttpContext context)
+    {
+        if (!TryAuthorize(
+                authorization,
+                OperatorPermission.RotateKeys,
+                "revoke-signing-key",
+                credentials,
+                clock,
+                audit,
+                telemetry,
+                context,
+                out OperatorPrincipal? principal,
+                out IResult? failure))
+        {
+            return failure!;
+        }
+
+        bool safeKeyId = request.KeyId is { Length: > 0 and <= 64 }
+            && request.KeyId.All(static character => character is
+                >= 'A' and <= 'Z'
+                or >= 'a' and <= 'z'
+                or >= '0' and <= '9'
+                or '-'
+                or '_');
+        if (!safeKeyId || !string.Equals(request.KeyId, request.ConfirmKeyId, StringComparison.Ordinal))
+        {
+            AuditRejected(audit, principal!, "revoke-signing-key", "signing-key", request.KeyId, context);
+            return BadRequest("A valid key ID and an exact repeated confirmation are required.");
+        }
+
+        bool revoked = service.RevokeSigningKey(request.KeyId);
+        string result = revoked ? "succeeded" : "not-found";
+        audit.Record(
+            principal!.Subject,
+            "revoke-signing-key",
+            result,
+            "signing-key",
+            request.KeyId,
+            Correlation(context));
+        return revoked
+            ? Results.Ok(new OperatorActionResponse { Status = "completed" })
+            : Error(RendezvousErrorCode.NotFound, "The requested resource was not found.");
+    }
+
+    private static IResult BeginDrain(
+        [FromBody] BeginDrainRequest request,
+        [FromHeader(Name = "Authorization")] string? authorization,
+        [FromServices] PrincipalCredentialService credentials,
+        [FromServices] IWallClock clock,
+        [FromServices] OperatorService service,
+        [FromServices] AuditTrail audit,
+        [FromServices] RendezvousTelemetry telemetry,
+        HttpContext context,
+        CancellationToken cancellationToken)
+    {
+        if (!TryAuthorize(
+                authorization,
+                OperatorPermission.ManagePolicy,
+                "begin-drain",
+                credentials,
+                clock,
+                audit,
+                telemetry,
+                context,
+                out OperatorPrincipal? principal,
+                out IResult? failure))
+        {
+            return failure!;
+        }
+
+        if (!string.Equals(request.Confirmation, "DRAIN", StringComparison.Ordinal))
+        {
+            AuditRejected(audit, principal!, "begin-drain", "service", "rendezvous", context);
+            return BadRequest("The confirmation value must be exactly 'DRAIN'.");
+        }
+
+        service.BeginDrain(cancellationToken);
+        audit.Record(
+            principal!.Subject,
+            "begin-drain",
+            "succeeded",
+            "service",
+            "rendezvous",
+            Correlation(context));
+        return Results.Ok(new OperatorActionResponse { Status = "draining" });
+    }
+
+    private static bool TryAuthorize(
+        string? authorization,
+        OperatorPermission requiredPermission,
+        string operation,
+        PrincipalCredentialService credentials,
+        IWallClock clock,
+        AuditTrail audit,
+        RendezvousTelemetry telemetry,
+        HttpContext context,
+        out OperatorPrincipal? principal,
+        out IResult? failure)
+    {
+        principal = null;
+        failure = null;
+        const string prefix = "Bearer ";
+        if (authorization is null
+            || !authorization.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))
+        {
+            telemetry.RecordOperatorAuthentication("rejected");
+            failure = AuthenticationRequired(context);
+            return false;
+        }
+
+        CredentialValidationResult validation = credentials.Validate(
+            authorization[prefix.Length..],
+            clock.UtcNow);
+        if (!validation.IsValid || validation.Principal is not OperatorPrincipal candidate)
+        {
+            telemetry.RecordOperatorAuthentication("rejected");
+            failure = AuthenticationRequired(context);
+            return false;
+        }
+
+        if (!candidate.Permissions.Contains(requiredPermission))
+        {
+            telemetry.RecordOperatorAuthentication("forbidden");
+            audit.Record(
+                candidate.Subject,
+                operation,
+                "forbidden",
+                "operator-operation",
+                operation,
+                Correlation(context));
+            failure = Error(RendezvousErrorCode.Forbidden, "The operator is not authorized for this operation.");
+            return false;
+        }
+
+        telemetry.RecordOperatorAuthentication("accepted");
+        principal = candidate;
+        return true;
+    }
+
+    private static IResult StoreActionResult(
+        StoreResultCode code,
+        AuditTrail audit,
+        OperatorPrincipal principal,
+        string action,
+        string targetKind,
+        string targetIdentifier,
+        HttpContext context,
+        int? affectedResources)
+    {
+        string auditResult = code == StoreResultCode.Success
+            ? "succeeded"
+            : code.ToString().ToLowerInvariant();
+        audit.Record(
+            principal.Subject,
+            action,
+            auditResult,
+            targetKind,
+            targetIdentifier,
+            Correlation(context));
+        return code switch
+        {
+            StoreResultCode.Success => Results.Ok(new OperatorActionResponse
+            {
+                Status = "completed",
+                AffectedResources = affectedResources,
+            }),
+            StoreResultCode.NotFound => Error(
+                RendezvousErrorCode.NotFound,
+                "The requested resource was not found."),
+            StoreResultCode.CapacityExceeded => Error(
+                RendezvousErrorCode.CapacityExceeded,
+                "The operation could not be retained within the configured capacity."),
+            StoreResultCode.ServiceUnavailable or StoreResultCode.Draining => Error(
+                RendezvousErrorCode.ServiceUnavailable,
+                "The service is not available for this operation."),
+            _ => Error(RendezvousErrorCode.Conflict, "The operation could not be completed."),
+        };
+    }
+
+    private static void AuditRejected(
+        AuditTrail audit,
+        OperatorPrincipal principal,
+        string action,
+        string targetKind,
+        string? targetIdentifier,
+        HttpContext context) => audit.Record(
+            principal.Subject,
+            action,
+            "rejected",
+            targetKind,
+            targetIdentifier ?? string.Empty,
+            Correlation(context));
+
+    private static string Correlation(HttpContext context) =>
+        context.Response.Headers[CorrelationHeader].ToString() is { Length: > 0 } value
+            ? value
+            : "unavailable";
+
+    private static IResult AuthenticationRequired(HttpContext context)
+    {
+        context.Response.Headers.WWWAuthenticate = "Bearer realm=\"operator\"";
+        return Error(
+            RendezvousErrorCode.AuthenticationRequired,
+            "A valid operator bearer credential is required.");
+    }
+
+    private static IResult BadRequest(string message) => Error(RendezvousErrorCode.InvalidRequest, message);
+
+    private static IResult Error(RendezvousErrorCode code, string message) => Results.Json(
+        new ApiError { Code = code, Message = message },
+        ContractJson.Options,
+        statusCode: code switch
+        {
+            RendezvousErrorCode.AuthenticationRequired => StatusCodes.Status401Unauthorized,
+            RendezvousErrorCode.Forbidden => StatusCodes.Status403Forbidden,
+            RendezvousErrorCode.NotFound => StatusCodes.Status404NotFound,
+            RendezvousErrorCode.CapacityExceeded => StatusCodes.Status429TooManyRequests,
+            RendezvousErrorCode.ServiceUnavailable => StatusCodes.Status503ServiceUnavailable,
+            RendezvousErrorCode.Conflict => StatusCodes.Status409Conflict,
+            _ => StatusCodes.Status400BadRequest,
+        });
+}

src/FinalFactory.Rendezvous.Server/Operations/OperatorModels.cs

@@ -0,0 +1,82 @@
+namespace FinalFactory.Rendezvous.Server.Operations;
+
+internal sealed record OperatorStatusResponse
+{
+    public required string Status { get; init; }
+    public required OperatorReadinessResponse Readiness { get; init; }
+    public required OperatorStoreResponse Store { get; init; }
+    public required IReadOnlyList<OperatorTenantResponse> Tenants { get; init; }
+    public required IReadOnlyList<OperatorSigningKeyResponse> SigningKeys { get; init; }
+    public required IReadOnlyDictionary<string, long> AuditCounts { get; init; }
+}
+
+internal sealed record OperatorReadinessResponse
+{
+    public required bool HttpListener { get; init; }
+    public required bool UdpIpv4Listener { get; init; }
+    public required bool UdpIpv6Listener { get; init; }
+    public required bool Provisioning { get; init; }
+    public required bool Store { get; init; }
+    public required bool Draining { get; init; }
+}
+
+internal sealed record OperatorStoreResponse
+{
+    public required int ActiveListings { get; init; }
+    public required int FreshPresenceBindings { get; init; }
+    public required int ActiveJoinAttempts { get; init; }
+    public required int RetainedOutcomeReports { get; init; }
+    public required int ReplayMarkers { get; init; }
+    public required int PrincipalRevocations { get; init; }
+    public required int IdempotencyEntries { get; init; }
+    public required long MaintenanceSweeps { get; init; }
+    public required long ExpiryChurn { get; init; }
+}
+
+internal sealed record OperatorTenantResponse
+{
+    public required string GameId { get; init; }
+    public required string EnvironmentId { get; init; }
+    public required string Status { get; init; }
+}
+
+internal sealed record OperatorSigningKeyResponse
+{
+    public required string KeyId { get; init; }
+    public required string Status { get; init; }
+    public required DateTimeOffset SignUntil { get; init; }
+    public required DateTimeOffset VerifyUntil { get; init; }
+    public string? GameId { get; init; }
+    public string? EnvironmentId { get; init; }
+    public required IReadOnlyList<string> CredentialKinds { get; init; }
+}
+
+internal sealed record OperatorActionResponse
+{
+    public required string Status { get; init; }
+    public int? AffectedResources { get; init; }
+}
+
+internal sealed record RevokeListingRequest
+{
+    public required string ListingId { get; init; }
+    public required string ConfirmListingId { get; init; }
+}
+
+internal sealed record RevokePrincipalRequest
+{
+    public required string Subject { get; init; }
+    public required string ConfirmSubject { get; init; }
+    public required int LifetimeSeconds { get; init; }
+}
+
+internal sealed record RevokeSigningKeyRequest
+{
+    public required string KeyId { get; init; }
+    public required string ConfirmKeyId { get; init; }
+}
+
+internal sealed record BeginDrainRequest
+{
+    public required string Confirmation { get; init; }
+}

src/FinalFactory.Rendezvous.Server/Operations/OperatorService.cs

@@ -0,0 +1,81 @@
+using FinalFactory.Rendezvous.Contracts;
+using FinalFactory.Rendezvous.Server.Observability;
+using FinalFactory.Rendezvous.Server.Provisioning;
+using FinalFactory.Rendezvous.Server.State;
+
+namespace FinalFactory.Rendezvous.Server.Operations;
+
+internal sealed class OperatorService(
+    InMemoryEphemeralRendezvousStore store,
+    ProvisioningRuntime provisioning,
+    RendezvousReadiness readiness,
+    AuditTrail audit,
+    IWallClock clock)
+{
+    public OperatorStatusResponse GetStatus()
+    {
+        ReadinessSnapshot readinessSnapshot = readiness.GetSnapshot();
+        EphemeralStoreSnapshot storeSnapshot = store.GetSnapshot();
+        return new OperatorStatusResponse
+        {
+            Status = readinessSnapshot.IsReady ? "ready" : "not-ready",
+            Readiness = new OperatorReadinessResponse
+            {
+                HttpListener = readinessSnapshot.HttpListenerReady,
+                UdpIpv4Listener = readinessSnapshot.UdpIpv4ListenerReady,
+                UdpIpv6Listener = readinessSnapshot.UdpIpv6ListenerReady,
+                Provisioning = readinessSnapshot.ProvisioningReady,
+                Store = readinessSnapshot.StoreAvailable,
+                Draining = readinessSnapshot.Draining,
+            },
+            Store = new OperatorStoreResponse
+            {
+                ActiveListings = storeSnapshot.ActiveListings,
+                FreshPresenceBindings = storeSnapshot.FreshPresenceBindings,
+                ActiveJoinAttempts = storeSnapshot.ActiveJoinAttempts,
+                RetainedOutcomeReports = storeSnapshot.RetainedOutcomeReports,
+                ReplayMarkers = storeSnapshot.ReplayMarkers,
+                PrincipalRevocations = storeSnapshot.PrincipalRevocations,
+                IdempotencyEntries = storeSnapshot.IdempotencyEntries,
+                MaintenanceSweeps = storeSnapshot.MaintenanceSweeps,
+                ExpiryChurn = storeSnapshot.ExpiryChurn,
+            },
+            Tenants = provisioning.Policies.EnabledPolicies
+                .OrderBy(static policy => policy.GameId.Value, StringComparer.Ordinal)
+                .ThenBy(static policy => policy.EnvironmentId.Value, StringComparer.Ordinal)
+                .Select(static policy => new OperatorTenantResponse
+                {
+                    GameId = policy.GameId.Value,
+                    EnvironmentId = policy.EnvironmentId.Value,
+                    Status = "enabled",
+                })
+                .ToArray(),
+            SigningKeys = provisioning.SigningKeys.GetStatuses(clock.UtcNow)
+                .Select(static key => new OperatorSigningKeyResponse
+                {
+                    KeyId = key.KeyId,
+                    Status = key.Status,
+                    SignUntil = key.SignUntil,
+                    VerifyUntil = key.VerifyUntil,
+                    GameId = key.GameId,
+                    EnvironmentId = key.EnvironmentId,
+                    CredentialKinds = key.CredentialKinds,
+                })
+                .ToArray(),
+            AuditCounts = audit.GetAggregateCounts(),
+        };
+    }
+
+    public StoreResult<bool> RevokeListing(
+        SessionListingId listingId,
+        CancellationToken cancellationToken) => store.RevokeListing(listingId, cancellationToken);
+
+    public StoreResult<int> RevokePrincipal(
+        string subject,
+        TimeSpan lifetime,
+        CancellationToken cancellationToken) => store.RevokePrincipal(subject, lifetime, cancellationToken);
+
+    public bool RevokeSigningKey(string keyId) => provisioning.SigningKeys.Revoke(keyId);
+
+    public void BeginDrain(CancellationToken cancellationToken) => store.BeginDrain(cancellationToken);
+}

src/FinalFactory.Rendezvous.Server/Program.cs

@@ -5,6 +5,8 @@ using FinalFactory.Rendezvous.Server.Browser;
 using FinalFactory.Rendezvous.Server.ConnectionOutcomes;
 using FinalFactory.Rendezvous.Server.Http;
 using FinalFactory.Rendezvous.Server.JoinAttempts;
+using FinalFactory.Rendezvous.Server.Observability;
+using FinalFactory.Rendezvous.Server.Operations;
 using FinalFactory.Rendezvous.Server.Provisioning;
 using FinalFactory.Rendezvous.Server.Sessions;
 using FinalFactory.Rendezvous.Server.State;
@@ -13,6 +15,9 @@ using Microsoft.AspNetCore.HttpOverrides;
 using Microsoft.OpenApi;
 
 WebApplicationBuilder builder = WebApplication.CreateBuilder(args);
+builder.Logging.AddFilter(
+    "Microsoft.AspNetCore.Diagnostics.ExceptionHandlerMiddleware",
+    LogLevel.None);
 bool isOpenApiGeneration = string.Equals(
     System.Reflection.Assembly.GetEntryAssembly()?.GetName().Name,
     "GetDocument.Insider",
@@ -61,6 +66,14 @@ builder.Services.AddOpenApi("v1", static options =>
             In = ParameterLocation.Header,
             Description = "Attempt-scoped client capability returned only to the joining caller.",
         };
+        const string operatorSchemeName = "OperatorBearer";
+        document.Components.SecuritySchemes[operatorSchemeName] = new OpenApiSecurityScheme
+        {
+            Type = SecuritySchemeType.Http,
+            Scheme = "bearer",
+            BearerFormat = "rv1 operator credential",
+            Description = "Operator-only credential with an explicit permission set.",
+        };
 
         HashSet<string> securedOperations = new(StringComparer.Ordinal)
         {
@@ -69,8 +82,17 @@ builder.Services.AddOpenApi("v1", static options =>
             "UpdateSession",
             "DeleteSession",
         };
+        HashSet<string> operatorOperations = new(StringComparer.Ordinal)
+        {
+            "GetOperatorStatus",
+            "RevokeOperatorListing",
+            "RevokeOperatorPrincipal",
+            "RevokeOperatorSigningKey",
+            "BeginOperatorDrain",
+        };
         OpenApiSecuritySchemeReference reference = new(schemeName, document, null);
         OpenApiSecuritySchemeReference attemptReference = new(attemptSchemeName, document, null);
+        OpenApiSecuritySchemeReference operatorReference = new(operatorSchemeName, document, null);
         foreach (OpenApiPathItem path in document.Paths.Values)
         {
             if (path.Operations is null)
@@ -99,29 +121,58 @@ builder.Services.AddOpenApi("v1", static options =>
                 });
             }
 
+            foreach (OpenApiOperation operation in path.Operations.Values.Where(
+                         operation => operatorOperations.Contains(
+                             operation.OperationId ?? string.Empty)))
+            {
+                operation.Security ??= [];
+                operation.Security.Add(new OpenApiSecurityRequirement
+                {
+                    [operatorReference] = [],
+                });
+            }
+
             foreach (OpenApiOperation operation in path.Operations.Values)
             {
-                if (operation.Responses is null
-                    || !operation.Responses.TryGetValue(
-                        StatusCodes.Status429TooManyRequests.ToString(
-                            System.Globalization.CultureInfo.InvariantCulture),
-                        out IOpenApiResponse? response)
-                    || response is not OpenApiResponse concreteResponse)
+                if (operation.Responses is null)
                 {
                     continue;
                 }
 
-                concreteResponse.Headers ??=
-                    new Dictionary<string, IOpenApiHeader>(StringComparer.OrdinalIgnoreCase);
-                concreteResponse.Headers["Retry-After"] = new OpenApiHeader
+                foreach ((string status, IOpenApiResponse response) in operation.Responses)
                 {
-                    Description = "Whole seconds before the caller should retry (1-60).",
-                    Schema = new OpenApiSchema
+                    if (response is not OpenApiResponse concreteResponse)
                     {
-                        Type = JsonSchemaType.Integer,
-                        Format = "int32",
-                    },
-                };
+                        continue;
+                    }
+
+                    concreteResponse.Headers ??=
+                        new Dictionary<string, IOpenApiHeader>(StringComparer.OrdinalIgnoreCase);
+                    concreteResponse.Headers["X-Rendezvous-Correlation-ID"] = new OpenApiHeader
+                    {
+                        Description = "Safe request correlation identifier generated by the service.",
+                        Schema = new OpenApiSchema
+                        {
+                            Type = JsonSchemaType.String,
+                        },
+                    };
+                    if (string.Equals(
+                            status,
+                            StatusCodes.Status429TooManyRequests.ToString(
+                                System.Globalization.CultureInfo.InvariantCulture),
+                            StringComparison.Ordinal))
+                    {
+                        concreteResponse.Headers["Retry-After"] = new OpenApiHeader
+                        {
+                            Description = "Whole seconds before the caller should retry (1-60).",
+                            Schema = new OpenApiSchema
+                            {
+                                Type = JsonSchemaType.Integer,
+                                Format = "int32",
+                            },
+                        };
+                    }
+                }
             }
         }
 
@@ -166,8 +217,18 @@ builder.Services
                    && addresses.All(
                        static value => IPAddress.TryParse(value, out _)),
         "Trusted proxy addresses must contain at most 32 literal IP addresses.")
+    .Validate(
+        options => options.OperatorAllowedAddresses is { Length: <= 32 } addresses
+                   && addresses.All(
+                       static value => IPAddress.TryParse(value, out _)),
+        "Operator allowed addresses must contain at most 32 literal IP addresses.")
     .ValidateOnStart();
 builder.Services.AddSingleton<AbuseProtectionService>();
+builder.Services
+    .AddOptions<AuditOptions>()
+    .BindConfiguration(AuditOptions.SectionName)
+    .ValidateDataAnnotations()
+    .ValidateOnStart();
 AbuseProtectionOptions configuredAbuseProtection = builder.Configuration
     .GetSection(AbuseProtectionOptions.SectionName)
     .Get<AbuseProtectionOptions>() ?? new AbuseProtectionOptions();
@@ -180,8 +241,13 @@ InMemoryEphemeralRendezvousStore stateStore = new(
     stateOptions,
     rendezvousClock,
     rendezvousClock);
+builder.Services.AddSingleton(stateStore);
 builder.Services.AddSingleton<IEphemeralRendezvousStore>(stateStore);
 builder.Services.AddSingleton<IWallClock>(rendezvousClock);
+builder.Services.AddSingleton<IMonotonicClock>(rendezvousClock);
+builder.Services.AddSingleton<RendezvousTelemetry>();
+builder.Services.AddSingleton<AuditTrail>();
+builder.Services.AddSingleton<RendezvousReadiness>();
 
 if (isOpenApiGeneration)
 {
@@ -214,6 +280,7 @@ else
     builder.Services.AddSingleton<JoinAttemptService>();
     builder.Services.AddSingleton<ConnectionOutcomeMetrics>();
     builder.Services.AddSingleton<ConnectionOutcomeService>();
+    builder.Services.AddSingleton<OperatorService>();
     builder.Services.AddSingleton(new ProvisioningReadiness(true));
 }
 
@@ -246,34 +313,13 @@ if (TrustedProxyForwarding.IsEnabled(configuredAbuseProtection))
 {
     app.UseForwardedHeaders();
 }
+app.UseMiddleware<TelemetryMiddleware>();
 app.UseExceptionHandler();
 app.UseMiddleware<HttpAbuseProtectionMiddleware>();
 app.MapOpenApi();
 app.MapRendezvousContractEndpoints();
-app.MapGet(
-        "/health/live",
-        static () => Results.Ok(new HealthResponse { Status = "live" }))
-    .Produces<HealthResponse>()
-    .Produces<ApiError>(StatusCodes.Status429TooManyRequests)
-    .WithName("GetLiveness")
-    .WithTags("Health");
-app.MapGet(
-    "/health/ready",
-    static (
-        UdpMediatorService mediator,
-        ProvisioningReadiness provisioning,
-        IEphemeralRendezvousStore state) =>
-        mediator.LocalEndpoint is null
-        || !provisioning.IsReady
-        || !state.IsAvailable
-        || state.IsDraining
-        ? Results.StatusCode(StatusCodes.Status503ServiceUnavailable)
-        : Results.Ok(new HealthResponse { Status = "ready" }))
-    .Produces<HealthResponse>()
-    .Produces<ApiError>(StatusCodes.Status429TooManyRequests)
-    .Produces(StatusCodes.Status503ServiceUnavailable)
-    .WithName("GetReadiness")
-    .WithTags("Health");
+app.MapOperatorEndpoints();
+app.MapRendezvousHealthEndpoints();
 
 await app.RunAsync();
 

src/FinalFactory.Rendezvous.Server/Provisioning/SigningKeyRing.cs

@@ -142,8 +142,36 @@ internal sealed class SigningKeyRing : IDisposable
         return VerificationKeyLookup.Available;
     }
 
-    public bool Revoke(string keyId) =>
-        _keys.ContainsKey(keyId) && _runtimeRevocations.TryAdd(keyId, 0);
+    public bool Revoke(string keyId)
+    {
+        if (!_keys.ContainsKey(keyId))
+        {
+            return false;
+        }
+
+        _runtimeRevocations.TryAdd(keyId, 0);
+        return true;
+    }
+
+    public IReadOnlyList<SigningKeyStatus> GetStatuses(DateTimeOffset now) => _keys.Values
+        .OrderBy(static key => key.KeyId, StringComparer.Ordinal)
+        .Select(key => new SigningKeyStatus(
+            key.KeyId,
+            IsRevoked(key)
+                ? "revoked"
+                : now < key.NotBefore
+                    ? "not-yet-valid"
+                    : now < key.SignUntil
+                        ? "signing"
+                        : now < key.VerifyUntil
+                            ? "verify-only"
+                            : "retired",
+            key.SignUntil,
+            key.VerifyUntil,
+            key.GameId,
+            key.EnvironmentId,
+            key.CredentialKinds.Select(static kind => kind.ToString()).Order().ToArray()))
+        .ToArray();
 
     public void Dispose()
     {
@@ -210,6 +238,15 @@ internal sealed class SigningKeyRing : IDisposable
     }
 }
 
+internal sealed record SigningKeyStatus(
+    string KeyId,
+    string Status,
+    DateTimeOffset SignUntil,
+    DateTimeOffset VerifyUntil,
+    string? GameId,
+    string? EnvironmentId,
+    IReadOnlyList<string> CredentialKinds);
+
 internal sealed class SigningKey : IDisposable
 {
     private byte[]? _material;

src/FinalFactory.Rendezvous.Server/State/EphemeralStateContracts.cs

@@ -297,6 +297,7 @@ internal sealed record StoredJoinAttempt
     public required SecretFingerprint ConnectionTicketFingerprint { get; init; }
     public NetworkEndpoint? DedicatedFallback { get; init; }
     public required DateTimeOffset ExpiresAt { get; init; }
+    public required TimeSpan CreatedAtMonotonic { get; init; }
     public required DateTimeOffset ConnectionTicketExpiresAt { get; init; }
     public AttemptEndpointBinding? HostEndpoint { get; init; }
     public AttemptEndpointBinding? ClientEndpoint { get; init; }

src/FinalFactory.Rendezvous.Server/State/InMemoryEphemeralRendezvousStore.cs

@@ -23,7 +23,10 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
     private TimeSpan? _drainDeadline;
     private TimeSpan _nextUdpMaintenance;
     private long _maintenanceSweepCount;
+    private long _expiryChurn;
     private bool _available = true;
+    private EphemeralStoreSnapshot? _metricsSnapshot;
+    private TimeSpan _metricsSnapshotAt = TimeSpan.MinValue;
 
     public InMemoryEphemeralRendezvousStore(
         EphemeralStoreOptions options,
@@ -66,6 +69,50 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
         }
     }
 
+    internal EphemeralStoreSnapshot GetSnapshot()
+    {
+        lock (_gate)
+        {
+            TimeSpan now = _monotonicClock.Elapsed;
+            Cleanup(now);
+            EphemeralStoreSnapshot snapshot = CreateSnapshot();
+            _metricsSnapshot = snapshot;
+            _metricsSnapshotAt = now;
+            return snapshot;
+        }
+    }
+
+    internal EphemeralStoreSnapshot GetMetricsSnapshot()
+    {
+        lock (_gate)
+        {
+            TimeSpan now = _monotonicClock.Elapsed;
+            if (_metricsSnapshot is null
+                || now < _metricsSnapshotAt
+                || now - _metricsSnapshotAt >= TimeSpan.FromMilliseconds(100))
+            {
+                Cleanup(now);
+                _metricsSnapshot = CreateSnapshot();
+                _metricsSnapshotAt = now;
+            }
+
+            return _metricsSnapshot;
+        }
+    }
+
+    private EphemeralStoreSnapshot CreateSnapshot() => new(
+        _listings.Count,
+        _presence.Count,
+        _attempts.Count,
+        _outcomeReports.Count,
+        _replay.Count,
+        _revocations.Count,
+        _idempotency.Count,
+        _maintenanceSweepCount,
+        _expiryChurn,
+        _available,
+        _drainDeadline.HasValue);
+
     public StoreResult<StoredListing> CreateListing(
         CreateListingCommand command,
         CancellationToken cancellationToken = default) => Atomic<StoredListing>(now =>
@@ -400,6 +447,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
 
         AttemptEntry attempt = new(
             command,
+            now,
             now + _options.JoinAttemptLifetime,
             WallDeadline(now, _options.JoinAttemptLifetime));
         _attempts.Add(command.AttemptId, attempt);
@@ -729,6 +777,10 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
             return new(StoreResultCode.CapacityExceeded);
         }
 
+        int activeResourcesBefore = _listings.Count
+            + _presence.Count
+            + _attempts.Count
+            + _outcomeReports.Count;
         _revocations[subject] = now + lifetime;
         SessionListingId[] listings = _listings
             .Where(item => string.Equals(item.Value.Definition.OwnerSubject, subject, StringComparison.Ordinal))
@@ -756,7 +808,11 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
             _outcomeReports.Remove(attemptId);
         }
 
-        return new(StoreResultCode.Success, listings.Length + attempts.Length);
+        int activeResourcesAfter = _listings.Count
+            + _presence.Count
+            + _attempts.Count
+            + _outcomeReports.Count;
+        return new(StoreResultCode.Success, activeResourcesBefore - activeResourcesAfter);
     }, cancellationToken);
 
     public void BeginDrain(CancellationToken cancellationToken = default)
@@ -768,6 +824,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
             if (!_drainDeadline.HasValue)
             {
                 _drainDeadline = _monotonicClock.Elapsed + _options.GracefulDrainLifetime;
+                _metricsSnapshot = null;
             }
         }
     }
@@ -778,6 +835,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
         {
             _available = false;
             ClearActiveState();
+            _metricsSnapshot = null;
         }
     }
 
@@ -801,7 +859,9 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
                 _nextUdpMaintenance = now + UdpMaintenanceInterval;
             }
 
-            return operation(now);
+            StoreResult<T> result = operation(now);
+            _metricsSnapshot = null;
+            return result;
         }
     }
 
@@ -838,44 +898,54 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
             ClearActiveState();
         }
 
-        RemoveExpired(_revocations, now);
-        RemoveExpired(_replay, now);
-        foreach (string key in _idempotency
-                     .Where(item => item.Value.Deadline <= now)
-                     .Select(static item => item.Key)
-                     .ToArray())
+        _expiryChurn += RemoveExpired(_revocations, now);
+        _expiryChurn += RemoveExpired(_replay, now);
+        string[] expiredIdempotency = _idempotency
+            .Where(item => item.Value.Deadline <= now)
+            .Select(static item => item.Key)
+            .ToArray();
+        _expiryChurn += expiredIdempotency.Length;
+        foreach (string key in expiredIdempotency)
         {
             _idempotency.Remove(key);
         }
 
-        foreach (MediationHandle handle in _presence
-                     .Where(item => item.Value.Deadline <= now)
-                     .Select(static item => item.Key)
-                     .ToArray())
+        MediationHandle[] expiredPresence = _presence
+            .Where(item => item.Value.Deadline <= now)
+            .Select(static item => item.Key)
+            .ToArray();
+        _expiryChurn += expiredPresence.Length;
+        foreach (MediationHandle handle in expiredPresence)
         {
             _presence.Remove(handle);
         }
 
-        foreach (JoinAttemptId attemptId in _attempts
-                     .Where(item => item.Value.Deadline <= now)
-                     .Select(static item => item.Key)
-                     .ToArray())
+        JoinAttemptId[] expiredAttempts = _attempts
+            .Where(item => item.Value.Deadline <= now)
+            .Select(static item => item.Key)
+            .ToArray();
+        _expiryChurn += expiredAttempts.Length;
+        foreach (JoinAttemptId attemptId in expiredAttempts)
         {
             RemoveAttempt(attemptId);
         }
 
-        foreach (JoinAttemptId attemptId in _outcomeReports
-                     .Where(item => item.Value.Deadline <= now)
-                     .Select(static item => item.Key)
-                     .ToArray())
+        JoinAttemptId[] expiredOutcomes = _outcomeReports
+            .Where(item => item.Value.Deadline <= now)
+            .Select(static item => item.Key)
+            .ToArray();
+        _expiryChurn += expiredOutcomes.Length;
+        foreach (JoinAttemptId attemptId in expiredOutcomes)
         {
             _outcomeReports.Remove(attemptId);
         }
 
-        foreach (SessionListingId listingId in _listings
-                     .Where(item => item.Value.LeaseDeadline <= now)
-                     .Select(static item => item.Key)
-                     .ToArray())
+        SessionListingId[] expiredListings = _listings
+            .Where(item => item.Value.LeaseDeadline <= now)
+            .Select(static item => item.Key)
+            .ToArray();
+        _expiryChurn += expiredListings.Length;
+        foreach (SessionListingId listingId in expiredListings)
         {
             RemoveListing(listingId);
         }
@@ -959,6 +1029,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
         ClientCapabilityFingerprint = entry.Command.ClientCapabilityFingerprint,
         ConnectionTicketFingerprint = entry.Command.ConnectionTicketFingerprint,
         DedicatedFallback = StoredListing.CopyEndpoint(entry.Command.DedicatedFallback),
+        CreatedAtMonotonic = entry.CreatedAtMonotonic,
         ExpiresAt = entry.WallExpiresAt,
         ConnectionTicketExpiresAt = entry.TicketWallExpiresAt ?? default,
         HostEndpoint = entry.HostEndpoint,
@@ -968,15 +1039,18 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
         IsCancelled = entry.IsCancelled,
     };
 
-    private static void RemoveExpired(Dictionary<string, TimeSpan> entries, TimeSpan now)
+    private static int RemoveExpired(Dictionary<string, TimeSpan> entries, TimeSpan now)
     {
-        foreach (string key in entries
-                     .Where(item => item.Value <= now)
-                     .Select(static item => item.Key)
-                     .ToArray())
+        string[] expired = entries
+            .Where(item => item.Value <= now)
+            .Select(static item => item.Key)
+            .ToArray();
+        foreach (string key in expired)
         {
             entries.Remove(key);
         }
+
+        return expired.Length;
     }
 
     private static void ValidateListing(ListingDefinition listing)
@@ -1101,10 +1175,12 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
 
     private sealed class AttemptEntry(
         CreateJoinAttemptCommand command,
+        TimeSpan createdAtMonotonic,
         TimeSpan deadline,
         DateTimeOffset wallExpiresAt)
     {
         public CreateJoinAttemptCommand Command { get; } = command;
+        public TimeSpan CreatedAtMonotonic { get; } = createdAtMonotonic;
         public SecretFingerprint HostCapabilityFingerprint { get; } = command.HostCapabilityFingerprint;
         public SecretFingerprint ClientCapabilityFingerprint { get; } = command.ClientCapabilityFingerprint;
         public SecretFingerprint ConnectionTicketFingerprint { get; } = command.ConnectionTicketFingerprint;
@@ -1137,3 +1213,16 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
         object ResourceId,
         TimeSpan Deadline);
 }
+
+internal sealed record EphemeralStoreSnapshot(
+    int ActiveListings,
+    int FreshPresenceBindings,
+    int ActiveJoinAttempts,
+    int RetainedOutcomeReports,
+    int ReplayMarkers,
+    int PrincipalRevocations,
+    int IdempotencyEntries,
+    long MaintenanceSweeps,
+    long ExpiryChurn,
+    bool IsAvailable,
+    bool IsDraining);

src/FinalFactory.Rendezvous.Server/Transport/NatMediationProcessor.cs

@@ -1,8 +1,10 @@
+using System.Diagnostics;
 using System.Net;
 using System.Net.Sockets;
 using FinalFactory.Rendezvous.Contracts;
 using FinalFactory.Rendezvous.Server.Abuse;
 using FinalFactory.Rendezvous.Server.JoinAttempts;
+using FinalFactory.Rendezvous.Server.Observability;
 using FinalFactory.Rendezvous.Server.Sessions;
 using FinalFactory.Rendezvous.Server.State;
 
@@ -38,7 +40,9 @@ internal sealed class NatMediationProcessor(
     IEphemeralRendezvousStore store,
     ISessionCapabilityService capabilities,
     JoinAttemptService joinAttempts,
-    AbuseProtectionService? abuseProtection = null)
+    AbuseProtectionService? abuseProtection = null,
+    RendezvousTelemetry? telemetry = null,
+    IMonotonicClock? monotonicClock = null)
 {
     public NatMediationResult ProcessDatagram(
         ReadOnlySpan<byte> encoded,
@@ -68,6 +72,26 @@ internal sealed class NatMediationProcessor(
         INatIntroductionSink introductionSink,
         CancellationToken cancellationToken = default)
     {
+        long started = Stopwatch.GetTimestamp();
+        using Activity? activity = telemetry?.StartActivity("UDP frozen", ActivityKind.Server);
+        NatMediationResult result = ProcessDatagramCore(
+            encoded,
+            observedPublicEndpoint,
+            introductionSink,
+            cancellationToken);
+        telemetry?.RecordUdp(
+            "frozen",
+            result.ToString(),
+            Stopwatch.GetElapsedTime(started).TotalMilliseconds);
+        return result;
+    }
+
+    private NatMediationResult ProcessDatagramCore(
+        ReadOnlySpan<byte> encoded,
+        IPEndPoint observedPublicEndpoint,
+        INatIntroductionSink introductionSink,
+        CancellationToken cancellationToken)
+    {
 
         if (!RendezvousUdpCodec.TryDecode(encoded, out PresenceDatagram? datagram, out _)
             || datagram is null
@@ -141,12 +165,22 @@ internal sealed class NatMediationProcessor(
         IPEndPoint observedPublicEndpoint,
         string token,
         INatIntroductionSink introductionSink,
-        CancellationToken cancellationToken = default) => ProcessRequestCore(
+        CancellationToken cancellationToken = default)
+    {
+        long started = Stopwatch.GetTimestamp();
+        using Activity? activity = telemetry?.StartActivity("UDP litenet", ActivityKind.Server);
+        NatMediationResult result = ProcessRequestCore(
             claimedLocalEndpoint,
             observedPublicEndpoint,
             token,
             introductionSink,
             cancellationToken);
+        telemetry?.RecordUdp(
+            "litenet",
+            result.ToString(),
+            Stopwatch.GetElapsedTime(started).TotalMilliseconds);
+        return result;
+    }
 
     private NatMediationResult ProcessRequestCore(
         IPEndPoint claimedLocalEndpoint,
@@ -255,6 +289,14 @@ internal sealed class NatMediationProcessor(
         try
         {
             introductionSink.Introduce(CreatePlan(consumed.Value, ticket.Value));
+            if (telemetry is not null && monotonicClock is not null)
+            {
+                telemetry.RecordPairingLatency(Math.Max(
+                    0,
+                    (monotonicClock.Elapsed - consumed.Value.Attempt.CreatedAtMonotonic)
+                    .TotalMilliseconds));
+            }
+
             return NatMediationResult.Introduced;
         }
         catch (Exception exception) when (exception is SocketException

src/FinalFactory.Rendezvous.Server/appsettings.Development.json

@@ -1,5 +1,8 @@
 {
   "Rendezvous": {
+    "AbuseProtection": {
+      "OperatorAllowedAddresses": ["127.0.0.1", "::1"]
+    },
     "Provisioning": {
       "Issuer": "final-factory-rendezvous-development",
       "Audience": "final-factory-rendezvous",
@@ -14,6 +17,14 @@
           "NotBefore": "2025-01-01T00:00:00Z",
           "SignUntil": "2035-01-01T00:00:00Z",
           "VerifyUntil": "2035-01-02T00:00:00Z"
+        },
+        {
+          "KeyId": "development-operator-1",
+          "SecretReference": "development:ephemeral/rendezvous-operator-signing",
+          "CredentialKinds": ["Operator"],
+          "NotBefore": "2025-01-01T00:00:00Z",
+          "SignUntil": "2035-01-01T00:00:00Z",
+          "VerifyUntil": "2035-01-02T00:00:00Z"
         }
       ],
       "Games": [

src/FinalFactory.Rendezvous.Server/appsettings.json

@@ -6,16 +6,25 @@
       "MaxDatagramsPerPoll": 256,
       "PollIntervalMilliseconds": 2
     },
+    "Audit": {
+      "MaxEntries": 10000,
+      "RetentionDays": 30
+    },
     "AbuseProtection": {
       "WindowSeconds": 1,
       "MaxTrackedKeys": 100000,
       "CriticalTrackedKeyReserve": 2048,
       "UdpTrackedKeyLimit": 70000,
       "TrustedProxyAddresses": [],
+      "OperatorAllowedAddresses": [],
       "HealthGlobalRequestsPerWindow": 1000,
       "HealthGlobalConcurrency": 32,
       "HealthIpPrefixRequestsPerWindow": 120,
       "HealthIpPrefixConcurrency": 8,
+      "OperatorGlobalRequestsPerWindow": 1000,
+      "OperatorGlobalConcurrency": 32,
+      "OperatorIpPrefixRequestsPerWindow": 120,
+      "OperatorIpPrefixConcurrency": 8,
       "HttpGlobalRequestsPerWindow": 20000,
       "HttpOptionalRequestsPerWindow": 18000,
       "HttpIpPrefixRequestsPerWindow": 500,