feat(server): add observability and operator controls (#16)

src/FinalFactory.Rendezvous.Server/State/EphemeralStateContracts.cs

@@ -297,6 +297,7 @@ internal sealed record StoredJoinAttempt
     public required SecretFingerprint ConnectionTicketFingerprint { get; init; }
     public NetworkEndpoint? DedicatedFallback { get; init; }
     public required DateTimeOffset ExpiresAt { get; init; }
+    public required TimeSpan CreatedAtMonotonic { get; init; }
     public required DateTimeOffset ConnectionTicketExpiresAt { get; init; }
     public AttemptEndpointBinding? HostEndpoint { get; init; }
     public AttemptEndpointBinding? ClientEndpoint { get; init; }

src/FinalFactory.Rendezvous.Server/State/InMemoryEphemeralRendezvousStore.cs

@@ -23,7 +23,10 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
     private TimeSpan? _drainDeadline;
     private TimeSpan _nextUdpMaintenance;
     private long _maintenanceSweepCount;
+    private long _expiryChurn;
     private bool _available = true;
+    private EphemeralStoreSnapshot? _metricsSnapshot;
+    private TimeSpan _metricsSnapshotAt = TimeSpan.MinValue;
 
     public InMemoryEphemeralRendezvousStore(
         EphemeralStoreOptions options,
@@ -66,6 +69,50 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
         }
     }
 
+    internal EphemeralStoreSnapshot GetSnapshot()
+    {
+        lock (_gate)
+        {
+            TimeSpan now = _monotonicClock.Elapsed;
+            Cleanup(now);
+            EphemeralStoreSnapshot snapshot = CreateSnapshot();
+            _metricsSnapshot = snapshot;
+            _metricsSnapshotAt = now;
+            return snapshot;
+        }
+    }
+
+    internal EphemeralStoreSnapshot GetMetricsSnapshot()
+    {
+        lock (_gate)
+        {
+            TimeSpan now = _monotonicClock.Elapsed;
+            if (_metricsSnapshot is null
+                || now < _metricsSnapshotAt
+                || now - _metricsSnapshotAt >= TimeSpan.FromMilliseconds(100))
+            {
+                Cleanup(now);
+                _metricsSnapshot = CreateSnapshot();
+                _metricsSnapshotAt = now;
+            }
+
+            return _metricsSnapshot;
+        }
+    }
+
+    private EphemeralStoreSnapshot CreateSnapshot() => new(
+        _listings.Count,
+        _presence.Count,
+        _attempts.Count,
+        _outcomeReports.Count,
+        _replay.Count,
+        _revocations.Count,
+        _idempotency.Count,
+        _maintenanceSweepCount,
+        _expiryChurn,
+        _available,
+        _drainDeadline.HasValue);
+
     public StoreResult<StoredListing> CreateListing(
         CreateListingCommand command,
         CancellationToken cancellationToken = default) => Atomic<StoredListing>(now =>
@@ -400,6 +447,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
 
         AttemptEntry attempt = new(
             command,
+            now,
             now + _options.JoinAttemptLifetime,
             WallDeadline(now, _options.JoinAttemptLifetime));
         _attempts.Add(command.AttemptId, attempt);
@@ -729,6 +777,10 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
             return new(StoreResultCode.CapacityExceeded);
         }
 
+        int activeResourcesBefore = _listings.Count
+            + _presence.Count
+            + _attempts.Count
+            + _outcomeReports.Count;
         _revocations[subject] = now + lifetime;
         SessionListingId[] listings = _listings
             .Where(item => string.Equals(item.Value.Definition.OwnerSubject, subject, StringComparison.Ordinal))
@@ -756,7 +808,11 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
             _outcomeReports.Remove(attemptId);
         }
 
-        return new(StoreResultCode.Success, listings.Length + attempts.Length);
+        int activeResourcesAfter = _listings.Count
+            + _presence.Count
+            + _attempts.Count
+            + _outcomeReports.Count;
+        return new(StoreResultCode.Success, activeResourcesBefore - activeResourcesAfter);
     }, cancellationToken);
 
     public void BeginDrain(CancellationToken cancellationToken = default)
@@ -768,6 +824,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
             if (!_drainDeadline.HasValue)
             {
                 _drainDeadline = _monotonicClock.Elapsed + _options.GracefulDrainLifetime;
+                _metricsSnapshot = null;
             }
         }
     }
@@ -778,6 +835,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
         {
             _available = false;
             ClearActiveState();
+            _metricsSnapshot = null;
         }
     }
 
@@ -801,7 +859,9 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
                 _nextUdpMaintenance = now + UdpMaintenanceInterval;
             }
 
-            return operation(now);
+            StoreResult<T> result = operation(now);
+            _metricsSnapshot = null;
+            return result;
         }
     }
 
@@ -838,44 +898,54 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
             ClearActiveState();
         }
 
-        RemoveExpired(_revocations, now);
-        RemoveExpired(_replay, now);
-        foreach (string key in _idempotency
-                     .Where(item => item.Value.Deadline <= now)
-                     .Select(static item => item.Key)
-                     .ToArray())
+        _expiryChurn += RemoveExpired(_revocations, now);
+        _expiryChurn += RemoveExpired(_replay, now);
+        string[] expiredIdempotency = _idempotency
+            .Where(item => item.Value.Deadline <= now)
+            .Select(static item => item.Key)
+            .ToArray();
+        _expiryChurn += expiredIdempotency.Length;
+        foreach (string key in expiredIdempotency)
         {
             _idempotency.Remove(key);
         }
 
-        foreach (MediationHandle handle in _presence
-                     .Where(item => item.Value.Deadline <= now)
-                     .Select(static item => item.Key)
-                     .ToArray())
+        MediationHandle[] expiredPresence = _presence
+            .Where(item => item.Value.Deadline <= now)
+            .Select(static item => item.Key)
+            .ToArray();
+        _expiryChurn += expiredPresence.Length;
+        foreach (MediationHandle handle in expiredPresence)
         {
             _presence.Remove(handle);
         }
 
-        foreach (JoinAttemptId attemptId in _attempts
-                     .Where(item => item.Value.Deadline <= now)
-                     .Select(static item => item.Key)
-                     .ToArray())
+        JoinAttemptId[] expiredAttempts = _attempts
+            .Where(item => item.Value.Deadline <= now)
+            .Select(static item => item.Key)
+            .ToArray();
+        _expiryChurn += expiredAttempts.Length;
+        foreach (JoinAttemptId attemptId in expiredAttempts)
         {
             RemoveAttempt(attemptId);
         }
 
-        foreach (JoinAttemptId attemptId in _outcomeReports
-                     .Where(item => item.Value.Deadline <= now)
-                     .Select(static item => item.Key)
-                     .ToArray())
+        JoinAttemptId[] expiredOutcomes = _outcomeReports
+            .Where(item => item.Value.Deadline <= now)
+            .Select(static item => item.Key)
+            .ToArray();
+        _expiryChurn += expiredOutcomes.Length;
+        foreach (JoinAttemptId attemptId in expiredOutcomes)
         {
             _outcomeReports.Remove(attemptId);
         }
 
-        foreach (SessionListingId listingId in _listings
-                     .Where(item => item.Value.LeaseDeadline <= now)
-                     .Select(static item => item.Key)
-                     .ToArray())
+        SessionListingId[] expiredListings = _listings
+            .Where(item => item.Value.LeaseDeadline <= now)
+            .Select(static item => item.Key)
+            .ToArray();
+        _expiryChurn += expiredListings.Length;
+        foreach (SessionListingId listingId in expiredListings)
         {
             RemoveListing(listingId);
         }
@@ -959,6 +1029,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
         ClientCapabilityFingerprint = entry.Command.ClientCapabilityFingerprint,
         ConnectionTicketFingerprint = entry.Command.ConnectionTicketFingerprint,
         DedicatedFallback = StoredListing.CopyEndpoint(entry.Command.DedicatedFallback),
+        CreatedAtMonotonic = entry.CreatedAtMonotonic,
         ExpiresAt = entry.WallExpiresAt,
         ConnectionTicketExpiresAt = entry.TicketWallExpiresAt ?? default,
         HostEndpoint = entry.HostEndpoint,
@@ -968,15 +1039,18 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
         IsCancelled = entry.IsCancelled,
     };
 
-    private static void RemoveExpired(Dictionary<string, TimeSpan> entries, TimeSpan now)
+    private static int RemoveExpired(Dictionary<string, TimeSpan> entries, TimeSpan now)
     {
-        foreach (string key in entries
-                     .Where(item => item.Value <= now)
-                     .Select(static item => item.Key)
-                     .ToArray())
+        string[] expired = entries
+            .Where(item => item.Value <= now)
+            .Select(static item => item.Key)
+            .ToArray();
+        foreach (string key in expired)
         {
             entries.Remove(key);
         }
+
+        return expired.Length;
     }
 
     private static void ValidateListing(ListingDefinition listing)
@@ -1101,10 +1175,12 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
 
     private sealed class AttemptEntry(
         CreateJoinAttemptCommand command,
+        TimeSpan createdAtMonotonic,
         TimeSpan deadline,
         DateTimeOffset wallExpiresAt)
     {
         public CreateJoinAttemptCommand Command { get; } = command;
+        public TimeSpan CreatedAtMonotonic { get; } = createdAtMonotonic;
         public SecretFingerprint HostCapabilityFingerprint { get; } = command.HostCapabilityFingerprint;
         public SecretFingerprint ClientCapabilityFingerprint { get; } = command.ClientCapabilityFingerprint;
         public SecretFingerprint ConnectionTicketFingerprint { get; } = command.ConnectionTicketFingerprint;
@@ -1137,3 +1213,16 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
         object ResourceId,
         TimeSpan Deadline);
 }
+
+internal sealed record EphemeralStoreSnapshot(
+    int ActiveListings,
+    int FreshPresenceBindings,
+    int ActiveJoinAttempts,
+    int RetainedOutcomeReports,
+    int ReplayMarkers,
+    int PrincipalRevocations,
+    int IdempotencyEntries,
+    long MaintenanceSweeps,
+    long ExpiryChurn,
+    bool IsAvailable,
+    bool IsDraining);