feat(operations): add capacity and resilience gates (#18)

2026-07-16 15:57:01 +02:00
parent 08729ae25c
commit 609dad7cf1
21 changed files with 1759 additions and 107 deletions
@@ -0,0 +1,580 @@
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Runtime;
+using System.Runtime.InteropServices;
+using FinalFactory.Rendezvous.Contracts;
+using FinalFactory.Rendezvous.Server.Observability;
+using FinalFactory.Rendezvous.Server.State;
+
+namespace FinalFactory.Rendezvous.Capacity;
+
+internal static class CapacityRunner
+{
+    private static readonly TenantScope Scope = new(new("space-game"), new("production"));
+    private const uint ProtocolVersion = 1;
+
+    public static Task<CapacityReport> RunAsync(CapacityOptions options)
+    {
+        ArgumentNullException.ThrowIfNull(options);
+        Process process = Process.GetCurrentProcess();
+        TimeSpan cpuBefore = process.TotalProcessorTime;
+        Stopwatch capacityPhaseTime = Stopwatch.StartNew();
+        List<string> failures = [];
+        List<CapacityMeasurement> measurements = [];
+        ManualClock clock = new();
+        InMemoryEphemeralRendezvousStore store = CreateStore(options, clock);
+        List<StoredListing> listings = new(options.Listings);
+        List<CreateJoinAttemptCommand> attempts = new(options.Attempts);
+        int registrationSamples = Math.Min(options.Samples, options.Listings);
+        int attemptSamples = Math.Min(options.Samples, options.Attempts);
+
+        for (int index = 0; index < options.Listings - registrationSamples; index++)
+        {
+            listings.Add(CreateVisibleListing(store, index));
+        }
+
+        measurements.Add(Measure(
+            "registration-and-presence",
+            registrationSamples,
+            budgetMilliseconds: 200,
+            minimumOperationsPerSecond: 200,
+            index => listings.Add(CreateVisibleListing(
+                store,
+                options.Listings - registrationSamples + index))));
+
+        measurements.Add(Measure(
+            "lease-renewal",
+            registrationSamples,
+            budgetMilliseconds: 200,
+            minimumOperationsPerSecond: 200,
+            index =>
+            {
+                StoredListing listing = listings[index];
+                StoreResult<StoredListing> renewed = store.RenewLease(new(
+                    listing.Definition.ListingId,
+                    listing.Definition.LeaseId,
+                    listing.Definition.LeaseFingerprint,
+                    listing.Definition.OwnerSubject,
+                    listing.Version));
+                RequireSuccess(renewed, "renewal");
+                listings[index] = renewed.Value!;
+            }));
+
+        int browseSamples = Math.Min(options.Samples, 250);
+        measurements.Add(Measure(
+            "visible-session-browse",
+            browseSamples,
+            budgetMilliseconds: 200,
+            minimumOperationsPerSecond: 200,
+            _ => RequireSuccess(
+                store.BrowseVisibleListings(new(
+                    Scope,
+                    ProtocolVersion,
+                    new RegionId("eu-central"),
+                    ContractLimits.BrowserPageMaxItems,
+                    ExcludeFull: true)),
+                "browse")));
+
+        for (int index = 0; index < options.Attempts - attemptSamples; index++)
+        {
+            CreateJoinAttemptCommand command = CreateAttempt(index, listings[index % listings.Count]);
+            RequireSuccess(store.CreateJoinAttempt(command), "join issuance");
+            attempts.Add(command);
+        }
+
+        measurements.Add(Measure(
+            "join-attempt-issuance",
+            attemptSamples,
+            budgetMilliseconds: 200,
+            minimumOperationsPerSecond: 200,
+            index =>
+            {
+                int sequence = options.Attempts - attemptSamples + index;
+                CreateJoinAttemptCommand command = CreateAttempt(
+                    sequence,
+                    listings[sequence % listings.Count]);
+                RequireSuccess(store.CreateJoinAttempt(command), "join issuance");
+                attempts.Add(command);
+            }));
+
+        int punchSamples = Math.Min(options.Samples, attempts.Count);
+        measurements.Add(MeasureConcurrentPunch(store, attempts, punchSamples));
+        EphemeralStoreSnapshot peak = store.GetSnapshot();
+
+        StoreResult<StoredJoinAttempt> overloaded = store.CreateJoinAttempt(
+            CreateAttempt(options.Attempts + 1, listings[^1]));
+        bool overloadWasTyped = peak.ActiveJoinAttempts == options.Attempts
+            && overloaded.Code == StoreResultCode.CapacityExceeded;
+        if (!overloadWasTyped)
+        {
+            failures.Add(
+                $"Expected typed CapacityExceeded at {options.Attempts} active attempts, "
+                + $"observed count={peak.ActiveJoinAttempts}, result={overloaded.Code}.");
+        }
+
+        int revocationSamples = Math.Min(Math.Max(1, options.Samples / 20), listings.Count / 2);
+        measurements.Add(Measure(
+            "principal-revocation",
+            revocationSamples,
+            budgetMilliseconds: 200,
+            minimumOperationsPerSecond: 50,
+            index => RequireSuccess(
+                store.RevokePrincipal(
+                    listings[index].Definition.OwnerSubject,
+                    TimeSpan.FromMinutes(1)),
+                "principal revocation")));
+
+        using (RendezvousTelemetry telemetry = new(store))
+        {
+            measurements.Add(Measure(
+                "telemetry-recording",
+                Math.Max(100, options.Samples),
+                budgetMilliseconds: 1,
+                minimumOperationsPerSecond: 10_000,
+                _ =>
+                {
+                    telemetry.RecordHttp("browse", 200, 1);
+                    telemetry.RecordUdp("contribution", "accepted", 1);
+                    telemetry.RecordPairingLatency(2);
+                }));
+        }
+
+        clock.Advance(TimeSpan.FromSeconds(61));
+        EphemeralStoreSnapshot? afterCoincidentExpiry = null;
+        measurements.Add(Measure(
+            "coincident-listing-attempt-expiry",
+            samples: 1,
+            budgetMilliseconds: 200,
+            minimumOperationsPerSecond: 0,
+            _ => afterCoincidentExpiry = store.GetSnapshot()));
+        if (afterCoincidentExpiry!.ActiveJoinAttempts != 0
+            || afterCoincidentExpiry.ActiveListings != 0)
+        {
+            failures.Add("Coincident 60-second cleanup retained expired listings or join attempts.");
+        }
+
+        clock.Advance(TimeSpan.FromSeconds(90));
+        _ = store.GetSnapshot();
+        StoredListing recoveryListing = CreateVisibleListing(store, options.Listings + 1);
+        StoreResult<StoredJoinAttempt> recovered = store.CreateJoinAttempt(
+            CreateAttempt(options.Attempts + 2, recoveryListing));
+        bool recoverySucceeded = recovered.Succeeded;
+        if (!recoverySucceeded)
+        {
+            failures.Add($"Store did not recover after attempt expiry: {recovered.Code}.");
+        }
+
+        clock.Advance(TimeSpan.FromSeconds(151));
+        EphemeralStoreSnapshot final = store.GetSnapshot();
+        if (final.ActiveListings != 0
+            || final.ActiveJoinAttempts != 0
+            || final.ReplayMarkers != 0
+            || final.IdempotencyEntries != 0
+            || final.RetainedOutcomeReports != 0)
+        {
+            failures.Add("Expiry cleanup left active or retained state after every configured deadline.");
+        }
+
+        capacityPhaseTime.Stop();
+        process.Refresh();
+        double capacityPhaseCpuPercent = 100
+            * (process.TotalProcessorTime - cpuBefore).TotalSeconds
+            / Math.Max(capacityPhaseTime.Elapsed.TotalSeconds * Environment.ProcessorCount, 0.000_001);
+        if (options.Profile == "candidate" && capacityPhaseCpuPercent > 70)
+        {
+            failures.Add(
+                $"Capacity-phase CPU {capacityPhaseCpuPercent:F1}% exceeded the 70% candidate budget.");
+        }
+
+        SoakEvidence soak = RunAcceleratedSoak(options, failures);
+        ManualClock restartClock = new();
+        EphemeralStoreSnapshot restarted = CreateStore(options, restartClock).GetSnapshot();
+        bool restartStartedEmpty = restarted.ActiveListings == 0
+            && restarted.ActiveJoinAttempts == 0
+            && restarted.ReplayMarkers == 0;
+        if (!restartStartedEmpty)
+        {
+            failures.Add("A restarted store did not begin empty.");
+        }
+
+        foreach (CapacityMeasurement measurement in measurements.Where(static item => !item.Passed))
+        {
+            failures.Add(
+                $"{measurement.Operation} missed its budget: p95={measurement.P95Milliseconds:F3} ms, "
+                + $"rate={measurement.OperationsPerSecond:F1}/s.");
+        }
+
+        GC.Collect();
+        GC.WaitForPendingFinalizers();
+        GC.Collect();
+        process.Refresh();
+        long managedAfterCleanup = GC.GetTotalMemory(forceFullCollection: true);
+        long memoryBudget = 1_610_612_736;
+        if (process.PeakWorkingSet64 > memoryBudget)
+        {
+            failures.Add(
+                $"Peak working set {process.PeakWorkingSet64} exceeded the 1.5 GiB profile budget.");
+        }
+
+        if (options.Profile == "candidate" && Environment.ProcessorCount != 2)
+        {
+            failures.Add(
+                $"Candidate evidence must expose exactly two CPUs; runtime exposed "
+                + $"{Environment.ProcessorCount}.");
+        }
+
+        CapacityReport report = new()
+        {
+            SchemaVersion = 2,
+            EvidenceVersion = "v2",
+            GeneratedAt = DateTimeOffset.UtcNow,
+            Profile = options.Profile,
+            Runtime = new(
+                RuntimeInformation.FrameworkDescription,
+                RuntimeInformation.OSDescription,
+                Environment.OSVersion.VersionString,
+                RuntimeInformation.ProcessArchitecture.ToString(),
+                ReadCpuModel(),
+                Environment.ProcessorCount,
+                Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_CPUSET") ?? "unrestricted",
+                ReadCgroupValue("/sys/fs/cgroup/cpu.max"),
+                ReadCgroupValue("/sys/fs/cgroup/memory.max"),
+                GCSettings.IsServerGC ? "server" : "workstation",
+                Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_COMMIT") ?? "unrecorded",
+                Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_TREE_STATE") ?? "unrecorded",
+                Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_COMMAND") ?? "unrecorded",
+                Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_IMAGE_DIGEST") ?? "not-containerized",
+                "fixed-sequences-random-identifiers",
+                capacityPhaseCpuPercent,
+                process.PeakWorkingSet64,
+                managedAfterCleanup),
+            Targets = new(
+                options.Listings,
+                options.Attempts,
+                200,
+                2_000,
+                200,
+                100,
+                70,
+                memoryBudget,
+                options.SoakCycles,
+                options.SoakSeconds),
+            Measurements = measurements,
+            State = new(
+                peak.ActiveListings,
+                peak.ActiveJoinAttempts,
+                peak.ReplayMarkers,
+                final.ActiveListings,
+                final.ActiveJoinAttempts,
+                final.ReplayMarkers,
+                final.ExpiryChurn,
+                final.MaintenanceSweeps,
+                soak.Cycles,
+                soak.Duration.TotalSeconds,
+                soak.PeakScheduledExpiryEntries,
+                soak.ManagedGrowthBytes,
+                soak.HandleGrowth,
+                restartStartedEmpty,
+                overloadWasTyped,
+                recoverySucceeded),
+            Failures = failures,
+            Passed = failures.Count == 0,
+        };
+        return Task.FromResult(report);
+    }
+
+    private static InMemoryEphemeralRendezvousStore CreateStore(
+        CapacityOptions options,
+        ManualClock clock) => new(
+        new EphemeralStoreOptions
+        {
+            MaxListings = options.Listings,
+            MaxPresenceBindings = options.Listings,
+            MaxJoinAttempts = options.Attempts,
+            MaxOutcomeReports = options.Attempts,
+            MaxIdempotencyEntries = options.Listings + options.Attempts + 1,
+        },
+        clock,
+        clock);
+
+    private static StoredListing CreateVisibleListing(
+        InMemoryEphemeralRendezvousStore store,
+        int sequence)
+    {
+        string owner = $"publisher-{sequence}";
+        SecretFingerprint leaseFingerprint = new($"lease-{sequence}");
+        SecretFingerprint presenceFingerprint = new($"presence-{sequence}");
+        ListingDefinition definition = new()
+        {
+            ListingId = new(Guid.NewGuid()),
+            LeaseId = new(Guid.NewGuid()),
+            Scope = Scope,
+            OwnerSubject = owner,
+            RegionId = new("eu-central"),
+            ProtocolVersion = ProtocolVersion,
+            BuildVersion = "1.0.0",
+            DisplayName = $"Capacity host {sequence}",
+            Visibility = ListingVisibility.Public,
+            TrustMode = PublisherTrustMode.ManagedDedicated,
+            CurrentPlayers = 1,
+            MaximumPlayers = 8,
+            Metadata = new Dictionary<string, string>(StringComparer.Ordinal),
+            LeaseFingerprint = leaseFingerprint,
+            HostPresenceHandle = new(Guid.NewGuid()),
+            HostPresenceFingerprint = presenceFingerprint,
+            CapabilityDerivationSalt = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+        };
+        StoreResult<StoredListing> created = store.CreateListing(new(
+            $"register-{sequence}",
+            $"register-request-{sequence}",
+            definition));
+        RequireSuccess(created, "registration");
+        StoreResult<StoredListing> bound = store.BindHostPresence(new(
+            definition.HostPresenceHandle,
+            presenceFingerprint,
+            PublicEndpoint(10_000 + sequence % 50_000),
+            null));
+        RequireSuccess(bound, "host presence");
+        return bound.Value!;
+    }
+
+    private static CreateJoinAttemptCommand CreateAttempt(int sequence, StoredListing listing) => new()
+    {
+        IdempotencyOwner = $"client-{sequence}",
+        IdempotencyKey = $"join-{sequence}",
+        RequestFingerprint = $"join-request-{sequence}",
+        ClientSubject = $"client-{sequence}",
+        AttemptId = new(Guid.NewGuid()),
+        MediationHandle = new(Guid.NewGuid()),
+        Scope = Scope,
+        ListingId = listing.Definition.ListingId,
+        ProtocolVersion = ProtocolVersion,
+        HostCapabilityFingerprint = new($"host-capability-{sequence}"),
+        ClientCapabilityFingerprint = new($"client-capability-{sequence}"),
+        ConnectionTicketFingerprint = new($"ticket-{sequence}"),
+        CapabilityDerivationSalt = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+    };
+
+    private static CapacityMeasurement MeasureConcurrentPunch(
+        InMemoryEphemeralRendezvousStore store,
+        List<CreateJoinAttemptCommand> attempts,
+        int samples)
+    {
+        ConcurrentBag<double> latencies = [];
+        Stopwatch total = Stopwatch.StartNew();
+        Parallel.ForEach(
+            Enumerable.Range(0, samples),
+            new ParallelOptions { MaxDegreeOfParallelism = Math.Min(64, Environment.ProcessorCount * 4) },
+            index =>
+            {
+                CreateJoinAttemptCommand attempt = attempts[index];
+                Stopwatch elapsed = Stopwatch.StartNew();
+                RequireSuccess(store.BindAttemptEndpoint(new(
+                    attempt.MediationHandle,
+                    AttemptPeerRole.Host,
+                    attempt.HostCapabilityFingerprint,
+                    PublicEndpoint(20_000 + index % 20_000),
+                    null)), "host punch");
+                RequireSuccess(store.BindAttemptEndpoint(new(
+                    attempt.MediationHandle,
+                    AttemptPeerRole.Client,
+                    attempt.ClientCapabilityFingerprint,
+                    PublicEndpoint(40_000 + index % 20_000),
+                    null)), "client punch");
+                RequireSuccess(store.ConsumeIntroduction(attempt.MediationHandle), "introduction");
+                latencies.Add(elapsed.Elapsed.TotalMilliseconds);
+            });
+        total.Stop();
+        return BuildMeasurement(
+            "simultaneous-punch-pairing",
+            latencies.ToArray(),
+            total.Elapsed,
+            budgetMilliseconds: 100,
+            minimumOperationsPerSecond: 2_000);
+    }
+
+    private static CapacityMeasurement Measure(
+        string operation,
+        int samples,
+        double budgetMilliseconds,
+        double minimumOperationsPerSecond,
+        Action<int> action)
+    {
+        double[] latencies = new double[samples];
+        Stopwatch total = Stopwatch.StartNew();
+        for (int index = 0; index < samples; index++)
+        {
+            long started = Stopwatch.GetTimestamp();
+            action(index);
+            latencies[index] = Stopwatch.GetElapsedTime(started).TotalMilliseconds;
+        }
+
+        total.Stop();
+        return BuildMeasurement(
+            operation,
+            latencies,
+            total.Elapsed,
+            budgetMilliseconds,
+            minimumOperationsPerSecond);
+    }
+
+    private static CapacityMeasurement BuildMeasurement(
+        string operation,
+        double[] latencies,
+        TimeSpan elapsed,
+        double budgetMilliseconds,
+        double minimumOperationsPerSecond)
+    {
+        Array.Sort(latencies);
+        double operationsPerSecond = latencies.Length / Math.Max(elapsed.TotalSeconds, 0.000_001);
+        double p95 = Percentile(latencies, 0.95);
+        return new(
+            operation,
+            latencies.Length,
+            Percentile(latencies, 0.50),
+            p95,
+            Percentile(latencies, 0.99),
+            operationsPerSecond,
+            minimumOperationsPerSecond,
+            budgetMilliseconds,
+            p95 <= budgetMilliseconds && operationsPerSecond >= minimumOperationsPerSecond);
+    }
+
+    private static double Percentile(double[] sorted, double percentile)
+    {
+        int index = Math.Clamp((int)Math.Ceiling(sorted.Length * percentile) - 1, 0, sorted.Length - 1);
+        return sorted[index];
+    }
+
+    private static SoakEvidence RunAcceleratedSoak(
+        CapacityOptions options,
+        List<string> failures)
+    {
+        GC.Collect();
+        GC.WaitForPendingFinalizers();
+        GC.Collect();
+        long managedBefore = GC.GetTotalMemory(forceFullCollection: true);
+        int handlesBefore = Process.GetCurrentProcess().HandleCount;
+        ManualClock clock = new();
+        CapacityOptions soakOptions = options with { Listings = 100, Attempts = 100 };
+        InMemoryEphemeralRendezvousStore store = CreateStore(soakOptions, clock);
+        Stopwatch elapsed = Stopwatch.StartNew();
+        int cycle = 0;
+        int peakScheduledExpiryEntries = 0;
+        while (cycle < options.SoakCycles
+               || elapsed.Elapsed < TimeSpan.FromSeconds(options.SoakSeconds))
+        {
+            StoredListing listing = CreateVisibleListing(store, cycle);
+            for (int refresh = 0; refresh < 10; refresh++)
+            {
+                clock.Advance(TimeSpan.FromTicks(1));
+                listing = RequireSuccess(store.RenewLease(new(
+                    listing.Definition.ListingId,
+                    listing.Definition.LeaseId,
+                    listing.Definition.LeaseFingerprint,
+                    listing.Definition.OwnerSubject,
+                    listing.Version)), "soak lease refresh");
+                listing = RequireSuccess(store.BindHostPresence(new(
+                    listing.Definition.HostPresenceHandle,
+                    listing.Definition.HostPresenceFingerprint,
+                    PublicEndpoint(10_000 + cycle % 50_000),
+                    null)), "soak presence refresh");
+            }
+
+            CreateJoinAttemptCommand attempt = CreateAttempt(cycle, listing);
+            RequireSuccess(store.CreateJoinAttempt(attempt), "soak join issuance");
+            RequireSuccess(store.ConsumeReplay(new("capacity-soak", $"replay-{cycle}")), "soak replay");
+            peakScheduledExpiryEntries = Math.Max(
+                peakScheduledExpiryEntries,
+                store.ScheduledExpiryEntryCount);
+            if (store.ScheduledExpiryEntryCount > 7)
+            {
+                failures.Add(
+                    $"Mutable deadline refresh grew the expiry queue to "
+                    + $"{store.ScheduledExpiryEntryCount} entries for one lifecycle.");
+                break;
+            }
+
+            clock.Advance(TimeSpan.FromSeconds(151));
+            EphemeralStoreSnapshot snapshot = store.GetSnapshot();
+            if (snapshot.ActiveListings != 0
+                || snapshot.ActiveJoinAttempts != 0
+                || snapshot.ReplayMarkers != 0
+                || snapshot.IdempotencyEntries != 0
+                || snapshot.RetainedOutcomeReports != 0)
+            {
+                failures.Add($"Accelerated soak retained state after cycle {cycle}.");
+                break;
+            }
+
+            cycle++;
+        }
+
+        elapsed.Stop();
+        GC.Collect();
+        GC.WaitForPendingFinalizers();
+        GC.Collect();
+        long managedGrowth = GC.GetTotalMemory(forceFullCollection: true) - managedBefore;
+        int handleGrowth = Process.GetCurrentProcess().HandleCount - handlesBefore;
+        if (managedGrowth > 67_108_864)
+        {
+            failures.Add($"Soak retained {managedGrowth} managed bytes; budget is 64 MiB.");
+        }
+
+        if (handleGrowth > 8)
+        {
+            failures.Add($"Soak retained {handleGrowth} process handles; budget is 8.");
+        }
+
+        return new(cycle, elapsed.Elapsed, peakScheduledExpiryEntries, managedGrowth, handleGrowth);
+    }
+
+    private static ObservedEndpoint PublicEndpoint(int port) =>
+        new(AddressFamilyKind.Ipv4, "203.0.113.10", port);
+
+    private static T RequireSuccess<T>(StoreResult<T> result, string operation)
+    {
+        if (!result.Succeeded)
+        {
+            throw new InvalidOperationException($"{operation} failed with {result.Code}.");
+        }
+
+        return result.Value!;
+    }
+
+    private static string ReadCpuModel()
+    {
+        const string cpuInfoPath = "/proc/cpuinfo";
+        if (!File.Exists(cpuInfoPath))
+        {
+            return "unavailable";
+        }
+
+        string? model = File.ReadLines(cpuInfoPath)
+            .FirstOrDefault(static line => line.StartsWith("model name", StringComparison.Ordinal));
+        int separator = model?.IndexOf(':') ?? -1;
+        return separator >= 0 ? model![(separator + 1)..].Trim() : "unavailable";
+    }
+
+    private static string ReadCgroupValue(string path) =>
+        File.Exists(path) ? File.ReadAllText(path).Trim() : "not-enforced";
+
+    private sealed class ManualClock : IWallClock, IMonotonicClock
+    {
+        public DateTimeOffset UtcNow { get; private set; } = DateTimeOffset.UtcNow;
+        public TimeSpan Elapsed { get; private set; }
+
+        public void Advance(TimeSpan duration)
+        {
+            UtcNow += duration;
+            Elapsed += duration;
+        }
+    }
+
+    private readonly record struct SoakEvidence(
+        int Cycles,
+        TimeSpan Duration,
+        int PeakScheduledExpiryEntries,
+        long ManagedGrowthBytes,
+        int HandleGrowth);
+}