using System.Collections.Concurrent; using System.Diagnostics; using System.Runtime; using System.Runtime.InteropServices; using FinalFactory.Rendezvous.Contracts; using FinalFactory.Rendezvous.Server.Observability; using FinalFactory.Rendezvous.Server.State; namespace FinalFactory.Rendezvous.Capacity; internal static class CapacityRunner { private static readonly TenantScope Scope = new(new("space-game"), new("production")); private const uint ProtocolVersion = 1; public static Task RunAsync(CapacityOptions options) { ArgumentNullException.ThrowIfNull(options); Process process = Process.GetCurrentProcess(); TimeSpan cpuBefore = process.TotalProcessorTime; Stopwatch capacityPhaseTime = Stopwatch.StartNew(); List failures = []; List measurements = []; ManualClock clock = new(); InMemoryEphemeralRendezvousStore store = CreateStore(options, clock); List listings = new(options.Listings); List attempts = new(options.Attempts); int registrationSamples = Math.Min(options.Samples, options.Listings); int attemptSamples = Math.Min(options.Samples, options.Attempts); for (int index = 0; index < options.Listings - registrationSamples; index++) { listings.Add(CreateVisibleListing(store, index)); } measurements.Add(Measure( "registration-and-presence", registrationSamples, budgetMilliseconds: 200, minimumOperationsPerSecond: 200, index => listings.Add(CreateVisibleListing( store, options.Listings - registrationSamples + index)))); measurements.Add(Measure( "lease-renewal", registrationSamples, budgetMilliseconds: 200, minimumOperationsPerSecond: 200, index => { StoredListing listing = listings[index]; StoreResult renewed = store.RenewLease(new( listing.Definition.ListingId, listing.Definition.LeaseId, listing.Definition.LeaseFingerprint, listing.Definition.OwnerSubject, listing.Version)); RequireSuccess(renewed, "renewal"); listings[index] = renewed.Value!; })); int browseSamples = Math.Min(options.Samples, 250); measurements.Add(Measure( "visible-session-browse", browseSamples, budgetMilliseconds: 200, minimumOperationsPerSecond: 200, _ => RequireSuccess( store.BrowseVisibleListings(new( Scope, ProtocolVersion, new RegionId("eu-central"), ContractLimits.BrowserPageMaxItems, ExcludeFull: true)), "browse"))); for (int index = 0; index < options.Attempts - attemptSamples; index++) { CreateJoinAttemptCommand command = CreateAttempt(index, listings[index % listings.Count]); RequireSuccess(store.CreateJoinAttempt(command), "join issuance"); attempts.Add(command); } measurements.Add(Measure( "join-attempt-issuance", attemptSamples, budgetMilliseconds: 200, minimumOperationsPerSecond: 200, index => { int sequence = options.Attempts - attemptSamples + index; CreateJoinAttemptCommand command = CreateAttempt( sequence, listings[sequence % listings.Count]); RequireSuccess(store.CreateJoinAttempt(command), "join issuance"); attempts.Add(command); })); int punchSamples = Math.Min(options.Samples, attempts.Count); measurements.Add(MeasureConcurrentPunch(store, attempts, punchSamples)); EphemeralStoreSnapshot peak = store.GetSnapshot(); StoreResult overloaded = store.CreateJoinAttempt( CreateAttempt(options.Attempts + 1, listings[^1])); bool overloadWasTyped = peak.ActiveJoinAttempts == options.Attempts && overloaded.Code == StoreResultCode.CapacityExceeded; if (!overloadWasTyped) { failures.Add( $"Expected typed CapacityExceeded at {options.Attempts} active attempts, " + $"observed count={peak.ActiveJoinAttempts}, result={overloaded.Code}."); } int revocationSamples = Math.Min(Math.Max(1, options.Samples / 20), listings.Count / 2); measurements.Add(Measure( "principal-revocation", revocationSamples, budgetMilliseconds: 200, minimumOperationsPerSecond: 50, index => RequireSuccess( store.RevokePrincipal( listings[index].Definition.OwnerSubject, TimeSpan.FromMinutes(1)), "principal revocation"))); using (RendezvousTelemetry telemetry = new(store)) { measurements.Add(Measure( "telemetry-recording", Math.Max(100, options.Samples), budgetMilliseconds: 1, minimumOperationsPerSecond: 10_000, _ => { telemetry.RecordHttp("browse", 200, 1); telemetry.RecordUdp("contribution", "accepted", 1); telemetry.RecordPairingLatency(2); })); } clock.Advance(TimeSpan.FromSeconds(61)); EphemeralStoreSnapshot? afterCoincidentExpiry = null; measurements.Add(Measure( "coincident-listing-attempt-expiry", samples: 1, budgetMilliseconds: 200, minimumOperationsPerSecond: 0, _ => afterCoincidentExpiry = store.GetSnapshot())); if (afterCoincidentExpiry!.ActiveJoinAttempts != 0 || afterCoincidentExpiry.ActiveListings != 0) { failures.Add("Coincident 60-second cleanup retained expired listings or join attempts."); } clock.Advance(TimeSpan.FromSeconds(90)); _ = store.GetSnapshot(); StoredListing recoveryListing = CreateVisibleListing(store, options.Listings + 1); StoreResult recovered = store.CreateJoinAttempt( CreateAttempt(options.Attempts + 2, recoveryListing)); bool recoverySucceeded = recovered.Succeeded; if (!recoverySucceeded) { failures.Add($"Store did not recover after attempt expiry: {recovered.Code}."); } clock.Advance(TimeSpan.FromSeconds(151)); EphemeralStoreSnapshot final = store.GetSnapshot(); if (final.ActiveListings != 0 || final.ActiveJoinAttempts != 0 || final.ReplayMarkers != 0 || final.IdempotencyEntries != 0 || final.RetainedOutcomeReports != 0) { failures.Add("Expiry cleanup left active or retained state after every configured deadline."); } capacityPhaseTime.Stop(); process.Refresh(); double capacityPhaseCpuPercent = 100 * (process.TotalProcessorTime - cpuBefore).TotalSeconds / Math.Max(capacityPhaseTime.Elapsed.TotalSeconds * Environment.ProcessorCount, 0.000_001); if (options.Profile == "candidate" && capacityPhaseCpuPercent > 70) { failures.Add( $"Capacity-phase CPU {capacityPhaseCpuPercent:F1}% exceeded the 70% candidate budget."); } SoakEvidence soak = RunAcceleratedSoak(options, failures); ManualClock restartClock = new(); EphemeralStoreSnapshot restarted = CreateStore(options, restartClock).GetSnapshot(); bool restartStartedEmpty = restarted.ActiveListings == 0 && restarted.ActiveJoinAttempts == 0 && restarted.ReplayMarkers == 0; if (!restartStartedEmpty) { failures.Add("A restarted store did not begin empty."); } foreach (CapacityMeasurement measurement in measurements.Where(static item => !item.Passed)) { failures.Add( $"{measurement.Operation} missed its budget: p95={measurement.P95Milliseconds:F3} ms, " + $"rate={measurement.OperationsPerSecond:F1}/s."); } GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); process.Refresh(); long managedAfterCleanup = GC.GetTotalMemory(forceFullCollection: true); long memoryBudget = 1_610_612_736; if (process.PeakWorkingSet64 > memoryBudget) { failures.Add( $"Peak working set {process.PeakWorkingSet64} exceeded the 1.5 GiB profile budget."); } if (options.Profile == "candidate" && Environment.ProcessorCount != 2) { failures.Add( $"Candidate evidence must expose exactly two CPUs; runtime exposed " + $"{Environment.ProcessorCount}."); } CapacityReport report = new() { SchemaVersion = 2, EvidenceVersion = "v2", GeneratedAt = DateTimeOffset.UtcNow, Profile = options.Profile, Runtime = new( RuntimeInformation.FrameworkDescription, RuntimeInformation.OSDescription, Environment.OSVersion.VersionString, RuntimeInformation.ProcessArchitecture.ToString(), ReadCpuModel(), Environment.ProcessorCount, Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_CPUSET") ?? "unrestricted", ReadCgroupValue("/sys/fs/cgroup/cpu.max"), ReadCgroupValue("/sys/fs/cgroup/memory.max"), GCSettings.IsServerGC ? "server" : "workstation", Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_COMMIT") ?? "unrecorded", Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_TREE_STATE") ?? "unrecorded", Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_COMMAND") ?? "unrecorded", Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_IMAGE_DIGEST") ?? "not-containerized", "fixed-sequences-random-identifiers", capacityPhaseCpuPercent, process.PeakWorkingSet64, managedAfterCleanup), Targets = new( options.Listings, options.Attempts, 200, 2_000, 200, 100, 70, memoryBudget, options.SoakCycles, options.SoakSeconds), Measurements = measurements, State = new( peak.ActiveListings, peak.ActiveJoinAttempts, peak.ReplayMarkers, final.ActiveListings, final.ActiveJoinAttempts, final.ReplayMarkers, final.ExpiryChurn, final.MaintenanceSweeps, soak.Cycles, soak.Duration.TotalSeconds, soak.PeakScheduledExpiryEntries, soak.ManagedGrowthBytes, soak.HandleGrowth, restartStartedEmpty, overloadWasTyped, recoverySucceeded), Failures = failures, Passed = failures.Count == 0, }; return Task.FromResult(report); } private static InMemoryEphemeralRendezvousStore CreateStore( CapacityOptions options, ManualClock clock) => new( new EphemeralStoreOptions { MaxListings = options.Listings, MaxPresenceBindings = options.Listings, MaxJoinAttempts = options.Attempts, MaxOutcomeReports = options.Attempts, MaxIdempotencyEntries = options.Listings + options.Attempts + 1, }, clock, clock); private static StoredListing CreateVisibleListing( InMemoryEphemeralRendezvousStore store, int sequence) { string owner = $"publisher-{sequence}"; SecretFingerprint leaseFingerprint = new($"lease-{sequence}"); SecretFingerprint presenceFingerprint = new($"presence-{sequence}"); ListingDefinition definition = new() { ListingId = new(Guid.NewGuid()), LeaseId = new(Guid.NewGuid()), Scope = Scope, OwnerSubject = owner, RegionId = new("eu-central"), ProtocolVersion = ProtocolVersion, BuildVersion = "1.0.0", DisplayName = $"Capacity host {sequence}", Visibility = ListingVisibility.Public, TrustMode = PublisherTrustMode.ManagedDedicated, CurrentPlayers = 1, MaximumPlayers = 8, Metadata = new Dictionary(StringComparer.Ordinal), LeaseFingerprint = leaseFingerprint, HostPresenceHandle = new(Guid.NewGuid()), HostPresenceFingerprint = presenceFingerprint, CapabilityDerivationSalt = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", }; StoreResult created = store.CreateListing(new( $"register-{sequence}", $"register-request-{sequence}", definition)); RequireSuccess(created, "registration"); StoreResult bound = store.BindHostPresence(new( definition.HostPresenceHandle, presenceFingerprint, PublicEndpoint(10_000 + sequence % 50_000), null)); RequireSuccess(bound, "host presence"); return bound.Value!; } private static CreateJoinAttemptCommand CreateAttempt(int sequence, StoredListing listing) => new() { IdempotencyOwner = $"client-{sequence}", IdempotencyKey = $"join-{sequence}", RequestFingerprint = $"join-request-{sequence}", ClientSubject = $"client-{sequence}", AttemptId = new(Guid.NewGuid()), MediationHandle = new(Guid.NewGuid()), Scope = Scope, ListingId = listing.Definition.ListingId, ProtocolVersion = ProtocolVersion, HostCapabilityFingerprint = new($"host-capability-{sequence}"), ClientCapabilityFingerprint = new($"client-capability-{sequence}"), ConnectionTicketFingerprint = new($"ticket-{sequence}"), CapabilityDerivationSalt = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", }; private static CapacityMeasurement MeasureConcurrentPunch( InMemoryEphemeralRendezvousStore store, List attempts, int samples) { ConcurrentBag latencies = []; Stopwatch total = Stopwatch.StartNew(); Parallel.ForEach( Enumerable.Range(0, samples), new ParallelOptions { MaxDegreeOfParallelism = Math.Min(64, Environment.ProcessorCount * 4) }, index => { CreateJoinAttemptCommand attempt = attempts[index]; Stopwatch elapsed = Stopwatch.StartNew(); RequireSuccess(store.BindAttemptEndpoint(new( attempt.MediationHandle, AttemptPeerRole.Host, attempt.HostCapabilityFingerprint, PublicEndpoint(20_000 + index % 20_000), null)), "host punch"); RequireSuccess(store.BindAttemptEndpoint(new( attempt.MediationHandle, AttemptPeerRole.Client, attempt.ClientCapabilityFingerprint, PublicEndpoint(40_000 + index % 20_000), null)), "client punch"); RequireSuccess(store.ConsumeIntroduction(attempt.MediationHandle), "introduction"); latencies.Add(elapsed.Elapsed.TotalMilliseconds); }); total.Stop(); return BuildMeasurement( "simultaneous-punch-pairing", latencies.ToArray(), total.Elapsed, budgetMilliseconds: 100, minimumOperationsPerSecond: 2_000); } private static CapacityMeasurement Measure( string operation, int samples, double budgetMilliseconds, double minimumOperationsPerSecond, Action action) { double[] latencies = new double[samples]; Stopwatch total = Stopwatch.StartNew(); for (int index = 0; index < samples; index++) { long started = Stopwatch.GetTimestamp(); action(index); latencies[index] = Stopwatch.GetElapsedTime(started).TotalMilliseconds; } total.Stop(); return BuildMeasurement( operation, latencies, total.Elapsed, budgetMilliseconds, minimumOperationsPerSecond); } private static CapacityMeasurement BuildMeasurement( string operation, double[] latencies, TimeSpan elapsed, double budgetMilliseconds, double minimumOperationsPerSecond) { Array.Sort(latencies); double operationsPerSecond = latencies.Length / Math.Max(elapsed.TotalSeconds, 0.000_001); double p95 = Percentile(latencies, 0.95); return new( operation, latencies.Length, Percentile(latencies, 0.50), p95, Percentile(latencies, 0.99), operationsPerSecond, minimumOperationsPerSecond, budgetMilliseconds, p95 <= budgetMilliseconds && operationsPerSecond >= minimumOperationsPerSecond); } private static double Percentile(double[] sorted, double percentile) { int index = Math.Clamp((int)Math.Ceiling(sorted.Length * percentile) - 1, 0, sorted.Length - 1); return sorted[index]; } private static SoakEvidence RunAcceleratedSoak( CapacityOptions options, List failures) { GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); long managedBefore = GC.GetTotalMemory(forceFullCollection: true); int handlesBefore = Process.GetCurrentProcess().HandleCount; ManualClock clock = new(); CapacityOptions soakOptions = options with { Listings = 100, Attempts = 100 }; InMemoryEphemeralRendezvousStore store = CreateStore(soakOptions, clock); Stopwatch elapsed = Stopwatch.StartNew(); int cycle = 0; int peakScheduledExpiryEntries = 0; while (cycle < options.SoakCycles || elapsed.Elapsed < TimeSpan.FromSeconds(options.SoakSeconds)) { StoredListing listing = CreateVisibleListing(store, cycle); for (int refresh = 0; refresh < 10; refresh++) { clock.Advance(TimeSpan.FromTicks(1)); listing = RequireSuccess(store.RenewLease(new( listing.Definition.ListingId, listing.Definition.LeaseId, listing.Definition.LeaseFingerprint, listing.Definition.OwnerSubject, listing.Version)), "soak lease refresh"); listing = RequireSuccess(store.BindHostPresence(new( listing.Definition.HostPresenceHandle, listing.Definition.HostPresenceFingerprint, PublicEndpoint(10_000 + cycle % 50_000), null)), "soak presence refresh"); } CreateJoinAttemptCommand attempt = CreateAttempt(cycle, listing); RequireSuccess(store.CreateJoinAttempt(attempt), "soak join issuance"); RequireSuccess(store.ConsumeReplay(new("capacity-soak", $"replay-{cycle}")), "soak replay"); peakScheduledExpiryEntries = Math.Max( peakScheduledExpiryEntries, store.ScheduledExpiryEntryCount); if (store.ScheduledExpiryEntryCount > 7) { failures.Add( $"Mutable deadline refresh grew the expiry queue to " + $"{store.ScheduledExpiryEntryCount} entries for one lifecycle."); break; } clock.Advance(TimeSpan.FromSeconds(151)); EphemeralStoreSnapshot snapshot = store.GetSnapshot(); if (snapshot.ActiveListings != 0 || snapshot.ActiveJoinAttempts != 0 || snapshot.ReplayMarkers != 0 || snapshot.IdempotencyEntries != 0 || snapshot.RetainedOutcomeReports != 0) { failures.Add($"Accelerated soak retained state after cycle {cycle}."); break; } cycle++; } elapsed.Stop(); GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); long managedGrowth = GC.GetTotalMemory(forceFullCollection: true) - managedBefore; int handleGrowth = Process.GetCurrentProcess().HandleCount - handlesBefore; if (managedGrowth > 67_108_864) { failures.Add($"Soak retained {managedGrowth} managed bytes; budget is 64 MiB."); } if (handleGrowth > 8) { failures.Add($"Soak retained {handleGrowth} process handles; budget is 8."); } return new(cycle, elapsed.Elapsed, peakScheduledExpiryEntries, managedGrowth, handleGrowth); } private static ObservedEndpoint PublicEndpoint(int port) => new(AddressFamilyKind.Ipv4, "203.0.113.10", port); private static T RequireSuccess(StoreResult result, string operation) { if (!result.Succeeded) { throw new InvalidOperationException($"{operation} failed with {result.Code}."); } return result.Value!; } private static string ReadCpuModel() { const string cpuInfoPath = "/proc/cpuinfo"; if (!File.Exists(cpuInfoPath)) { return "unavailable"; } string? model = File.ReadLines(cpuInfoPath) .FirstOrDefault(static line => line.StartsWith("model name", StringComparison.Ordinal)); int separator = model?.IndexOf(':') ?? -1; return separator >= 0 ? model![(separator + 1)..].Trim() : "unavailable"; } private static string ReadCgroupValue(string path) => File.Exists(path) ? File.ReadAllText(path).Trim() : "not-enforced"; private sealed class ManualClock : IWallClock, IMonotonicClock { public DateTimeOffset UtcNow { get; private set; } = DateTimeOffset.UtcNow; public TimeSpan Elapsed { get; private set; } public void Advance(TimeSpan duration) { UtcNow += duration; Elapsed += duration; } } private readonly record struct SoakEvidence( int Cycles, TimeSpan Duration, int PeakScheduledExpiryEntries, long ManagedGrowthBytes, int HandleGrowth); }