feat(operations): add capacity and resilience gates (#18)
This commit is contained in:
KyuubiYoru
@@ -0,0 +1,91 @@
|
||||
namespace FinalFactory.Rendezvous.Capacity;
|
||||
|
||||
internal sealed record CapacityOptions
|
||||
{
|
||||
public required string Profile { get; init; }
|
||||
public required int Listings { get; init; }
|
||||
public required int Attempts { get; init; }
|
||||
public required int Samples { get; init; }
|
||||
public required int SoakCycles { get; init; }
|
||||
public required int SoakSeconds { get; init; }
|
||||
public string? OutputPath { get; init; }
|
||||
|
||||
public static CapacityOptions Parse(string[] args)
|
||||
{
|
||||
Dictionary<string, string> values = ParseArguments(args);
|
||||
string profile = values.GetValueOrDefault("--profile") ?? "quick";
|
||||
(int listings, int attempts, int samples, int soakCycles, int soakSeconds) = profile switch
|
||||
{
|
||||
"quick" => (1_000, 500, 100, 20, 0),
|
||||
"candidate" => (25_000, 10_000, 1_000, 1_000, 300),
|
||||
_ => throw new ArgumentException("--profile must be 'quick' or 'candidate'."),
|
||||
};
|
||||
|
||||
return new()
|
||||
{
|
||||
Profile = profile,
|
||||
Listings = ParsePositive(values.GetValueOrDefault("--listings"), listings, "--listings"),
|
||||
Attempts = ParsePositive(values.GetValueOrDefault("--attempts"), attempts, "--attempts"),
|
||||
Samples = ParsePositive(values.GetValueOrDefault("--samples"), samples, "--samples"),
|
||||
SoakCycles = ParsePositive(
|
||||
values.GetValueOrDefault("--soak-cycles"),
|
||||
soakCycles,
|
||||
"--soak-cycles"),
|
||||
SoakSeconds = ParseNonNegative(
|
||||
values.GetValueOrDefault("--soak-seconds"),
|
||||
soakSeconds,
|
||||
"--soak-seconds"),
|
||||
OutputPath = values.GetValueOrDefault("--output"),
|
||||
};
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> ParseArguments(string[] args)
|
||||
{
|
||||
HashSet<string> allowed =
|
||||
[
|
||||
"--profile",
|
||||
"--listings",
|
||||
"--attempts",
|
||||
"--samples",
|
||||
"--soak-cycles",
|
||||
"--soak-seconds",
|
||||
"--output",
|
||||
];
|
||||
Dictionary<string, string> values = new(StringComparer.Ordinal);
|
||||
for (int index = 0; index < args.Length; index += 2)
|
||||
{
|
||||
string option = args[index];
|
||||
if (!allowed.Contains(option))
|
||||
{
|
||||
throw new ArgumentException($"Unknown option: {option}.");
|
||||
}
|
||||
|
||||
if (index == args.Length - 1
|
||||
|| args[index + 1].StartsWith("--", StringComparison.Ordinal))
|
||||
{
|
||||
throw new ArgumentException($"{option} requires a value.");
|
||||
}
|
||||
|
||||
if (!values.TryAdd(option, args[index + 1]))
|
||||
{
|
||||
throw new ArgumentException($"{option} may be supplied only once.");
|
||||
}
|
||||
}
|
||||
|
||||
return values;
|
||||
}
|
||||
|
||||
private static int ParsePositive(string? value, int fallback, string option) =>
|
||||
value is null
|
||||
? fallback
|
||||
: int.TryParse(value, out int parsed) && parsed > 0
|
||||
? parsed
|
||||
: throw new ArgumentException($"{option} must be a positive integer.");
|
||||
|
||||
private static int ParseNonNegative(string? value, int fallback, string option) =>
|
||||
value is null
|
||||
? fallback
|
||||
: int.TryParse(value, out int parsed) && parsed >= 0
|
||||
? parsed
|
||||
: throw new ArgumentException($"{option} must be a non-negative integer.");
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
namespace FinalFactory.Rendezvous.Capacity;
|
||||
|
||||
internal sealed record CapacityReport
|
||||
{
|
||||
public required int SchemaVersion { get; init; }
|
||||
public required string EvidenceVersion { get; init; }
|
||||
public required DateTimeOffset GeneratedAt { get; init; }
|
||||
public required string Profile { get; init; }
|
||||
public required RuntimeEvidence Runtime { get; init; }
|
||||
public required CapacityTargets Targets { get; init; }
|
||||
public required IReadOnlyList<CapacityMeasurement> Measurements { get; init; }
|
||||
public required StateEvidence State { get; init; }
|
||||
public required IReadOnlyList<string> Failures { get; init; }
|
||||
public required bool Passed { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record RuntimeEvidence(
|
||||
string Framework,
|
||||
string OperatingSystem,
|
||||
string Kernel,
|
||||
string Architecture,
|
||||
string CpuModel,
|
||||
int ProcessorCount,
|
||||
string CpuAffinity,
|
||||
string CpuQuota,
|
||||
string MemoryLimit,
|
||||
string GarbageCollector,
|
||||
string CommitSha,
|
||||
string TreeState,
|
||||
string Command,
|
||||
string ImageDigest,
|
||||
string WorkloadSeed,
|
||||
double CapacityPhaseAverageCpuPercent,
|
||||
long PeakWorkingSetBytes,
|
||||
long ManagedBytesAfterCleanup);
|
||||
|
||||
internal sealed record CapacityTargets(
|
||||
int VisibleListings,
|
||||
int ActiveJoinAttempts,
|
||||
int CoreControlOperationsPerSecond,
|
||||
int CoreMediationOperationsPerSecond,
|
||||
double CoreControlP95Milliseconds,
|
||||
double CoreMediationP95Milliseconds,
|
||||
double MaximumAverageCpuPercent,
|
||||
long MaximumWorkingSetBytes,
|
||||
int SoakCycles,
|
||||
int SoakDurationSeconds);
|
||||
|
||||
internal sealed record CapacityMeasurement(
|
||||
string Operation,
|
||||
int Samples,
|
||||
double P50Milliseconds,
|
||||
double P95Milliseconds,
|
||||
double P99Milliseconds,
|
||||
double OperationsPerSecond,
|
||||
double MinimumOperationsPerSecond,
|
||||
double BudgetMilliseconds,
|
||||
bool Passed);
|
||||
|
||||
internal sealed record StateEvidence(
|
||||
int PeakListings,
|
||||
int PeakAttempts,
|
||||
int PeakReplayMarkers,
|
||||
int FinalListings,
|
||||
int FinalAttempts,
|
||||
int FinalReplayMarkers,
|
||||
long ExpiryChurn,
|
||||
long MaintenanceSweeps,
|
||||
int SoakCyclesCompleted,
|
||||
double SoakDurationSeconds,
|
||||
int SoakPeakScheduledExpiryEntries,
|
||||
long SoakManagedGrowthBytes,
|
||||
int SoakHandleGrowth,
|
||||
bool RestartStartedEmpty,
|
||||
bool OverloadWasTyped,
|
||||
bool RecoverySucceeded);
|
||||
@@ -0,0 +1,580 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.Runtime;
|
||||
using System.Runtime.InteropServices;
|
||||
using FinalFactory.Rendezvous.Contracts;
|
||||
using FinalFactory.Rendezvous.Server.Observability;
|
||||
using FinalFactory.Rendezvous.Server.State;
|
||||
|
||||
namespace FinalFactory.Rendezvous.Capacity;
|
||||
|
||||
internal static class CapacityRunner
|
||||
{
|
||||
private static readonly TenantScope Scope = new(new("space-game"), new("production"));
|
||||
private const uint ProtocolVersion = 1;
|
||||
|
||||
public static Task<CapacityReport> RunAsync(CapacityOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
Process process = Process.GetCurrentProcess();
|
||||
TimeSpan cpuBefore = process.TotalProcessorTime;
|
||||
Stopwatch capacityPhaseTime = Stopwatch.StartNew();
|
||||
List<string> failures = [];
|
||||
List<CapacityMeasurement> measurements = [];
|
||||
ManualClock clock = new();
|
||||
InMemoryEphemeralRendezvousStore store = CreateStore(options, clock);
|
||||
List<StoredListing> listings = new(options.Listings);
|
||||
List<CreateJoinAttemptCommand> attempts = new(options.Attempts);
|
||||
int registrationSamples = Math.Min(options.Samples, options.Listings);
|
||||
int attemptSamples = Math.Min(options.Samples, options.Attempts);
|
||||
|
||||
for (int index = 0; index < options.Listings - registrationSamples; index++)
|
||||
{
|
||||
listings.Add(CreateVisibleListing(store, index));
|
||||
}
|
||||
|
||||
measurements.Add(Measure(
|
||||
"registration-and-presence",
|
||||
registrationSamples,
|
||||
budgetMilliseconds: 200,
|
||||
minimumOperationsPerSecond: 200,
|
||||
index => listings.Add(CreateVisibleListing(
|
||||
store,
|
||||
options.Listings - registrationSamples + index))));
|
||||
|
||||
measurements.Add(Measure(
|
||||
"lease-renewal",
|
||||
registrationSamples,
|
||||
budgetMilliseconds: 200,
|
||||
minimumOperationsPerSecond: 200,
|
||||
index =>
|
||||
{
|
||||
StoredListing listing = listings[index];
|
||||
StoreResult<StoredListing> renewed = store.RenewLease(new(
|
||||
listing.Definition.ListingId,
|
||||
listing.Definition.LeaseId,
|
||||
listing.Definition.LeaseFingerprint,
|
||||
listing.Definition.OwnerSubject,
|
||||
listing.Version));
|
||||
RequireSuccess(renewed, "renewal");
|
||||
listings[index] = renewed.Value!;
|
||||
}));
|
||||
|
||||
int browseSamples = Math.Min(options.Samples, 250);
|
||||
measurements.Add(Measure(
|
||||
"visible-session-browse",
|
||||
browseSamples,
|
||||
budgetMilliseconds: 200,
|
||||
minimumOperationsPerSecond: 200,
|
||||
_ => RequireSuccess(
|
||||
store.BrowseVisibleListings(new(
|
||||
Scope,
|
||||
ProtocolVersion,
|
||||
new RegionId("eu-central"),
|
||||
ContractLimits.BrowserPageMaxItems,
|
||||
ExcludeFull: true)),
|
||||
"browse")));
|
||||
|
||||
for (int index = 0; index < options.Attempts - attemptSamples; index++)
|
||||
{
|
||||
CreateJoinAttemptCommand command = CreateAttempt(index, listings[index % listings.Count]);
|
||||
RequireSuccess(store.CreateJoinAttempt(command), "join issuance");
|
||||
attempts.Add(command);
|
||||
}
|
||||
|
||||
measurements.Add(Measure(
|
||||
"join-attempt-issuance",
|
||||
attemptSamples,
|
||||
budgetMilliseconds: 200,
|
||||
minimumOperationsPerSecond: 200,
|
||||
index =>
|
||||
{
|
||||
int sequence = options.Attempts - attemptSamples + index;
|
||||
CreateJoinAttemptCommand command = CreateAttempt(
|
||||
sequence,
|
||||
listings[sequence % listings.Count]);
|
||||
RequireSuccess(store.CreateJoinAttempt(command), "join issuance");
|
||||
attempts.Add(command);
|
||||
}));
|
||||
|
||||
int punchSamples = Math.Min(options.Samples, attempts.Count);
|
||||
measurements.Add(MeasureConcurrentPunch(store, attempts, punchSamples));
|
||||
EphemeralStoreSnapshot peak = store.GetSnapshot();
|
||||
|
||||
StoreResult<StoredJoinAttempt> overloaded = store.CreateJoinAttempt(
|
||||
CreateAttempt(options.Attempts + 1, listings[^1]));
|
||||
bool overloadWasTyped = peak.ActiveJoinAttempts == options.Attempts
|
||||
&& overloaded.Code == StoreResultCode.CapacityExceeded;
|
||||
if (!overloadWasTyped)
|
||||
{
|
||||
failures.Add(
|
||||
$"Expected typed CapacityExceeded at {options.Attempts} active attempts, "
|
||||
+ $"observed count={peak.ActiveJoinAttempts}, result={overloaded.Code}.");
|
||||
}
|
||||
|
||||
int revocationSamples = Math.Min(Math.Max(1, options.Samples / 20), listings.Count / 2);
|
||||
measurements.Add(Measure(
|
||||
"principal-revocation",
|
||||
revocationSamples,
|
||||
budgetMilliseconds: 200,
|
||||
minimumOperationsPerSecond: 50,
|
||||
index => RequireSuccess(
|
||||
store.RevokePrincipal(
|
||||
listings[index].Definition.OwnerSubject,
|
||||
TimeSpan.FromMinutes(1)),
|
||||
"principal revocation")));
|
||||
|
||||
using (RendezvousTelemetry telemetry = new(store))
|
||||
{
|
||||
measurements.Add(Measure(
|
||||
"telemetry-recording",
|
||||
Math.Max(100, options.Samples),
|
||||
budgetMilliseconds: 1,
|
||||
minimumOperationsPerSecond: 10_000,
|
||||
_ =>
|
||||
{
|
||||
telemetry.RecordHttp("browse", 200, 1);
|
||||
telemetry.RecordUdp("contribution", "accepted", 1);
|
||||
telemetry.RecordPairingLatency(2);
|
||||
}));
|
||||
}
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(61));
|
||||
EphemeralStoreSnapshot? afterCoincidentExpiry = null;
|
||||
measurements.Add(Measure(
|
||||
"coincident-listing-attempt-expiry",
|
||||
samples: 1,
|
||||
budgetMilliseconds: 200,
|
||||
minimumOperationsPerSecond: 0,
|
||||
_ => afterCoincidentExpiry = store.GetSnapshot()));
|
||||
if (afterCoincidentExpiry!.ActiveJoinAttempts != 0
|
||||
|| afterCoincidentExpiry.ActiveListings != 0)
|
||||
{
|
||||
failures.Add("Coincident 60-second cleanup retained expired listings or join attempts.");
|
||||
}
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(90));
|
||||
_ = store.GetSnapshot();
|
||||
StoredListing recoveryListing = CreateVisibleListing(store, options.Listings + 1);
|
||||
StoreResult<StoredJoinAttempt> recovered = store.CreateJoinAttempt(
|
||||
CreateAttempt(options.Attempts + 2, recoveryListing));
|
||||
bool recoverySucceeded = recovered.Succeeded;
|
||||
if (!recoverySucceeded)
|
||||
{
|
||||
failures.Add($"Store did not recover after attempt expiry: {recovered.Code}.");
|
||||
}
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(151));
|
||||
EphemeralStoreSnapshot final = store.GetSnapshot();
|
||||
if (final.ActiveListings != 0
|
||||
|| final.ActiveJoinAttempts != 0
|
||||
|| final.ReplayMarkers != 0
|
||||
|| final.IdempotencyEntries != 0
|
||||
|| final.RetainedOutcomeReports != 0)
|
||||
{
|
||||
failures.Add("Expiry cleanup left active or retained state after every configured deadline.");
|
||||
}
|
||||
|
||||
capacityPhaseTime.Stop();
|
||||
process.Refresh();
|
||||
double capacityPhaseCpuPercent = 100
|
||||
* (process.TotalProcessorTime - cpuBefore).TotalSeconds
|
||||
/ Math.Max(capacityPhaseTime.Elapsed.TotalSeconds * Environment.ProcessorCount, 0.000_001);
|
||||
if (options.Profile == "candidate" && capacityPhaseCpuPercent > 70)
|
||||
{
|
||||
failures.Add(
|
||||
$"Capacity-phase CPU {capacityPhaseCpuPercent:F1}% exceeded the 70% candidate budget.");
|
||||
}
|
||||
|
||||
SoakEvidence soak = RunAcceleratedSoak(options, failures);
|
||||
ManualClock restartClock = new();
|
||||
EphemeralStoreSnapshot restarted = CreateStore(options, restartClock).GetSnapshot();
|
||||
bool restartStartedEmpty = restarted.ActiveListings == 0
|
||||
&& restarted.ActiveJoinAttempts == 0
|
||||
&& restarted.ReplayMarkers == 0;
|
||||
if (!restartStartedEmpty)
|
||||
{
|
||||
failures.Add("A restarted store did not begin empty.");
|
||||
}
|
||||
|
||||
foreach (CapacityMeasurement measurement in measurements.Where(static item => !item.Passed))
|
||||
{
|
||||
failures.Add(
|
||||
$"{measurement.Operation} missed its budget: p95={measurement.P95Milliseconds:F3} ms, "
|
||||
+ $"rate={measurement.OperationsPerSecond:F1}/s.");
|
||||
}
|
||||
|
||||
GC.Collect();
|
||||
GC.WaitForPendingFinalizers();
|
||||
GC.Collect();
|
||||
process.Refresh();
|
||||
long managedAfterCleanup = GC.GetTotalMemory(forceFullCollection: true);
|
||||
long memoryBudget = 1_610_612_736;
|
||||
if (process.PeakWorkingSet64 > memoryBudget)
|
||||
{
|
||||
failures.Add(
|
||||
$"Peak working set {process.PeakWorkingSet64} exceeded the 1.5 GiB profile budget.");
|
||||
}
|
||||
|
||||
if (options.Profile == "candidate" && Environment.ProcessorCount != 2)
|
||||
{
|
||||
failures.Add(
|
||||
$"Candidate evidence must expose exactly two CPUs; runtime exposed "
|
||||
+ $"{Environment.ProcessorCount}.");
|
||||
}
|
||||
|
||||
CapacityReport report = new()
|
||||
{
|
||||
SchemaVersion = 2,
|
||||
EvidenceVersion = "v2",
|
||||
GeneratedAt = DateTimeOffset.UtcNow,
|
||||
Profile = options.Profile,
|
||||
Runtime = new(
|
||||
RuntimeInformation.FrameworkDescription,
|
||||
RuntimeInformation.OSDescription,
|
||||
Environment.OSVersion.VersionString,
|
||||
RuntimeInformation.ProcessArchitecture.ToString(),
|
||||
ReadCpuModel(),
|
||||
Environment.ProcessorCount,
|
||||
Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_CPUSET") ?? "unrestricted",
|
||||
ReadCgroupValue("/sys/fs/cgroup/cpu.max"),
|
||||
ReadCgroupValue("/sys/fs/cgroup/memory.max"),
|
||||
GCSettings.IsServerGC ? "server" : "workstation",
|
||||
Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_COMMIT") ?? "unrecorded",
|
||||
Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_TREE_STATE") ?? "unrecorded",
|
||||
Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_COMMAND") ?? "unrecorded",
|
||||
Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_IMAGE_DIGEST") ?? "not-containerized",
|
||||
"fixed-sequences-random-identifiers",
|
||||
capacityPhaseCpuPercent,
|
||||
process.PeakWorkingSet64,
|
||||
managedAfterCleanup),
|
||||
Targets = new(
|
||||
options.Listings,
|
||||
options.Attempts,
|
||||
200,
|
||||
2_000,
|
||||
200,
|
||||
100,
|
||||
70,
|
||||
memoryBudget,
|
||||
options.SoakCycles,
|
||||
options.SoakSeconds),
|
||||
Measurements = measurements,
|
||||
State = new(
|
||||
peak.ActiveListings,
|
||||
peak.ActiveJoinAttempts,
|
||||
peak.ReplayMarkers,
|
||||
final.ActiveListings,
|
||||
final.ActiveJoinAttempts,
|
||||
final.ReplayMarkers,
|
||||
final.ExpiryChurn,
|
||||
final.MaintenanceSweeps,
|
||||
soak.Cycles,
|
||||
soak.Duration.TotalSeconds,
|
||||
soak.PeakScheduledExpiryEntries,
|
||||
soak.ManagedGrowthBytes,
|
||||
soak.HandleGrowth,
|
||||
restartStartedEmpty,
|
||||
overloadWasTyped,
|
||||
recoverySucceeded),
|
||||
Failures = failures,
|
||||
Passed = failures.Count == 0,
|
||||
};
|
||||
return Task.FromResult(report);
|
||||
}
|
||||
|
||||
private static InMemoryEphemeralRendezvousStore CreateStore(
|
||||
CapacityOptions options,
|
||||
ManualClock clock) => new(
|
||||
new EphemeralStoreOptions
|
||||
{
|
||||
MaxListings = options.Listings,
|
||||
MaxPresenceBindings = options.Listings,
|
||||
MaxJoinAttempts = options.Attempts,
|
||||
MaxOutcomeReports = options.Attempts,
|
||||
MaxIdempotencyEntries = options.Listings + options.Attempts + 1,
|
||||
},
|
||||
clock,
|
||||
clock);
|
||||
|
||||
private static StoredListing CreateVisibleListing(
|
||||
InMemoryEphemeralRendezvousStore store,
|
||||
int sequence)
|
||||
{
|
||||
string owner = $"publisher-{sequence}";
|
||||
SecretFingerprint leaseFingerprint = new($"lease-{sequence}");
|
||||
SecretFingerprint presenceFingerprint = new($"presence-{sequence}");
|
||||
ListingDefinition definition = new()
|
||||
{
|
||||
ListingId = new(Guid.NewGuid()),
|
||||
LeaseId = new(Guid.NewGuid()),
|
||||
Scope = Scope,
|
||||
OwnerSubject = owner,
|
||||
RegionId = new("eu-central"),
|
||||
ProtocolVersion = ProtocolVersion,
|
||||
BuildVersion = "1.0.0",
|
||||
DisplayName = $"Capacity host {sequence}",
|
||||
Visibility = ListingVisibility.Public,
|
||||
TrustMode = PublisherTrustMode.ManagedDedicated,
|
||||
CurrentPlayers = 1,
|
||||
MaximumPlayers = 8,
|
||||
Metadata = new Dictionary<string, string>(StringComparer.Ordinal),
|
||||
LeaseFingerprint = leaseFingerprint,
|
||||
HostPresenceHandle = new(Guid.NewGuid()),
|
||||
HostPresenceFingerprint = presenceFingerprint,
|
||||
CapabilityDerivationSalt = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
||||
};
|
||||
StoreResult<StoredListing> created = store.CreateListing(new(
|
||||
$"register-{sequence}",
|
||||
$"register-request-{sequence}",
|
||||
definition));
|
||||
RequireSuccess(created, "registration");
|
||||
StoreResult<StoredListing> bound = store.BindHostPresence(new(
|
||||
definition.HostPresenceHandle,
|
||||
presenceFingerprint,
|
||||
PublicEndpoint(10_000 + sequence % 50_000),
|
||||
null));
|
||||
RequireSuccess(bound, "host presence");
|
||||
return bound.Value!;
|
||||
}
|
||||
|
||||
private static CreateJoinAttemptCommand CreateAttempt(int sequence, StoredListing listing) => new()
|
||||
{
|
||||
IdempotencyOwner = $"client-{sequence}",
|
||||
IdempotencyKey = $"join-{sequence}",
|
||||
RequestFingerprint = $"join-request-{sequence}",
|
||||
ClientSubject = $"client-{sequence}",
|
||||
AttemptId = new(Guid.NewGuid()),
|
||||
MediationHandle = new(Guid.NewGuid()),
|
||||
Scope = Scope,
|
||||
ListingId = listing.Definition.ListingId,
|
||||
ProtocolVersion = ProtocolVersion,
|
||||
HostCapabilityFingerprint = new($"host-capability-{sequence}"),
|
||||
ClientCapabilityFingerprint = new($"client-capability-{sequence}"),
|
||||
ConnectionTicketFingerprint = new($"ticket-{sequence}"),
|
||||
CapabilityDerivationSalt = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
||||
};
|
||||
|
||||
private static CapacityMeasurement MeasureConcurrentPunch(
|
||||
InMemoryEphemeralRendezvousStore store,
|
||||
List<CreateJoinAttemptCommand> attempts,
|
||||
int samples)
|
||||
{
|
||||
ConcurrentBag<double> latencies = [];
|
||||
Stopwatch total = Stopwatch.StartNew();
|
||||
Parallel.ForEach(
|
||||
Enumerable.Range(0, samples),
|
||||
new ParallelOptions { MaxDegreeOfParallelism = Math.Min(64, Environment.ProcessorCount * 4) },
|
||||
index =>
|
||||
{
|
||||
CreateJoinAttemptCommand attempt = attempts[index];
|
||||
Stopwatch elapsed = Stopwatch.StartNew();
|
||||
RequireSuccess(store.BindAttemptEndpoint(new(
|
||||
attempt.MediationHandle,
|
||||
AttemptPeerRole.Host,
|
||||
attempt.HostCapabilityFingerprint,
|
||||
PublicEndpoint(20_000 + index % 20_000),
|
||||
null)), "host punch");
|
||||
RequireSuccess(store.BindAttemptEndpoint(new(
|
||||
attempt.MediationHandle,
|
||||
AttemptPeerRole.Client,
|
||||
attempt.ClientCapabilityFingerprint,
|
||||
PublicEndpoint(40_000 + index % 20_000),
|
||||
null)), "client punch");
|
||||
RequireSuccess(store.ConsumeIntroduction(attempt.MediationHandle), "introduction");
|
||||
latencies.Add(elapsed.Elapsed.TotalMilliseconds);
|
||||
});
|
||||
total.Stop();
|
||||
return BuildMeasurement(
|
||||
"simultaneous-punch-pairing",
|
||||
latencies.ToArray(),
|
||||
total.Elapsed,
|
||||
budgetMilliseconds: 100,
|
||||
minimumOperationsPerSecond: 2_000);
|
||||
}
|
||||
|
||||
private static CapacityMeasurement Measure(
|
||||
string operation,
|
||||
int samples,
|
||||
double budgetMilliseconds,
|
||||
double minimumOperationsPerSecond,
|
||||
Action<int> action)
|
||||
{
|
||||
double[] latencies = new double[samples];
|
||||
Stopwatch total = Stopwatch.StartNew();
|
||||
for (int index = 0; index < samples; index++)
|
||||
{
|
||||
long started = Stopwatch.GetTimestamp();
|
||||
action(index);
|
||||
latencies[index] = Stopwatch.GetElapsedTime(started).TotalMilliseconds;
|
||||
}
|
||||
|
||||
total.Stop();
|
||||
return BuildMeasurement(
|
||||
operation,
|
||||
latencies,
|
||||
total.Elapsed,
|
||||
budgetMilliseconds,
|
||||
minimumOperationsPerSecond);
|
||||
}
|
||||
|
||||
private static CapacityMeasurement BuildMeasurement(
|
||||
string operation,
|
||||
double[] latencies,
|
||||
TimeSpan elapsed,
|
||||
double budgetMilliseconds,
|
||||
double minimumOperationsPerSecond)
|
||||
{
|
||||
Array.Sort(latencies);
|
||||
double operationsPerSecond = latencies.Length / Math.Max(elapsed.TotalSeconds, 0.000_001);
|
||||
double p95 = Percentile(latencies, 0.95);
|
||||
return new(
|
||||
operation,
|
||||
latencies.Length,
|
||||
Percentile(latencies, 0.50),
|
||||
p95,
|
||||
Percentile(latencies, 0.99),
|
||||
operationsPerSecond,
|
||||
minimumOperationsPerSecond,
|
||||
budgetMilliseconds,
|
||||
p95 <= budgetMilliseconds && operationsPerSecond >= minimumOperationsPerSecond);
|
||||
}
|
||||
|
||||
private static double Percentile(double[] sorted, double percentile)
|
||||
{
|
||||
int index = Math.Clamp((int)Math.Ceiling(sorted.Length * percentile) - 1, 0, sorted.Length - 1);
|
||||
return sorted[index];
|
||||
}
|
||||
|
||||
private static SoakEvidence RunAcceleratedSoak(
|
||||
CapacityOptions options,
|
||||
List<string> failures)
|
||||
{
|
||||
GC.Collect();
|
||||
GC.WaitForPendingFinalizers();
|
||||
GC.Collect();
|
||||
long managedBefore = GC.GetTotalMemory(forceFullCollection: true);
|
||||
int handlesBefore = Process.GetCurrentProcess().HandleCount;
|
||||
ManualClock clock = new();
|
||||
CapacityOptions soakOptions = options with { Listings = 100, Attempts = 100 };
|
||||
InMemoryEphemeralRendezvousStore store = CreateStore(soakOptions, clock);
|
||||
Stopwatch elapsed = Stopwatch.StartNew();
|
||||
int cycle = 0;
|
||||
int peakScheduledExpiryEntries = 0;
|
||||
while (cycle < options.SoakCycles
|
||||
|| elapsed.Elapsed < TimeSpan.FromSeconds(options.SoakSeconds))
|
||||
{
|
||||
StoredListing listing = CreateVisibleListing(store, cycle);
|
||||
for (int refresh = 0; refresh < 10; refresh++)
|
||||
{
|
||||
clock.Advance(TimeSpan.FromTicks(1));
|
||||
listing = RequireSuccess(store.RenewLease(new(
|
||||
listing.Definition.ListingId,
|
||||
listing.Definition.LeaseId,
|
||||
listing.Definition.LeaseFingerprint,
|
||||
listing.Definition.OwnerSubject,
|
||||
listing.Version)), "soak lease refresh");
|
||||
listing = RequireSuccess(store.BindHostPresence(new(
|
||||
listing.Definition.HostPresenceHandle,
|
||||
listing.Definition.HostPresenceFingerprint,
|
||||
PublicEndpoint(10_000 + cycle % 50_000),
|
||||
null)), "soak presence refresh");
|
||||
}
|
||||
|
||||
CreateJoinAttemptCommand attempt = CreateAttempt(cycle, listing);
|
||||
RequireSuccess(store.CreateJoinAttempt(attempt), "soak join issuance");
|
||||
RequireSuccess(store.ConsumeReplay(new("capacity-soak", $"replay-{cycle}")), "soak replay");
|
||||
peakScheduledExpiryEntries = Math.Max(
|
||||
peakScheduledExpiryEntries,
|
||||
store.ScheduledExpiryEntryCount);
|
||||
if (store.ScheduledExpiryEntryCount > 7)
|
||||
{
|
||||
failures.Add(
|
||||
$"Mutable deadline refresh grew the expiry queue to "
|
||||
+ $"{store.ScheduledExpiryEntryCount} entries for one lifecycle.");
|
||||
break;
|
||||
}
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(151));
|
||||
EphemeralStoreSnapshot snapshot = store.GetSnapshot();
|
||||
if (snapshot.ActiveListings != 0
|
||||
|| snapshot.ActiveJoinAttempts != 0
|
||||
|| snapshot.ReplayMarkers != 0
|
||||
|| snapshot.IdempotencyEntries != 0
|
||||
|| snapshot.RetainedOutcomeReports != 0)
|
||||
{
|
||||
failures.Add($"Accelerated soak retained state after cycle {cycle}.");
|
||||
break;
|
||||
}
|
||||
|
||||
cycle++;
|
||||
}
|
||||
|
||||
elapsed.Stop();
|
||||
GC.Collect();
|
||||
GC.WaitForPendingFinalizers();
|
||||
GC.Collect();
|
||||
long managedGrowth = GC.GetTotalMemory(forceFullCollection: true) - managedBefore;
|
||||
int handleGrowth = Process.GetCurrentProcess().HandleCount - handlesBefore;
|
||||
if (managedGrowth > 67_108_864)
|
||||
{
|
||||
failures.Add($"Soak retained {managedGrowth} managed bytes; budget is 64 MiB.");
|
||||
}
|
||||
|
||||
if (handleGrowth > 8)
|
||||
{
|
||||
failures.Add($"Soak retained {handleGrowth} process handles; budget is 8.");
|
||||
}
|
||||
|
||||
return new(cycle, elapsed.Elapsed, peakScheduledExpiryEntries, managedGrowth, handleGrowth);
|
||||
}
|
||||
|
||||
private static ObservedEndpoint PublicEndpoint(int port) =>
|
||||
new(AddressFamilyKind.Ipv4, "203.0.113.10", port);
|
||||
|
||||
private static T RequireSuccess<T>(StoreResult<T> result, string operation)
|
||||
{
|
||||
if (!result.Succeeded)
|
||||
{
|
||||
throw new InvalidOperationException($"{operation} failed with {result.Code}.");
|
||||
}
|
||||
|
||||
return result.Value!;
|
||||
}
|
||||
|
||||
private static string ReadCpuModel()
|
||||
{
|
||||
const string cpuInfoPath = "/proc/cpuinfo";
|
||||
if (!File.Exists(cpuInfoPath))
|
||||
{
|
||||
return "unavailable";
|
||||
}
|
||||
|
||||
string? model = File.ReadLines(cpuInfoPath)
|
||||
.FirstOrDefault(static line => line.StartsWith("model name", StringComparison.Ordinal));
|
||||
int separator = model?.IndexOf(':') ?? -1;
|
||||
return separator >= 0 ? model![(separator + 1)..].Trim() : "unavailable";
|
||||
}
|
||||
|
||||
private static string ReadCgroupValue(string path) =>
|
||||
File.Exists(path) ? File.ReadAllText(path).Trim() : "not-enforced";
|
||||
|
||||
private sealed class ManualClock : IWallClock, IMonotonicClock
|
||||
{
|
||||
public DateTimeOffset UtcNow { get; private set; } = DateTimeOffset.UtcNow;
|
||||
public TimeSpan Elapsed { get; private set; }
|
||||
|
||||
public void Advance(TimeSpan duration)
|
||||
{
|
||||
UtcNow += duration;
|
||||
Elapsed += duration;
|
||||
}
|
||||
}
|
||||
|
||||
private readonly record struct SoakEvidence(
|
||||
int Cycles,
|
||||
TimeSpan Duration,
|
||||
int PeakScheduledExpiryEntries,
|
||||
long ManagedGrowthBytes,
|
||||
int HandleGrowth);
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<AssemblyName>FinalFactory.Rendezvous.Capacity</AssemblyName>
|
||||
<RootNamespace>FinalFactory.Rendezvous.Capacity</RootNamespace>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../src/FinalFactory.Rendezvous.Contracts/FinalFactory.Rendezvous.Contracts.csproj" />
|
||||
<ProjectReference Include="../../src/FinalFactory.Rendezvous.Server/FinalFactory.Rendezvous.Server.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,37 @@
|
||||
using System.Text.Json;
|
||||
|
||||
namespace FinalFactory.Rendezvous.Capacity;
|
||||
|
||||
internal static class Program
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = true,
|
||||
};
|
||||
|
||||
public static async Task<int> Main(string[] args)
|
||||
{
|
||||
CapacityOptions options;
|
||||
try
|
||||
{
|
||||
options = CapacityOptions.Parse(args);
|
||||
}
|
||||
catch (ArgumentException exception)
|
||||
{
|
||||
Console.Error.WriteLine(exception.Message);
|
||||
return 2;
|
||||
}
|
||||
|
||||
CapacityReport report = await CapacityRunner.RunAsync(options).ConfigureAwait(false);
|
||||
string json = JsonSerializer.Serialize(report, JsonOptions);
|
||||
Console.WriteLine(json);
|
||||
if (options.OutputPath is not null)
|
||||
{
|
||||
string fullPath = Path.GetFullPath(options.OutputPath);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!);
|
||||
await File.WriteAllTextAsync(fullPath, json + Environment.NewLine).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return report.Passed ? 0 : 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"version": 2,
|
||||
"dependencies": {
|
||||
"net10.0": {
|
||||
"finalfactory.rendezvous.contracts": {
|
||||
"type": "Project"
|
||||
},
|
||||
"finalfactory.rendezvous.server": {
|
||||
"type": "Project",
|
||||
"dependencies": {
|
||||
"FinalFactory.Rendezvous.Contracts": "[1.0.0, )",
|
||||
"LiteNetLib": "[2.1.4, )",
|
||||
"Microsoft.AspNetCore.OpenApi": "[10.0.9, )"
|
||||
}
|
||||
},
|
||||
"LiteNetLib": {
|
||||
"type": "CentralTransitive",
|
||||
"requested": "[2.1.4, )",
|
||||
"resolved": "2.1.4",
|
||||
"contentHash": "KWlxvMw3Urpqj9joD96LRiK+LC62pQNs/zkXRJc+rHnxgkGp+vV703xzDrxRmv+V1YhCFfIGzs5nrVWtREIlyA=="
|
||||
},
|
||||
"Microsoft.AspNetCore.OpenApi": {
|
||||
"type": "CentralTransitive",
|
||||
"requested": "[10.0.9, )",
|
||||
"resolved": "10.0.9",
|
||||
"contentHash": "1ihb8FO9cGgEK1/m3CTtT/SfnynwmiZib0W2pcDVj3KSWk/Sca4VOXEtaptKQc582zpFrzTFiwkGRCglt6H+WQ==",
|
||||
"dependencies": {
|
||||
"Microsoft.OpenApi": "2.0.0"
|
||||
}
|
||||
},
|
||||
"Microsoft.OpenApi": {
|
||||
"type": "CentralTransitive",
|
||||
"requested": "[2.7.5, )",
|
||||
"resolved": "2.7.5",
|
||||
"contentHash": "0FA67RSnRM4tcBKqiqVu/HPdZ9+QOKbmeRjxRUGTCjPU4C0bmUhd97Dso7Yild5P7nOV6GxJ2xrK0Kv/O9xp0w=="
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user