feat(operations): add capacity and resilience gates (#18)

tests/FinalFactory.Rendezvous.Capacity/CapacityOptions.cs

@@ -0,0 +1,91 @@
+namespace FinalFactory.Rendezvous.Capacity;
+
+internal sealed record CapacityOptions
+{
+    public required string Profile { get; init; }
+    public required int Listings { get; init; }
+    public required int Attempts { get; init; }
+    public required int Samples { get; init; }
+    public required int SoakCycles { get; init; }
+    public required int SoakSeconds { get; init; }
+    public string? OutputPath { get; init; }
+
+    public static CapacityOptions Parse(string[] args)
+    {
+        Dictionary<string, string> values = ParseArguments(args);
+        string profile = values.GetValueOrDefault("--profile") ?? "quick";
+        (int listings, int attempts, int samples, int soakCycles, int soakSeconds) = profile switch
+        {
+            "quick" => (1_000, 500, 100, 20, 0),
+            "candidate" => (25_000, 10_000, 1_000, 1_000, 300),
+            _ => throw new ArgumentException("--profile must be 'quick' or 'candidate'."),
+        };
+
+        return new()
+        {
+            Profile = profile,
+            Listings = ParsePositive(values.GetValueOrDefault("--listings"), listings, "--listings"),
+            Attempts = ParsePositive(values.GetValueOrDefault("--attempts"), attempts, "--attempts"),
+            Samples = ParsePositive(values.GetValueOrDefault("--samples"), samples, "--samples"),
+            SoakCycles = ParsePositive(
+                values.GetValueOrDefault("--soak-cycles"),
+                soakCycles,
+                "--soak-cycles"),
+            SoakSeconds = ParseNonNegative(
+                values.GetValueOrDefault("--soak-seconds"),
+                soakSeconds,
+                "--soak-seconds"),
+            OutputPath = values.GetValueOrDefault("--output"),
+        };
+    }
+
+    private static Dictionary<string, string> ParseArguments(string[] args)
+    {
+        HashSet<string> allowed =
+        [
+            "--profile",
+            "--listings",
+            "--attempts",
+            "--samples",
+            "--soak-cycles",
+            "--soak-seconds",
+            "--output",
+        ];
+        Dictionary<string, string> values = new(StringComparer.Ordinal);
+        for (int index = 0; index < args.Length; index += 2)
+        {
+            string option = args[index];
+            if (!allowed.Contains(option))
+            {
+                throw new ArgumentException($"Unknown option: {option}.");
+            }
+
+            if (index == args.Length - 1
+                || args[index + 1].StartsWith("--", StringComparison.Ordinal))
+            {
+                throw new ArgumentException($"{option} requires a value.");
+            }
+
+            if (!values.TryAdd(option, args[index + 1]))
+            {
+                throw new ArgumentException($"{option} may be supplied only once.");
+            }
+        }
+
+        return values;
+    }
+
+    private static int ParsePositive(string? value, int fallback, string option) =>
+        value is null
+            ? fallback
+            : int.TryParse(value, out int parsed) && parsed > 0
+                ? parsed
+                : throw new ArgumentException($"{option} must be a positive integer.");
+
+    private static int ParseNonNegative(string? value, int fallback, string option) =>
+        value is null
+            ? fallback
+            : int.TryParse(value, out int parsed) && parsed >= 0
+                ? parsed
+                : throw new ArgumentException($"{option} must be a non-negative integer.");
+}

tests/FinalFactory.Rendezvous.Capacity/CapacityReport.cs

@@ -0,0 +1,76 @@
+namespace FinalFactory.Rendezvous.Capacity;
+
+internal sealed record CapacityReport
+{
+    public required int SchemaVersion { get; init; }
+    public required string EvidenceVersion { get; init; }
+    public required DateTimeOffset GeneratedAt { get; init; }
+    public required string Profile { get; init; }
+    public required RuntimeEvidence Runtime { get; init; }
+    public required CapacityTargets Targets { get; init; }
+    public required IReadOnlyList<CapacityMeasurement> Measurements { get; init; }
+    public required StateEvidence State { get; init; }
+    public required IReadOnlyList<string> Failures { get; init; }
+    public required bool Passed { get; init; }
+}
+
+internal sealed record RuntimeEvidence(
+    string Framework,
+    string OperatingSystem,
+    string Kernel,
+    string Architecture,
+    string CpuModel,
+    int ProcessorCount,
+    string CpuAffinity,
+    string CpuQuota,
+    string MemoryLimit,
+    string GarbageCollector,
+    string CommitSha,
+    string TreeState,
+    string Command,
+    string ImageDigest,
+    string WorkloadSeed,
+    double CapacityPhaseAverageCpuPercent,
+    long PeakWorkingSetBytes,
+    long ManagedBytesAfterCleanup);
+
+internal sealed record CapacityTargets(
+    int VisibleListings,
+    int ActiveJoinAttempts,
+    int CoreControlOperationsPerSecond,
+    int CoreMediationOperationsPerSecond,
+    double CoreControlP95Milliseconds,
+    double CoreMediationP95Milliseconds,
+    double MaximumAverageCpuPercent,
+    long MaximumWorkingSetBytes,
+    int SoakCycles,
+    int SoakDurationSeconds);
+
+internal sealed record CapacityMeasurement(
+    string Operation,
+    int Samples,
+    double P50Milliseconds,
+    double P95Milliseconds,
+    double P99Milliseconds,
+    double OperationsPerSecond,
+    double MinimumOperationsPerSecond,
+    double BudgetMilliseconds,
+    bool Passed);
+
+internal sealed record StateEvidence(
+    int PeakListings,
+    int PeakAttempts,
+    int PeakReplayMarkers,
+    int FinalListings,
+    int FinalAttempts,
+    int FinalReplayMarkers,
+    long ExpiryChurn,
+    long MaintenanceSweeps,
+    int SoakCyclesCompleted,
+    double SoakDurationSeconds,
+    int SoakPeakScheduledExpiryEntries,
+    long SoakManagedGrowthBytes,
+    int SoakHandleGrowth,
+    bool RestartStartedEmpty,
+    bool OverloadWasTyped,
+    bool RecoverySucceeded);

tests/FinalFactory.Rendezvous.Capacity/CapacityRunner.cs

@@ -0,0 +1,580 @@
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Runtime;
+using System.Runtime.InteropServices;
+using FinalFactory.Rendezvous.Contracts;
+using FinalFactory.Rendezvous.Server.Observability;
+using FinalFactory.Rendezvous.Server.State;
+
+namespace FinalFactory.Rendezvous.Capacity;
+
+internal static class CapacityRunner
+{
+    private static readonly TenantScope Scope = new(new("space-game"), new("production"));
+    private const uint ProtocolVersion = 1;
+
+    public static Task<CapacityReport> RunAsync(CapacityOptions options)
+    {
+        ArgumentNullException.ThrowIfNull(options);
+        Process process = Process.GetCurrentProcess();
+        TimeSpan cpuBefore = process.TotalProcessorTime;
+        Stopwatch capacityPhaseTime = Stopwatch.StartNew();
+        List<string> failures = [];
+        List<CapacityMeasurement> measurements = [];
+        ManualClock clock = new();
+        InMemoryEphemeralRendezvousStore store = CreateStore(options, clock);
+        List<StoredListing> listings = new(options.Listings);
+        List<CreateJoinAttemptCommand> attempts = new(options.Attempts);
+        int registrationSamples = Math.Min(options.Samples, options.Listings);
+        int attemptSamples = Math.Min(options.Samples, options.Attempts);
+
+        for (int index = 0; index < options.Listings - registrationSamples; index++)
+        {
+            listings.Add(CreateVisibleListing(store, index));
+        }
+
+        measurements.Add(Measure(
+            "registration-and-presence",
+            registrationSamples,
+            budgetMilliseconds: 200,
+            minimumOperationsPerSecond: 200,
+            index => listings.Add(CreateVisibleListing(
+                store,
+                options.Listings - registrationSamples + index))));
+
+        measurements.Add(Measure(
+            "lease-renewal",
+            registrationSamples,
+            budgetMilliseconds: 200,
+            minimumOperationsPerSecond: 200,
+            index =>
+            {
+                StoredListing listing = listings[index];
+                StoreResult<StoredListing> renewed = store.RenewLease(new(
+                    listing.Definition.ListingId,
+                    listing.Definition.LeaseId,
+                    listing.Definition.LeaseFingerprint,
+                    listing.Definition.OwnerSubject,
+                    listing.Version));
+                RequireSuccess(renewed, "renewal");
+                listings[index] = renewed.Value!;
+            }));
+
+        int browseSamples = Math.Min(options.Samples, 250);
+        measurements.Add(Measure(
+            "visible-session-browse",
+            browseSamples,
+            budgetMilliseconds: 200,
+            minimumOperationsPerSecond: 200,
+            _ => RequireSuccess(
+                store.BrowseVisibleListings(new(
+                    Scope,
+                    ProtocolVersion,
+                    new RegionId("eu-central"),
+                    ContractLimits.BrowserPageMaxItems,
+                    ExcludeFull: true)),
+                "browse")));
+
+        for (int index = 0; index < options.Attempts - attemptSamples; index++)
+        {
+            CreateJoinAttemptCommand command = CreateAttempt(index, listings[index % listings.Count]);
+            RequireSuccess(store.CreateJoinAttempt(command), "join issuance");
+            attempts.Add(command);
+        }
+
+        measurements.Add(Measure(
+            "join-attempt-issuance",
+            attemptSamples,
+            budgetMilliseconds: 200,
+            minimumOperationsPerSecond: 200,
+            index =>
+            {
+                int sequence = options.Attempts - attemptSamples + index;
+                CreateJoinAttemptCommand command = CreateAttempt(
+                    sequence,
+                    listings[sequence % listings.Count]);
+                RequireSuccess(store.CreateJoinAttempt(command), "join issuance");
+                attempts.Add(command);
+            }));
+
+        int punchSamples = Math.Min(options.Samples, attempts.Count);
+        measurements.Add(MeasureConcurrentPunch(store, attempts, punchSamples));
+        EphemeralStoreSnapshot peak = store.GetSnapshot();
+
+        StoreResult<StoredJoinAttempt> overloaded = store.CreateJoinAttempt(
+            CreateAttempt(options.Attempts + 1, listings[^1]));
+        bool overloadWasTyped = peak.ActiveJoinAttempts == options.Attempts
+            && overloaded.Code == StoreResultCode.CapacityExceeded;
+        if (!overloadWasTyped)
+        {
+            failures.Add(
+                $"Expected typed CapacityExceeded at {options.Attempts} active attempts, "
+                + $"observed count={peak.ActiveJoinAttempts}, result={overloaded.Code}.");
+        }
+
+        int revocationSamples = Math.Min(Math.Max(1, options.Samples / 20), listings.Count / 2);
+        measurements.Add(Measure(
+            "principal-revocation",
+            revocationSamples,
+            budgetMilliseconds: 200,
+            minimumOperationsPerSecond: 50,
+            index => RequireSuccess(
+                store.RevokePrincipal(
+                    listings[index].Definition.OwnerSubject,
+                    TimeSpan.FromMinutes(1)),
+                "principal revocation")));
+
+        using (RendezvousTelemetry telemetry = new(store))
+        {
+            measurements.Add(Measure(
+                "telemetry-recording",
+                Math.Max(100, options.Samples),
+                budgetMilliseconds: 1,
+                minimumOperationsPerSecond: 10_000,
+                _ =>
+                {
+                    telemetry.RecordHttp("browse", 200, 1);
+                    telemetry.RecordUdp("contribution", "accepted", 1);
+                    telemetry.RecordPairingLatency(2);
+                }));
+        }
+
+        clock.Advance(TimeSpan.FromSeconds(61));
+        EphemeralStoreSnapshot? afterCoincidentExpiry = null;
+        measurements.Add(Measure(
+            "coincident-listing-attempt-expiry",
+            samples: 1,
+            budgetMilliseconds: 200,
+            minimumOperationsPerSecond: 0,
+            _ => afterCoincidentExpiry = store.GetSnapshot()));
+        if (afterCoincidentExpiry!.ActiveJoinAttempts != 0
+            || afterCoincidentExpiry.ActiveListings != 0)
+        {
+            failures.Add("Coincident 60-second cleanup retained expired listings or join attempts.");
+        }
+
+        clock.Advance(TimeSpan.FromSeconds(90));
+        _ = store.GetSnapshot();
+        StoredListing recoveryListing = CreateVisibleListing(store, options.Listings + 1);
+        StoreResult<StoredJoinAttempt> recovered = store.CreateJoinAttempt(
+            CreateAttempt(options.Attempts + 2, recoveryListing));
+        bool recoverySucceeded = recovered.Succeeded;
+        if (!recoverySucceeded)
+        {
+            failures.Add($"Store did not recover after attempt expiry: {recovered.Code}.");
+        }
+
+        clock.Advance(TimeSpan.FromSeconds(151));
+        EphemeralStoreSnapshot final = store.GetSnapshot();
+        if (final.ActiveListings != 0
+            || final.ActiveJoinAttempts != 0
+            || final.ReplayMarkers != 0
+            || final.IdempotencyEntries != 0
+            || final.RetainedOutcomeReports != 0)
+        {
+            failures.Add("Expiry cleanup left active or retained state after every configured deadline.");
+        }
+
+        capacityPhaseTime.Stop();
+        process.Refresh();
+        double capacityPhaseCpuPercent = 100
+            * (process.TotalProcessorTime - cpuBefore).TotalSeconds
+            / Math.Max(capacityPhaseTime.Elapsed.TotalSeconds * Environment.ProcessorCount, 0.000_001);
+        if (options.Profile == "candidate" && capacityPhaseCpuPercent > 70)
+        {
+            failures.Add(
+                $"Capacity-phase CPU {capacityPhaseCpuPercent:F1}% exceeded the 70% candidate budget.");
+        }
+
+        SoakEvidence soak = RunAcceleratedSoak(options, failures);
+        ManualClock restartClock = new();
+        EphemeralStoreSnapshot restarted = CreateStore(options, restartClock).GetSnapshot();
+        bool restartStartedEmpty = restarted.ActiveListings == 0
+            && restarted.ActiveJoinAttempts == 0
+            && restarted.ReplayMarkers == 0;
+        if (!restartStartedEmpty)
+        {
+            failures.Add("A restarted store did not begin empty.");
+        }
+
+        foreach (CapacityMeasurement measurement in measurements.Where(static item => !item.Passed))
+        {
+            failures.Add(
+                $"{measurement.Operation} missed its budget: p95={measurement.P95Milliseconds:F3} ms, "
+                + $"rate={measurement.OperationsPerSecond:F1}/s.");
+        }
+
+        GC.Collect();
+        GC.WaitForPendingFinalizers();
+        GC.Collect();
+        process.Refresh();
+        long managedAfterCleanup = GC.GetTotalMemory(forceFullCollection: true);
+        long memoryBudget = 1_610_612_736;
+        if (process.PeakWorkingSet64 > memoryBudget)
+        {
+            failures.Add(
+                $"Peak working set {process.PeakWorkingSet64} exceeded the 1.5 GiB profile budget.");
+        }
+
+        if (options.Profile == "candidate" && Environment.ProcessorCount != 2)
+        {
+            failures.Add(
+                $"Candidate evidence must expose exactly two CPUs; runtime exposed "
+                + $"{Environment.ProcessorCount}.");
+        }
+
+        CapacityReport report = new()
+        {
+            SchemaVersion = 2,
+            EvidenceVersion = "v2",
+            GeneratedAt = DateTimeOffset.UtcNow,
+            Profile = options.Profile,
+            Runtime = new(
+                RuntimeInformation.FrameworkDescription,
+                RuntimeInformation.OSDescription,
+                Environment.OSVersion.VersionString,
+                RuntimeInformation.ProcessArchitecture.ToString(),
+                ReadCpuModel(),
+                Environment.ProcessorCount,
+                Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_CPUSET") ?? "unrestricted",
+                ReadCgroupValue("/sys/fs/cgroup/cpu.max"),
+                ReadCgroupValue("/sys/fs/cgroup/memory.max"),
+                GCSettings.IsServerGC ? "server" : "workstation",
+                Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_COMMIT") ?? "unrecorded",
+                Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_TREE_STATE") ?? "unrecorded",
+                Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_COMMAND") ?? "unrecorded",
+                Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_IMAGE_DIGEST") ?? "not-containerized",
+                "fixed-sequences-random-identifiers",
+                capacityPhaseCpuPercent,
+                process.PeakWorkingSet64,
+                managedAfterCleanup),
+            Targets = new(
+                options.Listings,
+                options.Attempts,
+                200,
+                2_000,
+                200,
+                100,
+                70,
+                memoryBudget,
+                options.SoakCycles,
+                options.SoakSeconds),
+            Measurements = measurements,
+            State = new(
+                peak.ActiveListings,
+                peak.ActiveJoinAttempts,
+                peak.ReplayMarkers,
+                final.ActiveListings,
+                final.ActiveJoinAttempts,
+                final.ReplayMarkers,
+                final.ExpiryChurn,
+                final.MaintenanceSweeps,
+                soak.Cycles,
+                soak.Duration.TotalSeconds,
+                soak.PeakScheduledExpiryEntries,
+                soak.ManagedGrowthBytes,
+                soak.HandleGrowth,
+                restartStartedEmpty,
+                overloadWasTyped,
+                recoverySucceeded),
+            Failures = failures,
+            Passed = failures.Count == 0,
+        };
+        return Task.FromResult(report);
+    }
+
+    private static InMemoryEphemeralRendezvousStore CreateStore(
+        CapacityOptions options,
+        ManualClock clock) => new(
+        new EphemeralStoreOptions
+        {
+            MaxListings = options.Listings,
+            MaxPresenceBindings = options.Listings,
+            MaxJoinAttempts = options.Attempts,
+            MaxOutcomeReports = options.Attempts,
+            MaxIdempotencyEntries = options.Listings + options.Attempts + 1,
+        },
+        clock,
+        clock);
+
+    private static StoredListing CreateVisibleListing(
+        InMemoryEphemeralRendezvousStore store,
+        int sequence)
+    {
+        string owner = $"publisher-{sequence}";
+        SecretFingerprint leaseFingerprint = new($"lease-{sequence}");
+        SecretFingerprint presenceFingerprint = new($"presence-{sequence}");
+        ListingDefinition definition = new()
+        {
+            ListingId = new(Guid.NewGuid()),
+            LeaseId = new(Guid.NewGuid()),
+            Scope = Scope,
+            OwnerSubject = owner,
+            RegionId = new("eu-central"),
+            ProtocolVersion = ProtocolVersion,
+            BuildVersion = "1.0.0",
+            DisplayName = $"Capacity host {sequence}",
+            Visibility = ListingVisibility.Public,
+            TrustMode = PublisherTrustMode.ManagedDedicated,
+            CurrentPlayers = 1,
+            MaximumPlayers = 8,
+            Metadata = new Dictionary<string, string>(StringComparer.Ordinal),
+            LeaseFingerprint = leaseFingerprint,
+            HostPresenceHandle = new(Guid.NewGuid()),
+            HostPresenceFingerprint = presenceFingerprint,
+            CapabilityDerivationSalt = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+        };
+        StoreResult<StoredListing> created = store.CreateListing(new(
+            $"register-{sequence}",
+            $"register-request-{sequence}",
+            definition));
+        RequireSuccess(created, "registration");
+        StoreResult<StoredListing> bound = store.BindHostPresence(new(
+            definition.HostPresenceHandle,
+            presenceFingerprint,
+            PublicEndpoint(10_000 + sequence % 50_000),
+            null));
+        RequireSuccess(bound, "host presence");
+        return bound.Value!;
+    }
+
+    private static CreateJoinAttemptCommand CreateAttempt(int sequence, StoredListing listing) => new()
+    {
+        IdempotencyOwner = $"client-{sequence}",
+        IdempotencyKey = $"join-{sequence}",
+        RequestFingerprint = $"join-request-{sequence}",
+        ClientSubject = $"client-{sequence}",
+        AttemptId = new(Guid.NewGuid()),
+        MediationHandle = new(Guid.NewGuid()),
+        Scope = Scope,
+        ListingId = listing.Definition.ListingId,
+        ProtocolVersion = ProtocolVersion,
+        HostCapabilityFingerprint = new($"host-capability-{sequence}"),
+        ClientCapabilityFingerprint = new($"client-capability-{sequence}"),
+        ConnectionTicketFingerprint = new($"ticket-{sequence}"),
+        CapabilityDerivationSalt = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+    };
+
+    private static CapacityMeasurement MeasureConcurrentPunch(
+        InMemoryEphemeralRendezvousStore store,
+        List<CreateJoinAttemptCommand> attempts,
+        int samples)
+    {
+        ConcurrentBag<double> latencies = [];
+        Stopwatch total = Stopwatch.StartNew();
+        Parallel.ForEach(
+            Enumerable.Range(0, samples),
+            new ParallelOptions { MaxDegreeOfParallelism = Math.Min(64, Environment.ProcessorCount * 4) },
+            index =>
+            {
+                CreateJoinAttemptCommand attempt = attempts[index];
+                Stopwatch elapsed = Stopwatch.StartNew();
+                RequireSuccess(store.BindAttemptEndpoint(new(
+                    attempt.MediationHandle,
+                    AttemptPeerRole.Host,
+                    attempt.HostCapabilityFingerprint,
+                    PublicEndpoint(20_000 + index % 20_000),
+                    null)), "host punch");
+                RequireSuccess(store.BindAttemptEndpoint(new(
+                    attempt.MediationHandle,
+                    AttemptPeerRole.Client,
+                    attempt.ClientCapabilityFingerprint,
+                    PublicEndpoint(40_000 + index % 20_000),
+                    null)), "client punch");
+                RequireSuccess(store.ConsumeIntroduction(attempt.MediationHandle), "introduction");
+                latencies.Add(elapsed.Elapsed.TotalMilliseconds);
+            });
+        total.Stop();
+        return BuildMeasurement(
+            "simultaneous-punch-pairing",
+            latencies.ToArray(),
+            total.Elapsed,
+            budgetMilliseconds: 100,
+            minimumOperationsPerSecond: 2_000);
+    }
+
+    private static CapacityMeasurement Measure(
+        string operation,
+        int samples,
+        double budgetMilliseconds,
+        double minimumOperationsPerSecond,
+        Action<int> action)
+    {
+        double[] latencies = new double[samples];
+        Stopwatch total = Stopwatch.StartNew();
+        for (int index = 0; index < samples; index++)
+        {
+            long started = Stopwatch.GetTimestamp();
+            action(index);
+            latencies[index] = Stopwatch.GetElapsedTime(started).TotalMilliseconds;
+        }
+
+        total.Stop();
+        return BuildMeasurement(
+            operation,
+            latencies,
+            total.Elapsed,
+            budgetMilliseconds,
+            minimumOperationsPerSecond);
+    }
+
+    private static CapacityMeasurement BuildMeasurement(
+        string operation,
+        double[] latencies,
+        TimeSpan elapsed,
+        double budgetMilliseconds,
+        double minimumOperationsPerSecond)
+    {
+        Array.Sort(latencies);
+        double operationsPerSecond = latencies.Length / Math.Max(elapsed.TotalSeconds, 0.000_001);
+        double p95 = Percentile(latencies, 0.95);
+        return new(
+            operation,
+            latencies.Length,
+            Percentile(latencies, 0.50),
+            p95,
+            Percentile(latencies, 0.99),
+            operationsPerSecond,
+            minimumOperationsPerSecond,
+            budgetMilliseconds,
+            p95 <= budgetMilliseconds && operationsPerSecond >= minimumOperationsPerSecond);
+    }
+
+    private static double Percentile(double[] sorted, double percentile)
+    {
+        int index = Math.Clamp((int)Math.Ceiling(sorted.Length * percentile) - 1, 0, sorted.Length - 1);
+        return sorted[index];
+    }
+
+    private static SoakEvidence RunAcceleratedSoak(
+        CapacityOptions options,
+        List<string> failures)
+    {
+        GC.Collect();
+        GC.WaitForPendingFinalizers();
+        GC.Collect();
+        long managedBefore = GC.GetTotalMemory(forceFullCollection: true);
+        int handlesBefore = Process.GetCurrentProcess().HandleCount;
+        ManualClock clock = new();
+        CapacityOptions soakOptions = options with { Listings = 100, Attempts = 100 };
+        InMemoryEphemeralRendezvousStore store = CreateStore(soakOptions, clock);
+        Stopwatch elapsed = Stopwatch.StartNew();
+        int cycle = 0;
+        int peakScheduledExpiryEntries = 0;
+        while (cycle < options.SoakCycles
+               || elapsed.Elapsed < TimeSpan.FromSeconds(options.SoakSeconds))
+        {
+            StoredListing listing = CreateVisibleListing(store, cycle);
+            for (int refresh = 0; refresh < 10; refresh++)
+            {
+                clock.Advance(TimeSpan.FromTicks(1));
+                listing = RequireSuccess(store.RenewLease(new(
+                    listing.Definition.ListingId,
+                    listing.Definition.LeaseId,
+                    listing.Definition.LeaseFingerprint,
+                    listing.Definition.OwnerSubject,
+                    listing.Version)), "soak lease refresh");
+                listing = RequireSuccess(store.BindHostPresence(new(
+                    listing.Definition.HostPresenceHandle,
+                    listing.Definition.HostPresenceFingerprint,
+                    PublicEndpoint(10_000 + cycle % 50_000),
+                    null)), "soak presence refresh");
+            }
+
+            CreateJoinAttemptCommand attempt = CreateAttempt(cycle, listing);
+            RequireSuccess(store.CreateJoinAttempt(attempt), "soak join issuance");
+            RequireSuccess(store.ConsumeReplay(new("capacity-soak", $"replay-{cycle}")), "soak replay");
+            peakScheduledExpiryEntries = Math.Max(
+                peakScheduledExpiryEntries,
+                store.ScheduledExpiryEntryCount);
+            if (store.ScheduledExpiryEntryCount > 7)
+            {
+                failures.Add(
+                    $"Mutable deadline refresh grew the expiry queue to "
+                    + $"{store.ScheduledExpiryEntryCount} entries for one lifecycle.");
+                break;
+            }
+
+            clock.Advance(TimeSpan.FromSeconds(151));
+            EphemeralStoreSnapshot snapshot = store.GetSnapshot();
+            if (snapshot.ActiveListings != 0
+                || snapshot.ActiveJoinAttempts != 0
+                || snapshot.ReplayMarkers != 0
+                || snapshot.IdempotencyEntries != 0
+                || snapshot.RetainedOutcomeReports != 0)
+            {
+                failures.Add($"Accelerated soak retained state after cycle {cycle}.");
+                break;
+            }
+
+            cycle++;
+        }
+
+        elapsed.Stop();
+        GC.Collect();
+        GC.WaitForPendingFinalizers();
+        GC.Collect();
+        long managedGrowth = GC.GetTotalMemory(forceFullCollection: true) - managedBefore;
+        int handleGrowth = Process.GetCurrentProcess().HandleCount - handlesBefore;
+        if (managedGrowth > 67_108_864)
+        {
+            failures.Add($"Soak retained {managedGrowth} managed bytes; budget is 64 MiB.");
+        }
+
+        if (handleGrowth > 8)
+        {
+            failures.Add($"Soak retained {handleGrowth} process handles; budget is 8.");
+        }
+
+        return new(cycle, elapsed.Elapsed, peakScheduledExpiryEntries, managedGrowth, handleGrowth);
+    }
+
+    private static ObservedEndpoint PublicEndpoint(int port) =>
+        new(AddressFamilyKind.Ipv4, "203.0.113.10", port);
+
+    private static T RequireSuccess<T>(StoreResult<T> result, string operation)
+    {
+        if (!result.Succeeded)
+        {
+            throw new InvalidOperationException($"{operation} failed with {result.Code}.");
+        }
+
+        return result.Value!;
+    }
+
+    private static string ReadCpuModel()
+    {
+        const string cpuInfoPath = "/proc/cpuinfo";
+        if (!File.Exists(cpuInfoPath))
+        {
+            return "unavailable";
+        }
+
+        string? model = File.ReadLines(cpuInfoPath)
+            .FirstOrDefault(static line => line.StartsWith("model name", StringComparison.Ordinal));
+        int separator = model?.IndexOf(':') ?? -1;
+        return separator >= 0 ? model![(separator + 1)..].Trim() : "unavailable";
+    }
+
+    private static string ReadCgroupValue(string path) =>
+        File.Exists(path) ? File.ReadAllText(path).Trim() : "not-enforced";
+
+    private sealed class ManualClock : IWallClock, IMonotonicClock
+    {
+        public DateTimeOffset UtcNow { get; private set; } = DateTimeOffset.UtcNow;
+        public TimeSpan Elapsed { get; private set; }
+
+        public void Advance(TimeSpan duration)
+        {
+            UtcNow += duration;
+            Elapsed += duration;
+        }
+    }
+
+    private readonly record struct SoakEvidence(
+        int Cycles,
+        TimeSpan Duration,
+        int PeakScheduledExpiryEntries,
+        long ManagedGrowthBytes,
+        int HandleGrowth);
+}

tests/FinalFactory.Rendezvous.Capacity/FinalFactory.Rendezvous.Capacity.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <AssemblyName>FinalFactory.Rendezvous.Capacity</AssemblyName>
+    <RootNamespace>FinalFactory.Rendezvous.Capacity</RootNamespace>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="../../src/FinalFactory.Rendezvous.Contracts/FinalFactory.Rendezvous.Contracts.csproj" />
+    <ProjectReference Include="../../src/FinalFactory.Rendezvous.Server/FinalFactory.Rendezvous.Server.csproj" />
+  </ItemGroup>
+</Project>

tests/FinalFactory.Rendezvous.Capacity/Program.cs

@@ -0,0 +1,37 @@
+using System.Text.Json;
+
+namespace FinalFactory.Rendezvous.Capacity;
+
+internal static class Program
+{
+    private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
+    {
+        WriteIndented = true,
+    };
+
+    public static async Task<int> Main(string[] args)
+    {
+        CapacityOptions options;
+        try
+        {
+            options = CapacityOptions.Parse(args);
+        }
+        catch (ArgumentException exception)
+        {
+            Console.Error.WriteLine(exception.Message);
+            return 2;
+        }
+
+        CapacityReport report = await CapacityRunner.RunAsync(options).ConfigureAwait(false);
+        string json = JsonSerializer.Serialize(report, JsonOptions);
+        Console.WriteLine(json);
+        if (options.OutputPath is not null)
+        {
+            string fullPath = Path.GetFullPath(options.OutputPath);
+            Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!);
+            await File.WriteAllTextAsync(fullPath, json + Environment.NewLine).ConfigureAwait(false);
+        }
+
+        return report.Passed ? 0 : 1;
+    }
+}

tests/FinalFactory.Rendezvous.Capacity/packages.lock.json

@@ -0,0 +1,39 @@
+{
+  "version": 2,
+  "dependencies": {
+    "net10.0": {
+      "finalfactory.rendezvous.contracts": {
+        "type": "Project"
+      },
+      "finalfactory.rendezvous.server": {
+        "type": "Project",
+        "dependencies": {
+          "FinalFactory.Rendezvous.Contracts": "[1.0.0, )",
+          "LiteNetLib": "[2.1.4, )",
+          "Microsoft.AspNetCore.OpenApi": "[10.0.9, )"
+        }
+      },
+      "LiteNetLib": {
+        "type": "CentralTransitive",
+        "requested": "[2.1.4, )",
+        "resolved": "2.1.4",
+        "contentHash": "KWlxvMw3Urpqj9joD96LRiK+LC62pQNs/zkXRJc+rHnxgkGp+vV703xzDrxRmv+V1YhCFfIGzs5nrVWtREIlyA=="
+      },
+      "Microsoft.AspNetCore.OpenApi": {
+        "type": "CentralTransitive",
+        "requested": "[10.0.9, )",
+        "resolved": "10.0.9",
+        "contentHash": "1ihb8FO9cGgEK1/m3CTtT/SfnynwmiZib0W2pcDVj3KSWk/Sca4VOXEtaptKQc582zpFrzTFiwkGRCglt6H+WQ==",
+        "dependencies": {
+          "Microsoft.OpenApi": "2.0.0"
+        }
+      },
+      "Microsoft.OpenApi": {
+        "type": "CentralTransitive",
+        "requested": "[2.7.5, )",
+        "resolved": "2.7.5",
+        "contentHash": "0FA67RSnRM4tcBKqiqVu/HPdZ9+QOKbmeRjxRUGTCjPU4C0bmUhd97Dso7Yild5P7nOV6GxJ2xrK0Kv/O9xp0w=="
+      }
+    }
+  }
+}

tests/FinalFactory.Rendezvous.Tests/Client/RendezvousClientIntegrationTests.cs

@@ -1,3 +1,6 @@
+using System.Diagnostics;
+using System.Net;
+using System.Net.Sockets;
 using FinalFactory.Rendezvous.Client;
 using FinalFactory.Rendezvous.Contracts;
 using FinalFactory.Rendezvous.Server.Abuse;
@@ -19,6 +22,49 @@ namespace FinalFactory.Rendezvous.Tests.Client;
 
 public sealed class RendezvousClientIntegrationTests
 {
+    [Fact]
+    public async Task RestartReturnsTypedUnavailabilityThenAllowsHostReregistration()
+    {
+        int port = ReserveTcpPort();
+        string address = $"http://127.0.0.1:{port}";
+        using HttpClient client = new() { BaseAddress = new Uri(address) };
+        RendezvousClientOptions noRetry = new()
+        {
+            MaximumSafeRetries = 0,
+            RequestTimeout = TimeSpan.FromSeconds(1),
+        };
+        ClientTestHost first = await ClientTestHost.StartAsync(address);
+        RendezvousPublisherClient publisher = new(client, noRetry);
+        RendezvousClientResult<PublishedSession> registered = await publisher.RegisterAsync(
+            CreateRegistration(100),
+            first.PublisherCredential);
+        PublishedSession initialSession = AssertSuccess(registered);
+        BindPresence(first, initialSession, 41_100);
+        RendezvousSessionBrowserClient browser = new(client, noRetry);
+        BrowseSessionsResponse beforeRestart = AssertSuccess(await browser.BrowseAsync(BrowseRequest()));
+        Assert.Equal(initialSession.ListingId, Assert.Single(beforeRestart.Items).ListingId);
+        Stopwatch restart = Stopwatch.StartNew();
+        await first.DisposeAsync();
+
+        RendezvousClientResult<BrowseSessionsResponse> unavailable = await browser.BrowseAsync(BrowseRequest());
+        Assert.False(unavailable.IsSuccess);
+        Assert.Equal(RendezvousErrorCode.ServiceUnavailable, unavailable.Error);
+
+        await using ClientTestHost second = await ClientTestHost.StartAsync(address);
+        RendezvousClientResult<PublishedSession> reregistered = await publisher.RegisterAsync(
+            CreateRegistration(101),
+            second.PublisherCredential);
+        PublishedSession replacementSession = AssertSuccess(reregistered);
+        Assert.NotEqual(initialSession.ListingId, replacementSession.ListingId);
+        BindPresence(second, replacementSession, 41_101);
+        BrowseSessionsResponse afterRestart = AssertSuccess(await browser.BrowseAsync(BrowseRequest()));
+        Assert.Equal(replacementSession.ListingId, Assert.Single(afterRestart.Items).ListingId);
+        Assert.DoesNotContain(afterRestart.Items, item => item.ListingId == initialSession.ListingId);
+        Assert.True(
+            restart.Elapsed < TimeSpan.FromSeconds(5),
+            $"Local restart and host re-registration took {restart.Elapsed}.");
+    }
+
     [Fact]
     public async Task PublisherAndBrowserClientsCompleteTheRealSessionLifecycleAndPaging()
     {
@@ -102,6 +148,27 @@ public sealed class RendezvousClientIntegrationTests
         return Assert.IsAssignableFrom<T>(result.Value);
     }
 
+    private static void BindPresence(ClientTestHost host, PublishedSession session, int port)
+    {
+        Assert.True(host.Capabilities.TryFingerprint(
+            session.HostPresenceCapability,
+            out SecretFingerprint fingerprint));
+        Assert.Equal(StoreResultCode.Success, host.Store.BindHostPresence(new(
+            session.HostPresenceHandle,
+            fingerprint,
+            new(AddressFamilyKind.Ipv4, "203.0.113.80", port),
+            null)).Code);
+    }
+
+    private static BrowseSessionsRequest BrowseRequest() => new()
+    {
+        GameId = new("space-game"),
+        EnvironmentId = new("production"),
+        RegionId = new("eu-central"),
+        ProtocolVersion = 7,
+        PageSize = 10,
+    };
+
     private static RegisterSessionRequest CreateRegistration(int index) => new()
     {
         IdempotencyKey = $"sdk-integration-{index}",
@@ -139,7 +206,7 @@ public sealed class RendezvousClientIntegrationTests
         internal EphemeralCapabilityIssuer Capabilities { get; }
         internal string PublisherCredential { get; }
 
-        internal static async Task<ClientTestHost> StartAsync()
+        internal static async Task<ClientTestHost> StartAsync(string? bindAddress = null)
         {
             ManualRendezvousClock clock = new(ProvisioningTestData.Now);
             EphemeralStoreOptions stateOptions = new();
@@ -153,7 +220,7 @@ public sealed class RendezvousClientIntegrationTests
             string credential = provisioning.Credentials.Issue(principal, clock.UtcNow);
 
             WebApplicationBuilder builder = WebApplication.CreateBuilder();
-            builder.WebHost.UseUrls("http://127.0.0.1:0");
+            builder.WebHost.UseUrls(bindAddress ?? "http://127.0.0.1:0");
             builder.Services.ConfigureHttpJsonOptions(static options =>
                 ContractJson.Configure(options.SerializerOptions));
             builder.Services.Configure<RouteHandlerOptions>(static options =>
@@ -180,10 +247,10 @@ public sealed class RendezvousClientIntegrationTests
             app.MapRendezvousContractEndpoints();
             await app.StartAsync();
             IServer server = app.Services.GetRequiredService<IServer>();
-            string address = Assert.Single(server.Features.Get<IServerAddressesFeature>()!.Addresses);
+            string serviceAddress = Assert.Single(server.Features.Get<IServerAddressesFeature>()!.Addresses);
             return new(
                 app,
-                new HttpClient { BaseAddress = new Uri(address) },
+                new HttpClient { BaseAddress = new Uri(serviceAddress) },
                 store,
                 capabilities,
                 credential);
@@ -196,4 +263,13 @@ public sealed class RendezvousClientIntegrationTests
             await _application.DisposeAsync();
         }
     }
+
+    private static int ReserveTcpPort()
+    {
+        TcpListener listener = new(IPAddress.Loopback, 0);
+        listener.Start();
+        int port = ((IPEndPoint)listener.LocalEndpoint).Port;
+        listener.Stop();
+        return port;
+    }
 }

tests/FinalFactory.Rendezvous.Tests/Deployment/ProductionProcessTests.cs

@@ -61,6 +61,32 @@ public sealed class ProductionProcessTests
             Assert.InRange(shutdown.Elapsed, TimeSpan.FromMilliseconds(700), TimeSpan.FromSeconds(5));
             AssertTcpPortIsReleased(httpPort);
             AssertUdpPortIsReleased(udpPort);
+
+            process.Dispose();
+            process = null;
+            Stopwatch replacementReady = Stopwatch.StartNew();
+            ProcessStartInfo replacementInfo = CreateStartInfo(httpPort, udpPort, secretPath);
+            process = Process.Start(replacementInfo)
+                ?? throw new InvalidOperationException("The replacement production process did not start.");
+            Task<string> replacementOutput = process.StandardOutput.ReadToEndAsync();
+            Task<string> replacementError = process.StandardError.ReadToEndAsync();
+            await WaitForReadyAsync(httpPort, process, TimeSpan.FromSeconds(15));
+            Assert.True(
+                replacementReady.Elapsed < TimeSpan.FromSeconds(15),
+                $"Replacement readiness took {replacementReady.Elapsed}.");
+            AssertUdpPortIsBound(udpPort);
+
+            await SendSigtermAsync(process);
+            using CancellationTokenSource replacementTimeout = new(TimeSpan.FromSeconds(6));
+            await process.WaitForExitAsync(replacementTimeout.Token);
+            string replacementFinalOutput = await replacementOutput;
+            string replacementFinalError = await replacementError;
+            Assert.True(
+                process.ExitCode == 0,
+                $"Replacement exited with {process.ExitCode}. "
+                + $"stdout: {replacementFinalOutput} stderr: {replacementFinalError}");
+            AssertTcpPortIsReleased(httpPort);
+            AssertUdpPortIsReleased(udpPort);
         }
         finally
         {
@@ -79,6 +105,97 @@ public sealed class ProductionProcessTests
         }
     }
 
+    [Fact]
+    public async Task ProductionTransportSoakKeepsHandlesMemoryAndSocketsBounded()
+    {
+        if (!OperatingSystem.IsLinux())
+        {
+            return;
+        }
+
+        int httpPort = ReserveTcpPort();
+        int udpPort = ReserveUdpPort();
+        string secretPath = Path.Combine(
+            Path.GetTempPath(),
+            $"rendezvous-transport-soak-secret-{Guid.NewGuid():N}");
+        await File.WriteAllBytesAsync(secretPath, RandomNumberGenerator.GetBytes(32));
+        Process? process = null;
+        try
+        {
+            process = Process.Start(CreateStartInfo(httpPort, udpPort, secretPath))
+                ?? throw new InvalidOperationException("The production soak process did not start.");
+            Task<string> standardOutput = process.StandardOutput.ReadToEndAsync();
+            Task<string> standardError = process.StandardError.ReadToEndAsync();
+            await WaitForReadyAsync(httpPort, process, TimeSpan.FromSeconds(10));
+            process.Refresh();
+            int baselineHandles = process.HandleCount;
+            long baselineWorkingSet = process.WorkingSet64;
+            int peakHandles = baselineHandles;
+            byte[] invalidDatagram = RandomNumberGenerator.GetBytes(64);
+            IPEndPoint udpEndpoint = new(IPAddress.Loopback, udpPort);
+            using HttpClient client = new() { Timeout = TimeSpan.FromSeconds(1) };
+            using UdpClient udp = new();
+            Stopwatch soak = Stopwatch.StartNew();
+            int cycles = 0;
+            int accepted = 0;
+            int shed = 0;
+            while (soak.Elapsed < TimeSpan.FromSeconds(10))
+            {
+                using HttpResponseMessage response = await client.GetAsync(
+                    $"http://127.0.0.1:{httpPort}/health/live");
+                if (response.StatusCode == HttpStatusCode.OK)
+                {
+                    accepted++;
+                }
+                else
+                {
+                    Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode);
+                    shed++;
+                }
+
+                await udp.SendAsync(invalidDatagram, udpEndpoint);
+                cycles++;
+                if (cycles % 100 == 0)
+                {
+                    process.Refresh();
+                    peakHandles = Math.Max(peakHandles, process.HandleCount);
+                }
+            }
+
+            Assert.True(cycles >= 100, $"Transport soak completed only {cycles} cycles.");
+            Assert.True(accepted > 0, "Transport soak never admitted a health request.");
+            Assert.True(shed > 0, "Transport soak never exercised typed HTTP load shedding.");
+            await Task.Delay(TimeSpan.FromSeconds(2));
+            using (HttpResponseMessage recovered = await client.GetAsync(
+                       $"http://127.0.0.1:{httpPort}/health/live"))
+            {
+                Assert.Equal(HttpStatusCode.OK, recovered.StatusCode);
+            }
+
+            process.Refresh();
+            Assert.InRange(peakHandles, 0, baselineHandles + 32);
+            Assert.InRange(process.HandleCount, 0, baselineHandles + 16);
+            Assert.InRange(process.WorkingSet64, 0, baselineWorkingSet + 67_108_864);
+            AssertUdpPortIsBound(udpPort);
+
+            await SendSigtermAsync(process);
+            using CancellationTokenSource shutdownTimeout = new(TimeSpan.FromSeconds(6));
+            await process.WaitForExitAsync(shutdownTimeout.Token);
+            string output = await standardOutput;
+            string error = await standardError;
+            Assert.True(
+                process.ExitCode == 0,
+                $"Transport soak process failed. stdout: {output} stderr: {error}");
+            AssertTcpPortIsReleased(httpPort);
+            AssertUdpPortIsReleased(udpPort);
+        }
+        finally
+        {
+            await StopProcessTreeAsync(process);
+            File.Delete(secretPath);
+        }
+    }
+
     [Fact]
     public async Task DocumentedSmokeScriptReachesHttpAndAuthenticatedUdpFlow()
     {

tests/FinalFactory.Rendezvous.Tests/State/InMemoryEphemeralRendezvousStoreTests.cs

@@ -411,6 +411,92 @@ public sealed class InMemoryEphemeralRendezvousStoreTests
         Assert.True(fixture.Store.CreateJoinAttempt(attempt).Succeeded);
         Assert.Equal(StoreResultCode.CapacityExceeded, fixture.Store.CreateJoinAttempt(
             fixture.AttemptCommand(listing, "client-quota-2") with { ScopeAttemptLimit = 1 }).Code);
+
+        fixture.Clock.Advance(TimeSpan.FromSeconds(30));
+        Assert.True(fixture.Store.BindHostPresence(new(
+            first.Listing.HostPresenceHandle,
+            first.Listing.HostPresenceFingerprint,
+            EphemeralStateFixture.PublicEndpoint(42_101),
+            null)).Succeeded);
+        Assert.True(fixture.Store.CreateJoinAttempt(
+            fixture.AttemptCommand(listing, "client-quota-3") with { ScopeAttemptLimit = 1 }).Succeeded);
+    }
+
+    [Fact]
+    public void RemovingAListingReleasesItsConstantTimeOwnerQuota()
+    {
+        EphemeralStateFixture fixture = new();
+        CreateListingCommand first = fixture.ListingCommand(owner: "publisher-quota") with
+        {
+            OwnerListingLimit = 1,
+        };
+        CreateListingCommand second = fixture.ListingCommand(owner: "publisher-quota") with
+        {
+            OwnerListingLimit = 1,
+        };
+        StoreResult<StoredListing> created = fixture.Store.CreateListing(first);
+        Assert.True(created.Succeeded);
+        Assert.Equal(StoreResultCode.CapacityExceeded, fixture.Store.CreateListing(second).Code);
+
+        Assert.True(fixture.Store.DeleteListing(new(
+            first.Listing.ListingId,
+            first.Listing.LeaseId,
+            first.Listing.LeaseFingerprint,
+            first.Listing.OwnerSubject)).Succeeded);
+        Assert.True(fixture.Store.CreateListing(second).Succeeded);
+    }
+
+    [Fact]
+    public void RepeatedMutableDeadlineRefreshesKeepOneScheduledEntryPerKey()
+    {
+        EphemeralStateFixture fixture = new();
+        StoredListing listing = fixture.CreateVisibleListing(out _);
+
+        for (int iteration = 0; iteration < 10_000; iteration++)
+        {
+            fixture.Clock.Advance(TimeSpan.FromTicks(1));
+            StoreResult<StoredListing> renewed = fixture.Store.RenewLease(new(
+                listing.Definition.ListingId,
+                listing.Definition.LeaseId,
+                listing.Definition.LeaseFingerprint,
+                listing.Definition.OwnerSubject,
+                listing.Version));
+            Assert.True(renewed.Succeeded, $"Renewal {iteration} failed with {renewed.Code}.");
+            listing = renewed.Value!;
+            StoreResult<StoredListing> presence = fixture.Store.BindHostPresence(new(
+                listing.Definition.HostPresenceHandle,
+                listing.Definition.HostPresenceFingerprint,
+                EphemeralStateFixture.PublicEndpoint(42_200),
+                null));
+            Assert.True(presence.Succeeded, $"Presence {iteration} failed with {presence.Code}.");
+            StoreResult<int> revoked = fixture.Store.RevokePrincipal(
+                "repeated-revocation",
+                TimeSpan.FromMinutes(1));
+            Assert.True(revoked.Succeeded, $"Revocation {iteration} failed with {revoked.Code}.");
+        }
+
+        Assert.Equal(4, fixture.Store.ScheduledExpiryEntryCount);
+        fixture.Clock.Advance(TimeSpan.FromSeconds(19));
+        Assert.True(fixture.Store.GetListing(listing.Definition.ListingId, true).Succeeded);
+        Assert.Equal(4, fixture.Store.ScheduledExpiryEntryCount);
+    }
+
+    [Fact]
+    public void RepeatedPrincipalRevocationCanExtendButCannotShortenProtection()
+    {
+        EphemeralStateFixture fixture = new();
+        const string subject = "protected-publisher";
+
+        Assert.True(fixture.Store.RevokePrincipal(subject, TimeSpan.FromMinutes(10)).Succeeded);
+        fixture.Clock.Advance(TimeSpan.FromSeconds(1));
+        Assert.True(fixture.Store.RevokePrincipal(subject, TimeSpan.FromSeconds(1)).Succeeded);
+        fixture.Clock.Advance(TimeSpan.FromSeconds(2));
+        Assert.Equal(
+            StoreResultCode.Revoked,
+            fixture.Store.CreateListing(fixture.ListingCommand(owner: subject)).Code);
+
+        fixture.Clock.Advance(TimeSpan.FromSeconds(597));
+        Assert.True(fixture.Store.CreateListing(fixture.ListingCommand(owner: subject)).Succeeded);
     }
 
     [Fact]