feat(operations): add capacity and resilience gates (#18)

.gitea/workflows/ci.yml

@@ -36,6 +36,9 @@ jobs:
       - name: Test
         run: dotnet test Rendezvous.slnx --configuration Release --no-build
 
+      - name: Run quick capacity and resilience gate
+        run: ./scripts/run-capacity-gate.sh
+
       - name: Test privileged Linux namespace topology when available
         shell: bash
         run: |

.gitignore

@@ -7,5 +7,6 @@ TestResults/
 *.user
 *.userosscache
 deploy/compose/.smoke.env
+artifacts/
 deploy/compose/secrets/*
 !deploy/compose/secrets/.gitignore

README.md

@@ -83,9 +83,9 @@ Rendezvous is under active roadmap development. The versioned contracts,
 directory leases, authenticated join attempts, LiteNetLib mediator, caller-owned
 SDK coordination, typed connection outcomes, thin public-SDK diagnostic client,
 deterministic NAT topology harness, hostile-input controls,
-observability/operator surface, and secure single-active Linux deployment are
+observability/operator surface, secure single-active Linux deployment, and
-implemented. Capacity, resilience, packaging, and final production-readiness
+numeric capacity/resilience gates are implemented. Packaging, consumer pilots,
-gates remain in progress;
+and final production-readiness gates remain in progress;
 participating games must not treat the current repository as a finished production
 service until those gates land.
 
@@ -103,6 +103,9 @@ operator controls are defined in the
 The pinned non-root container, production topology, graceful drain, Linux
 hardening, smoke procedure, and recovery lifecycle are documented in
 [secure single-active Linux deployment](docs/deployment/linux.md).
+The numeric core-state candidate profile, public launch objectives, accelerated
+soak, resilience matrix, and single-active scaling decision are recorded in
+[capacity and resilience gates](docs/operations/capacity-and-resilience.md).
 The scriptable host/browser/join diagnostic and its stable automation contract are
 documented in the [TestClient integration guide](docs/integration/test-client.md).
 The always-on three-party scenarios, optional Linux namespace topology, and

Rendezvous.slnx

@@ -6,6 +6,7 @@
     <Project Path="src/FinalFactory.Rendezvous.TestClient/FinalFactory.Rendezvous.TestClient.csproj" />
   </Folder>
   <Folder Name="/tests/">
+    <Project Path="tests/FinalFactory.Rendezvous.Capacity/FinalFactory.Rendezvous.Capacity.csproj" />
     <Project Path="tests/FinalFactory.Rendezvous.Tests/FinalFactory.Rendezvous.Tests.csproj" />
   </Folder>
 </Solution>

deploy/systemd/rendezvous.service

@@ -40,7 +40,8 @@ LockPersonality=true
 SystemCallArchitectures=native
 UMask=0077
 LimitNOFILE=4096
-MemoryMax=512M
+CPUQuota=200%
+MemoryMax=2G
 TasksMax=128
 
 [Install]

docs/architecture/0003-state-privacy-availability-and-budgets.md

@@ -129,9 +129,18 @@ until the owner records:
 - which games may enable anonymous unlisted player hosting;
 - deployment regions, data-processing jurisdiction, and approval of the stated
   30-day audit/13-month aggregate retention periods;
-- the per-game dedicated fallback endpoint policy;
+- the per-game dedicated fallback endpoint policy.
-- the measured supported profile and whether the 99.5% single-active objective
+
-  is sufficient or shared-state/high-availability work must be brought forward.
+Issue #18 measured and ratified the original 2-vCPU/2-GiB, 25,000-listing,
+10,000-attempt core-state candidate profile and retained the 99.5% single-active
+topology. It does not claim that core measurements prove public HTTP/UDP SLOs.
+The versioned evidence, RTO, failure domains, and explicit signals that trigger
+shared-state/high-availability work are recorded in the
+[capacity and resilience gate](../operations/capacity-and-resilience.md). The
+real-network canary in #23 must confirm that the proposed regional launch load
+fits this profile and validate the public SLOs; it may lower the launch cap but
+may not silently enable a
+second active instance.
 
 These are configuration and launch decisions, not permission to weaken the
 tenant, replay, endpoint-verification, or secret-handling controls.

docs/deployment/linux.md

@@ -118,8 +118,11 @@ Keep the host clock synchronized with authenticated NTP. Credential and key
 windows use wall time; lease, timeout, drain, and rate-limit deadlines use a
 monotonic clock. Alert on clock synchronization loss before rotating keys.
 
-Start with the Compose limits (one CPU, 512 MiB, 128 PIDs, 4096 descriptors),
+The checked-in Compose limits (one CPU and 512 MiB) are for its isolated smoke
-measure real traffic, then change the limits and the server budgets together.
+profile, not a production capacity claim. The measured core-state candidate
+uses 2 vCPU and 2 GiB with the same 128-PID/4096-descriptor ceilings; see
+the [capacity and resilience gate](../operations/capacity-and-resilience.md).
+Measure real traffic, then change resource limits and server budgets together.
 Memory pressure or CPU throttling must not extend orchestrator termination past
 `DrainDeadlineSeconds` plus five seconds.
 
@@ -157,8 +160,8 @@ sudo systemctl enable --now rendezvous.service
 Create the dedicated `rendezvous` user without a login shell. Keep
 `/opt/rendezvous` and `/etc/rendezvous` root-owned and non-writable by that user;
 install each required key with `root:rendezvous` ownership and mode `0440`. The
-unit applies the same resource, filesystem, privilege, network-family, and
+unit applies the measured 2-vCPU/2-GiB core-state candidate profile plus the
-shutdown constraints as Compose.
+same filesystem, privilege, network-family, and shutdown hardening as Compose.
 
 ## HTTP and UDP smoke
 

docs/operations/capacity-and-resilience.md

@@ -0,0 +1,189 @@
+# Capacity, resilience, and availability gate
+
+Tracking: #18
+
+This gate turns the v1 budgets in ADR 0003 into a repeatable release decision.
+It does not turn Rendezvous into a horizontally scalable service: v1 remains one
+active process with bounded in-memory state. A second process may be a cold
+standby, but it must not accept traffic until the first process has stopped and
+released the public HTTP and UDP endpoints.
+
+## Launch envelope and approved core-state profile
+
+The approved core-state profile is one Linux process limited to 2 vCPU and
+2 GiB RAM. Public HTTP/UDP numbers are launch objectives that require the #23
+real-network canary before they become a supported service claim:
+
+| Dimension | Value | Evidence status |
+| --- | --- | --- |
+| Visible listings | 25,000 | Enforced and measured here |
+| Active join attempts | 10,000 | Enforced and measured here |
+| Core control path | 200 operations/second; p95 at most 200 ms | Measured here |
+| Core mediation path | 2,000 pairings/second; p95 at most 100 ms | Measured here |
+| Sustained HTTP demand | 200 requests/second | #23 launch objective; not yet a supported claim |
+| Sustained UDP demand | 2,000 datagrams/second | #23 launch objective; not yet a supported claim |
+| Public HTTP/UDP latency | p95 at most 200 ms / 100 ms | #23 launch objective; not yet a supported claim |
+| Capacity-phase average CPU / peak working memory | below 70% / below 1.5 GiB | Measured for the core candidate |
+| Valid in-profile monthly availability | 99.5%, excluding announced maintenance | Operational objective |
+| Process-ready RTO / host-visible recovery | 15 seconds / 90 seconds | 15 seconds automated; 90-second deployment drill required |
+
+The proposed public-network mix is 20% registration/update, 30% lease-critical
+renew/delete, 30% browse, and 20% join authorization for HTTP. The UDP mix is
+60% authenticated host-presence refresh, 30% attempt contributions, and 10%
+invalid or duplicate traffic that must be dropped early. A deployment may use a
+lower per-game profile, but must not claim a higher one without new versioned
+evidence.
+
+The capacity harness fills the complete state ceilings, then measures
+registration plus presence, renewal, a 100-item compatible browse, join
+issuance, simultaneous two-peer pairing, principal revocation, and telemetry.
+It applies 200/100 ms guardrails and minimum 200 control / 2,000 mediation
+operations per second to the core hot path. Those measurements deliberately
+exclude Kestrel, LiteNetLib, TLS, JSON, socket scheduling, and the documented
+mixed traffic shape. The #23 real-network canary must exercise those layers,
+rate-shape the mix, record errors and shedding, and meet the public objectives
+before launch; a core result is not a public-network latency or throughput claim.
+
+## Reproduce the evidence
+
+Every push runs the quick profile and the selected fault matrix:
+
+```bash
+./scripts/run-capacity-gate.sh
+```
+
+Run the production candidate on an otherwise idle Linux host and restrict the
+runtime to two logical CPUs. The default candidate includes a five-minute,
+high-intensity expiry soak; use 3,600 seconds for a release-candidate endurance
+run:
+
+```bash
+export RENDEZVOUS_CAPACITY_PROFILE=candidate
+export RENDEZVOUS_CAPACITY_CPUSET=0,1
+export RENDEZVOUS_CAPACITY_OUTPUT="$PWD/artifacts/capacity/candidate.json"
+./scripts/run-capacity-gate.sh
+
+# Release-candidate endurance override:
+dotnet run --project tests/FinalFactory.Rendezvous.Capacity \
+  --configuration Release --no-build -- \
+  --profile candidate --soak-seconds 3600 \
+  --output artifacts/capacity/candidate-endurance.json
+```
+
+The machine must have at least 2 GiB available to the process. For formal
+deployment evidence, run inside the same cgroup/container shape as production.
+The v2 JSON embeds the commit and tree state, command, image context, CPU model,
+kernel, affinity, cgroup quota/limit, collector mode, and workload seed. Supply
+`RENDEZVOUS_EVIDENCE_IMAGE_DIGEST` when running a release image. Do not compare
+results collected under a debugger,
+concurrent build, thermal throttling, or oversubscribed CI host.
+
+The checked-in baseline is
+[`candidate-2cpu.json`](../evidence/capacity/v2/candidate-2cpu.json). It was
+produced on .NET 10.0.9/Linux x64 with CPU affinity restricted to two logical
+CPUs. It filled 25,000 listings and 10,000 attempts, peaked at about 162 MiB,
+and cleared all active/retained state. The five-minute baseline supersedes any
+earlier local probe when its timestamp and target duration differ.
+
+## Soak and bounded-state interpretation
+
+Each soak cycle creates a listing, repeatedly renews its lease and refreshes
+presence, creates a join attempt, replay marker, and retained outcome, checks
+that scheduled expiry entries remain proportional to live keys, then advances
+the injected monotonic clock beyond all
+deadlines, and verifies that listings, attempts, replay, idempotency, and outcome
+state return to zero. The candidate also measures managed-memory and process
+handle deltas after full collection. Failure is any retained state, more than
+64 MiB retained managed memory, more than eight retained handles, a working set
+above 1.5 GiB, an untyped capacity result, or failure to admit work after expiry.
+
+This accelerated soak intentionally executes far more state lifecycle/cleanup
+events than wall-clock traffic would permit. It catches stale deadline-queue
+entries, cache growth, replay/idempotency retention, and cleanup cost. Because
+it does not open Kestrel/LiteNetLib connections, its process-handle delta is only
+a harness guard and is not evidence of transport stability by itself. The
+selected production-process gate adds a ten-second real HTTP/UDP transport soak,
+samples child-process handles and RSS, asserts bounded growth, then verifies a
+clean SIGTERM and socket release. #23 must extend that into the full rate-shaped
+multi-client canary while sampling queues, managed memory, and state
+cardinalities. A one-hour core override remains required before tagging a
+production release.
+
+## Fault and recovery matrix
+
+`run-capacity-gate.sh` runs these deterministic production paths before the
+numeric profile:
+
+| Fault | Required result |
+| --- | --- |
+| HTTP/UDP overload and tracker exhaustion | Typed HTTP `429`/`CapacityExceeded`, silent UDP drop, bounded tracker keys, recovery after the window |
+| Optional traffic saturation | Lease-critical renew/update/delete capacity remains available |
+| Store/dependency unavailable | Readiness fails; new authorization returns typed `ServiceUnavailable`; liveness remains independent |
+| Graceful drain/SIGTERM | New work returns `Draining`; existing pairing may finish; process exits 0 and releases TCP/UDP before the deadline |
+| Hard restart | In-flight state is lost; SDK reports typed `ServiceUnavailable`; a host re-registers, rebinds presence, and becomes the only browser-visible replacement |
+| UDP listener bind/restart | Readiness stays false without the required listener; rebinding the advertised port restores native LiteNetLib pairing |
+| Wall-clock jump/skew | Monotonic lease/attempt authority is neither shortened nor extended; credential skew remains capped at 30 seconds |
+| Signing-secret rotation | New key signs, overlap verifies, retired/revoked key rejects, missing material fails startup |
+| Principal revocation | Listing, presence, attempts, and outcome paths are removed atomically within the latency budget |
+
+No external database exists in v1, so “dependency/store failure” means the
+process-local atomic store is marked unavailable or a required listener/key is
+unready. The service fails closed rather than pretending a degraded writable
+mode exists.
+
+## Bandwidth and amplification
+
+- Accepted application datagrams are at most 1,200 bytes.
+- Malformed, oversized, unauthenticated, stale, replayed, wrong-role, and
+  rate-limited traffic receives zero response bytes.
+- A completing authenticated contribution produces at most one introduction to
+  each observed peer, and the combined response is at most 2.0 times that
+  contribution's bytes.
+- The frozen-envelope and native LiteNetLib socket tests measure this on the real
+  UDP listener; the hostile corpus and allocation gate exercise 10,000+ inputs
+  without input-sized logs, tasks, or queues.
+
+Bandwidth planning must therefore reserve ingress for the configured 2,000
+datagrams/second plus edge overhead and egress for a worst-case verified 2.0
+amplification. Actual successful pairs normally use two contributions and two
+introductions; normal gameplay leaves Rendezvous entirely.
+
+## Availability decision
+
+Single-active remains the v1 topology. The measured core profile proves bounded
+state and substantial core-path headroom, while public launch capacity remains
+conditional on #23. The service has a bounded stop-before-start restart path.
+Its failure domain is deliberately
+one process/node/public UDP endpoint: node, kernel, host network, DNS/TLS edge,
+secret configuration, or operator error can remove all readiness until the cold
+replacement owns the same source-preserving endpoint.
+
+The 99.5% objective permits about 216 minutes of unannounced downtime in a
+30-day month. Operations must target process readiness within 15 seconds and
+host-visible re-registration within 90 seconds, page when no ready instance
+exists, and include detection plus recovery in the monthly budget. The current
+in-process test validates typed downtime, same-port HTTP restart, fresh
+registration, presence rebinding, and browser visibility in under five seconds;
+the production-process test separately validates graceful termination, TCP/UDP
+release, replacement startup on the same endpoints, UDP readiness, and the
+15-second process-ready RTO. Cold-standby activation policy and the 90-second
+operator-to-host recovery objective still require a deployment drill before
+release. Rollout and rollback use the
+deployment runbook's drain, stop, socket-release, start, smoke sequence; never
+overlap old and new active processes.
+
+Bring shared TTL/CAS state and deterministic mediator routing forward before
+enabling two active instances if any of these occurs:
+
+- one node cannot sustain 150% of the measured 30-day peak while meeting SLOs;
+- CPU stays above 70%, memory above 75%, attempt depth above 70%, or limiter
+  drops/latency remain elevated after abusive traffic is excluded;
+- the availability target rises above 99.5% or planned maintenance must preserve
+  listings; or
+- one region requires multiple simultaneously active mediator endpoints.
+
+Rendezvous makes no multi-instance claim today, so a two-node atomic-pairing
+test is intentionally not applicable. It becomes a hard release gate with the
+shared-state/routing implementation; until then `SingleActiveInstance=false`
+fails production startup. Multi-region and relay remain separate evidence-driven
+decisions.

docs/operations/observability-and-operator-runbook.md

@@ -115,7 +115,9 @@ permission denial is audited with actor and target fingerprints.
 Key revocation is process-local in the current single-instance store. Apply the
 same revocation to every instance, then replace configuration before restarting;
 a restart reconstructs the configured key ring. Principal revocation is bounded
-to ten minutes and removes that principal's active listings and attempts. Use
+to ten minutes and removes that principal's active listings and attempts. A
+repeat action may extend an active revocation but never shortens it; wait for its
+original deadline rather than treating a shorter repeat as an un-revoke. Use
 listing revocation for one targeted session and drain before planned shutdown.
 
 ## Audit retention and incident handling

scripts/run-capacity-gate.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+PROFILE="${RENDEZVOUS_CAPACITY_PROFILE:-quick}"
+OUTPUT="${RENDEZVOUS_CAPACITY_OUTPUT:-$ROOT/artifacts/capacity/rendezvous-capacity-v2.json}"
+CPUSET="${RENDEZVOUS_CAPACITY_CPUSET:-}"
+PROJECT="$ROOT/tests/FinalFactory.Rendezvous.Capacity/FinalFactory.Rendezvous.Capacity.csproj"
+TESTS="$ROOT/tests/FinalFactory.Rendezvous.Tests/FinalFactory.Rendezvous.Tests.csproj"
+
+if [[ "$PROFILE" != quick && "$PROFILE" != candidate ]]; then
+    printf 'RENDEZVOUS_CAPACITY_PROFILE must be quick or candidate.\n' >&2
+    exit 2
+fi
+command -v dotnet >/dev/null || {
+    printf 'Missing required command: dotnet\n' >&2
+    exit 2
+}
+if [[ -n "$CPUSET" ]]; then
+    command -v taskset >/dev/null || {
+        printf 'taskset is required when RENDEZVOUS_CAPACITY_CPUSET is set.\n' >&2
+        exit 2
+    }
+fi
+
+cd "$ROOT"
+dotnet restore "$ROOT/Rendezvous.slnx" --locked-mode
+dotnet build "$ROOT/Rendezvous.slnx" --configuration Release --no-restore
+
+filter='FullyQualifiedName~TrackerCapacityFailsClosedWithoutGrowingAndAWindowResetRecovers|FullyQualifiedName~OptionalTrafficCannotConsumeTheLeaseOperationReserve|FullyQualifiedName~ConcurrentAbusiveBurstStaysBoundedAndCannotBlockCriticalHttp|FullyQualifiedName~HttpOverloadIsTypedAndOversizedBodiesAreRejectedBeforeDispatch|FullyQualifiedName~WallClockMovementDoesNotExpireOrExtendLease|FullyQualifiedName~RepeatedMutableDeadlineRefreshesKeepOneScheduledEntryPerKey|FullyQualifiedName~RepeatedPrincipalRevocationCanExtendButCannotShortenProtection|FullyQualifiedName~RestartHasNewGenerationAndNoEphemeralState|FullyQualifiedName~RestartReturnsTypedUnavailabilityThenAllowsHostReregistration|FullyQualifiedName~DrainRejectsNewWorkAllowsInflightCompletionThenClearsState|FullyQualifiedName~UnavailableStoreFailsNewAuthorizationClosedAndErasesActiveState|FullyQualifiedName~KeyRotationHonorsOverlapAndRejectsRetiredKeys|FullyQualifiedName~OperatorSurfaceSeparatesAuthenticationConfirmsActionsAndRedactsInspection|FullyQualifiedName~SigtermDrainsThenReleasesHttpAndUdpSockets|FullyQualifiedName~ProductionTransportSoakKeepsHandlesMemoryAndSocketsBounded|FullyQualifiedName~NativeLiteNetLibRequestsIntroduceTheAuthorizedPair'
+dotnet test "$TESTS" --configuration Release --no-build --filter "$filter" \
+    --logger 'console;verbosity=minimal'
+
+mkdir -p "$(dirname "$OUTPUT")"
+arguments=(
+    dotnet run --project "$PROJECT" --configuration Release --no-build --
+    --profile "$PROFILE" --output "$OUTPUT"
+)
+export RENDEZVOUS_EVIDENCE_COMMIT="${RENDEZVOUS_EVIDENCE_COMMIT:-$(git rev-parse HEAD)}"
+if [[ -z "${RENDEZVOUS_EVIDENCE_TREE_STATE:-}" ]]; then
+    if [[ -n "$(git status --porcelain)" ]]; then
+        export RENDEZVOUS_EVIDENCE_TREE_STATE=dirty
+    else
+        export RENDEZVOUS_EVIDENCE_TREE_STATE=clean
+    fi
+fi
+export RENDEZVOUS_EVIDENCE_CPUSET="${CPUSET:-unrestricted}"
+export RENDEZVOUS_EVIDENCE_COMMAND="RENDEZVOUS_CAPACITY_PROFILE=$PROFILE RENDEZVOUS_CAPACITY_CPUSET=${CPUSET:-unrestricted} ./scripts/run-capacity-gate.sh"
+if [[ -n "$CPUSET" ]]; then
+    taskset -c "$CPUSET" "${arguments[@]}"
+else
+    "${arguments[@]}"
+fi
+
+printf 'Capacity and resilience gate passed; evidence: %s\n' "$OUTPUT"

src/FinalFactory.Rendezvous.Server/Properties/AssemblyInfo.cs

@@ -1,3 +1,4 @@
 using System.Runtime.CompilerServices;
 
 [assembly: InternalsVisibleTo("FinalFactory.Rendezvous.Tests")]
+[assembly: InternalsVisibleTo("FinalFactory.Rendezvous.Capacity")]

src/FinalFactory.Rendezvous.Server/State/InMemoryEphemeralRendezvousStore.cs

@@ -11,16 +11,29 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
     private readonly DateTimeOffset _wallOrigin;
     private readonly TimeSpan _monotonicOrigin;
     private readonly Dictionary<SessionListingId, ListingEntry> _listings = [];
+    private readonly Dictionary<string, int> _listingCountsByOwner = new(StringComparer.Ordinal);
+    private readonly PriorityQueue<DeadlineEntry<SessionListingId>, long> _listingExpiries = new();
+    private readonly HashSet<SessionListingId> _scheduledListingExpiries = [];
     private readonly Dictionary<LeaseId, SessionListingId> _leases = [];
     private readonly Dictionary<MediationHandle, SessionListingId> _presenceHandles = [];
     private readonly Dictionary<MediationHandle, PresenceEntry> _presence = [];
+    private readonly PriorityQueue<DeadlineEntry<MediationHandle>, long> _presenceExpiries = new();
+    private readonly HashSet<MediationHandle> _scheduledPresenceExpiries = [];
     private readonly Dictionary<JoinAttemptId, AttemptEntry> _attempts = [];
-    private readonly PriorityQueue<AttemptExpiry, long> _attemptExpiries = new();
+    private readonly Dictionary<TenantScope, int> _attemptCountsByScope = [];
+    private readonly Dictionary<SessionListingId, HashSet<JoinAttemptId>> _attemptsByListing = [];
+    private readonly PriorityQueue<DeadlineEntry<JoinAttemptId>, long> _attemptExpiries = new();
     private readonly Dictionary<JoinAttemptId, OutcomeReportEntry> _outcomeReports = [];
+    private readonly Dictionary<SessionListingId, HashSet<JoinAttemptId>> _outcomesByListing = [];
+    private readonly PriorityQueue<DeadlineEntry<JoinAttemptId>, long> _outcomeExpiries = new();
     private readonly Dictionary<MediationHandle, JoinAttemptId> _attemptHandles = [];
     private readonly Dictionary<string, IdempotencyEntry> _idempotency = new(StringComparer.Ordinal);
+    private readonly PriorityQueue<DeadlineEntry<string>, long> _idempotencyExpiries = new();
     private readonly Dictionary<string, TimeSpan> _replay = new(StringComparer.Ordinal);
+    private readonly PriorityQueue<DeadlineEntry<string>, long> _replayExpiries = new();
     private readonly Dictionary<string, TimeSpan> _revocations = new(StringComparer.Ordinal);
+    private readonly PriorityQueue<DeadlineEntry<string>, long> _revocationExpiries = new();
+    private readonly HashSet<string> _scheduledRevocationExpiries = new(StringComparer.Ordinal);
     private TimeSpan? _drainDeadline;
     private TimeSpan _nextUdpMaintenance;
     private long _maintenanceSweepCount;
@@ -47,6 +60,22 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
 
     public Guid InstanceId { get; }
     internal long MaintenanceSweepCount => Interlocked.Read(ref _maintenanceSweepCount);
+    internal int ScheduledExpiryEntryCount
+    {
+        get
+        {
+            lock (_gate)
+            {
+                return _listingExpiries.Count
+                    + _presenceExpiries.Count
+                    + _attemptExpiries.Count
+                    + _outcomeExpiries.Count
+                    + _idempotencyExpiries.Count
+                    + _replayExpiries.Count
+                    + _revocationExpiries.Count;
+            }
+        }
+    }
 
     public bool IsAvailable
     {
@@ -161,10 +190,8 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
 
         if (_listings.Count >= _options.MaxListings
             || _idempotency.Count >= _options.MaxIdempotencyEntries
-            || _listings.Values.Count(entry => string.Equals(
+            || _listingCountsByOwner.GetValueOrDefault(command.Listing.OwnerSubject)
-                entry.Definition.OwnerSubject,
+                >= command.OwnerListingLimit)
-                command.Listing.OwnerSubject,
-                StringComparison.Ordinal)) >= command.OwnerListingLimit)
         {
             return new(StoreResultCode.CapacityExceeded);
         }
@@ -183,12 +210,21 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
             WallDeadline(now, _options.LeaseLifetime),
             version: 1);
         _listings.Add(frozen.ListingId, entry);
+        ScheduleMutableDeadline(
+            _listingExpiries,
+            _scheduledListingExpiries,
+            frozen.ListingId,
+            entry.LeaseDeadline);
+        _listingCountsByOwner[frozen.OwnerSubject] =
+            _listingCountsByOwner.GetValueOrDefault(frozen.OwnerSubject) + 1;
         _leases.Add(frozen.LeaseId, frozen.ListingId);
         _presenceHandles.Add(frozen.HostPresenceHandle, frozen.ListingId);
-        _idempotency.Add(idempotencyKey, new(
+        IdempotencyEntry idempotency = new(
             command.RequestFingerprint,
             frozen.ListingId,
-            now + _options.IdempotencyLifetime));
+            now + _options.IdempotencyLifetime);
+        _idempotency.Add(idempotencyKey, idempotency);
+        EnqueueDeadline(_idempotencyExpiries, idempotencyKey, idempotency.Deadline);
         return new(StoreResultCode.Success, Snapshot(entry));
     }, cancellationToken);
 
@@ -346,10 +382,20 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
             return new(StoreResultCode.CapacityExceeded);
         }
 
-        _presence[command.Handle] = new(
+        bool isNewPresence = !_presence.ContainsKey(command.Handle);
+        PresenceEntry presence = new(
             command.PublicEndpoint,
             command.LocalEndpoint,
             now + _options.PresenceLifetime);
+        _presence[command.Handle] = presence;
+        if (isNewPresence)
+        {
+            ScheduleMutableDeadline(
+                _presenceExpiries,
+                _scheduledPresenceExpiries,
+                command.Handle,
+                presence.Deadline);
+        }
         return new(StoreResultCode.Success, Snapshot(entry));
     }, cancellationToken, eagerCleanup: false);
 
@@ -444,8 +490,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
         if (_attempts.Count >= _options.MaxJoinAttempts
             || _outcomeReports.Count >= _options.MaxOutcomeReports
             || _idempotency.Count >= _options.MaxIdempotencyEntries
-            || _attempts.Values.Count(entry => entry.Command.Scope == command.Scope)
+            || _attemptCountsByScope.GetValueOrDefault(command.Scope) >= command.ScopeAttemptLimit)
-                >= command.ScopeAttemptLimit)
         {
             return new(StoreResultCode.CapacityExceeded);
         }
@@ -461,19 +506,25 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
             now + _options.JoinAttemptLifetime,
             WallDeadline(now, _options.JoinAttemptLifetime));
         _attempts.Add(command.AttemptId, attempt);
-        _attemptExpiries.Enqueue(
+        AddToIndex(_attemptsByListing, command.ListingId, command.AttemptId);
-            new AttemptExpiry(command.AttemptId, attempt.Deadline),
+        _attemptCountsByScope[command.Scope] =
-            attempt.Deadline.Ticks);
+            _attemptCountsByScope.GetValueOrDefault(command.Scope) + 1;
-        _outcomeReports.Add(command.AttemptId, new(
+        EnqueueDeadline(_attemptExpiries, command.AttemptId, attempt.Deadline);
+        OutcomeReportEntry outcome = new(
             command.ListingId,
             command.ClientSubject,
             command.ClientCapabilityFingerprint,
-            now + _options.JoinAttemptLifetime + _options.IdempotencyLifetime));
+            now + _options.JoinAttemptLifetime + _options.IdempotencyLifetime);
+        _outcomeReports.Add(command.AttemptId, outcome);
+        AddToIndex(_outcomesByListing, command.ListingId, command.AttemptId);
+        EnqueueDeadline(_outcomeExpiries, command.AttemptId, outcome.Deadline);
         _attemptHandles.Add(command.MediationHandle, command.AttemptId);
-        _idempotency.Add(idempotencyKey, new(
+        IdempotencyEntry idempotency = new(
             command.RequestFingerprint,
             command.AttemptId,
-            now + _options.IdempotencyLifetime));
+            now + _options.IdempotencyLifetime);
+        _idempotency.Add(idempotencyKey, idempotency);
+        EnqueueDeadline(_idempotencyExpiries, idempotencyKey, idempotency.Deadline);
         return new(StoreResultCode.Success, Snapshot(attempt));
     }, cancellationToken);
 
@@ -757,7 +808,9 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
             throw new ArgumentOutOfRangeException(nameof(consumption), "Replay lifetime exceeds the configured ceiling.");
         }
 
-        _replay.Add(key, now + lifetime);
+        TimeSpan deadline = now + lifetime;
+        _replay.Add(key, deadline);
+        EnqueueDeadline(_replayExpiries, key, deadline);
         return new(StoreResultCode.Success, true);
     }, cancellationToken);
 
@@ -794,7 +847,24 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
             + _presence.Count
             + _attempts.Count
             + _outcomeReports.Count;
-        _revocations[subject] = now + lifetime;
+        TimeSpan deadline = now + lifetime;
+        bool isNewRevocation = !_revocations.TryGetValue(subject, out TimeSpan existingDeadline);
+        if (!isNewRevocation && existingDeadline > deadline)
+        {
+            // A repeated operator action may extend protection but cannot silently
+            // shorten an already-authoritative security revocation.
+            deadline = existingDeadline;
+        }
+
+        _revocations[subject] = deadline;
+        if (isNewRevocation)
+        {
+            ScheduleMutableDeadline(
+                _revocationExpiries,
+                _scheduledRevocationExpiries,
+                subject,
+                deadline);
+        }
         SessionListingId[] listings = _listings
             .Where(item => string.Equals(item.Value.Definition.OwnerSubject, subject, StringComparison.Ordinal))
             .Select(static item => item.Key)
@@ -818,7 +888,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
         }
         foreach (JoinAttemptId attemptId in outcomeReports)
         {
-            _outcomeReports.Remove(attemptId);
+            RemoveOutcome(attemptId);
         }
 
         int activeResourcesAfter = _listings.Count
@@ -911,63 +981,42 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
             ClearActiveState();
         }
 
-        _expiryChurn += RemoveExpired(_revocations, now);
+        _expiryChurn += RemoveExpiredMutableDeadlines(
-        _expiryChurn += RemoveExpired(_replay, now);
+            _revocations,
-        string[] expiredIdempotency = _idempotency
+            _revocationExpiries,
-            .Where(item => item.Value.Deadline <= now)
+            _scheduledRevocationExpiries,
-            .Select(static item => item.Key)
+            now);
-            .ToArray();
+        _expiryChurn += RemoveExpiredDeadlines(_replay, _replayExpiries, now);
-        _expiryChurn += expiredIdempotency.Length;
+        _expiryChurn += RemoveExpiredIdempotency(now);
-        foreach (string key in expiredIdempotency)
+        _expiryChurn += RemoveExpiredPresence(now);
-        {
-            _idempotency.Remove(key);
-        }
-
-        MediationHandle[] expiredPresence = _presence
-            .Where(item => item.Value.Deadline <= now)
-            .Select(static item => item.Key)
-            .ToArray();
-        _expiryChurn += expiredPresence.Length;
-        foreach (MediationHandle handle in expiredPresence)
-        {
-            _presence.Remove(handle);
-        }
-
         _expiryChurn += RemoveExpiredAttempts(now);
-
+        _expiryChurn += RemoveExpiredOutcomes(now);
-        JoinAttemptId[] expiredOutcomes = _outcomeReports
+        _expiryChurn += RemoveExpiredListings(now);
-            .Where(item => item.Value.Deadline <= now)
-            .Select(static item => item.Key)
-            .ToArray();
-        _expiryChurn += expiredOutcomes.Length;
-        foreach (JoinAttemptId attemptId in expiredOutcomes)
-        {
-            _outcomeReports.Remove(attemptId);
-        }
-
-        SessionListingId[] expiredListings = _listings
-            .Where(item => item.Value.LeaseDeadline <= now)
-            .Select(static item => item.Key)
-            .ToArray();
-        _expiryChurn += expiredListings.Length;
-        foreach (SessionListingId listingId in expiredListings)
-        {
-            RemoveListing(listingId);
-        }
     }
 
     private void ClearActiveState()
     {
         _listings.Clear();
+        _listingCountsByOwner.Clear();
+        _listingExpiries.Clear();
+        _scheduledListingExpiries.Clear();
         _leases.Clear();
         _presenceHandles.Clear();
         _presence.Clear();
+        _presenceExpiries.Clear();
+        _scheduledPresenceExpiries.Clear();
         _attempts.Clear();
+        _attemptCountsByScope.Clear();
+        _attemptsByListing.Clear();
         _attemptExpiries.Clear();
         _outcomeReports.Clear();
+        _outcomesByListing.Clear();
+        _outcomeExpiries.Clear();
         _attemptHandles.Clear();
         _idempotency.Clear();
+        _idempotencyExpiries.Clear();
         _replay.Clear();
+        _replayExpiries.Clear();
     }
 
     private void RemoveListing(SessionListingId listingId)
@@ -978,23 +1027,23 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
         }
 
         _leases.Remove(listing.Definition.LeaseId);
+        DecrementCount(_listingCountsByOwner, listing.Definition.OwnerSubject);
         _presenceHandles.Remove(listing.Definition.HostPresenceHandle);
         _presence.Remove(listing.Definition.HostPresenceHandle);
-        foreach (JoinAttemptId attemptId in _attempts
+        if (_attemptsByListing.TryGetValue(listingId, out HashSet<JoinAttemptId>? attempts))
-                     .Where(item => item.Value.Command.ListingId == listingId)
-                     .Select(static item => item.Key)
-                     .ToArray())
         {
-            RemoveAttempt(attemptId);
+            foreach (JoinAttemptId attemptId in attempts.ToArray())
+            {
+                RemoveAttempt(attemptId);
+            }
         }
 
-
+        if (_outcomesByListing.TryGetValue(listingId, out HashSet<JoinAttemptId>? outcomes))
-        foreach (JoinAttemptId attemptId in _outcomeReports
-                     .Where(item => item.Value.ListingId == listingId)
-                     .Select(static item => item.Key)
-                     .ToArray())
         {
-            _outcomeReports.Remove(attemptId);
+            foreach (JoinAttemptId attemptId in outcomes.ToArray())
+            {
+                RemoveOutcome(attemptId);
+            }
         }
     }
 
@@ -1003,20 +1052,75 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
         if (_attempts.Remove(attemptId, out AttemptEntry? attempt))
         {
             _attemptHandles.Remove(attempt.Command.MediationHandle);
+            DecrementCount(_attemptCountsByScope, attempt.Command.Scope);
+            RemoveFromIndex(_attemptsByListing, attempt.Command.ListingId, attemptId);
+        }
+    }
+
+    private void RemoveOutcome(JoinAttemptId attemptId)
+    {
+        if (_outcomeReports.Remove(attemptId, out OutcomeReportEntry? outcome))
+        {
+            RemoveFromIndex(_outcomesByListing, outcome.ListingId, attemptId);
+        }
+    }
+
+    private static void AddToIndex<TKey>(
+        Dictionary<TKey, HashSet<JoinAttemptId>> index,
+        TKey key,
+        JoinAttemptId attemptId)
+        where TKey : notnull
+    {
+        if (!index.TryGetValue(key, out HashSet<JoinAttemptId>? values))
+        {
+            values = [];
+            index.Add(key, values);
+        }
+
+        values.Add(attemptId);
+    }
+
+    private static void RemoveFromIndex<TKey>(
+        Dictionary<TKey, HashSet<JoinAttemptId>> index,
+        TKey key,
+        JoinAttemptId attemptId)
+        where TKey : notnull
+    {
+        if (index.TryGetValue(key, out HashSet<JoinAttemptId>? values)
+            && values.Remove(attemptId)
+            && values.Count == 0)
+        {
+            index.Remove(key);
+        }
+    }
+
+    private static void DecrementCount<TKey>(Dictionary<TKey, int> counts, TKey key)
+        where TKey : notnull
+    {
+        int remaining = counts[key] - 1;
+        if (remaining == 0)
+        {
+            counts.Remove(key);
+        }
+        else
+        {
+            counts[key] = remaining;
         }
     }
 
     private int RemoveExpiredAttempts(TimeSpan now)
     {
         int removed = 0;
-        while (_attemptExpiries.TryPeek(out AttemptExpiry candidate, out long deadlineTicks)
+        while (_attemptExpiries.TryPeek(
+                   out DeadlineEntry<JoinAttemptId> candidate,
+                   out long deadlineTicks)
                && deadlineTicks <= now.Ticks)
         {
             _attemptExpiries.Dequeue();
-            if (_attempts.TryGetValue(candidate.AttemptId, out AttemptEntry? current)
+            if (_attempts.TryGetValue(candidate.Key, out AttemptEntry? current)
                 && current.Deadline == candidate.Deadline)
             {
-                RemoveAttempt(candidate.AttemptId);
+                RemoveAttempt(candidate.Key);
                 removed++;
             }
         }
@@ -1024,6 +1128,184 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
         return removed;
     }
 
+    private int RemoveExpiredListings(TimeSpan now)
+    {
+        int removed = 0;
+        while (_listingExpiries.TryPeek(
+                   out DeadlineEntry<SessionListingId> candidate,
+                   out long deadlineTicks)
+               && deadlineTicks <= now.Ticks)
+        {
+            _listingExpiries.Dequeue();
+            _scheduledListingExpiries.Remove(candidate.Key);
+            if (!_listings.TryGetValue(candidate.Key, out ListingEntry? current))
+            {
+                continue;
+            }
+
+            if (current.LeaseDeadline > now)
+            {
+                ScheduleMutableDeadline(
+                    _listingExpiries,
+                    _scheduledListingExpiries,
+                    candidate.Key,
+                    current.LeaseDeadline);
+            }
+            else
+            {
+                RemoveListing(candidate.Key);
+                removed++;
+            }
+        }
+
+        return removed;
+    }
+
+    private int RemoveExpiredPresence(TimeSpan now)
+    {
+        int removed = 0;
+        while (_presenceExpiries.TryPeek(
+                   out DeadlineEntry<MediationHandle> candidate,
+                   out long deadlineTicks)
+               && deadlineTicks <= now.Ticks)
+        {
+            _presenceExpiries.Dequeue();
+            _scheduledPresenceExpiries.Remove(candidate.Key);
+            if (!_presence.TryGetValue(candidate.Key, out PresenceEntry? current))
+            {
+                continue;
+            }
+
+            if (current.Deadline > now)
+            {
+                ScheduleMutableDeadline(
+                    _presenceExpiries,
+                    _scheduledPresenceExpiries,
+                    candidate.Key,
+                    current.Deadline);
+            }
+            else if (_presence.Remove(candidate.Key))
+            {
+                removed++;
+            }
+        }
+
+        return removed;
+    }
+
+    private int RemoveExpiredOutcomes(TimeSpan now)
+    {
+        int removed = 0;
+        while (_outcomeExpiries.TryPeek(
+                   out DeadlineEntry<JoinAttemptId> candidate,
+                   out long deadlineTicks)
+               && deadlineTicks <= now.Ticks)
+        {
+            _outcomeExpiries.Dequeue();
+            if (_outcomeReports.TryGetValue(candidate.Key, out OutcomeReportEntry? current)
+                && current.Deadline == candidate.Deadline)
+            {
+                RemoveOutcome(candidate.Key);
+                removed++;
+            }
+        }
+
+        return removed;
+    }
+
+    private int RemoveExpiredIdempotency(TimeSpan now)
+    {
+        int removed = 0;
+        while (_idempotencyExpiries.TryPeek(
+                   out DeadlineEntry<string> candidate,
+                   out long deadlineTicks)
+               && deadlineTicks <= now.Ticks)
+        {
+            _idempotencyExpiries.Dequeue();
+            if (_idempotency.TryGetValue(candidate.Key, out IdempotencyEntry? current)
+                && current.Deadline == candidate.Deadline
+                && _idempotency.Remove(candidate.Key))
+            {
+                removed++;
+            }
+        }
+
+        return removed;
+    }
+
+    private static int RemoveExpiredDeadlines<TKey>(
+        Dictionary<TKey, TimeSpan> entries,
+        PriorityQueue<DeadlineEntry<TKey>, long> expiries,
+        TimeSpan now)
+        where TKey : notnull
+    {
+        int removed = 0;
+        while (expiries.TryPeek(out DeadlineEntry<TKey> candidate, out long deadlineTicks)
+               && deadlineTicks <= now.Ticks)
+        {
+            expiries.Dequeue();
+            if (entries.TryGetValue(candidate.Key, out TimeSpan current)
+                && current == candidate.Deadline
+                && entries.Remove(candidate.Key))
+            {
+                removed++;
+            }
+        }
+
+        return removed;
+    }
+
+    private static int RemoveExpiredMutableDeadlines<TKey>(
+        Dictionary<TKey, TimeSpan> entries,
+        PriorityQueue<DeadlineEntry<TKey>, long> expiries,
+        HashSet<TKey> scheduled,
+        TimeSpan now)
+        where TKey : notnull
+    {
+        int removed = 0;
+        while (expiries.TryPeek(out DeadlineEntry<TKey> candidate, out long deadlineTicks)
+               && deadlineTicks <= now.Ticks)
+        {
+            expiries.Dequeue();
+            scheduled.Remove(candidate.Key);
+            if (!entries.TryGetValue(candidate.Key, out TimeSpan current))
+            {
+                continue;
+            }
+
+            if (current > now)
+            {
+                ScheduleMutableDeadline(expiries, scheduled, candidate.Key, current);
+            }
+            else if (entries.Remove(candidate.Key))
+            {
+                removed++;
+            }
+        }
+
+        return removed;
+    }
+
+    private static void ScheduleMutableDeadline<TKey>(
+        PriorityQueue<DeadlineEntry<TKey>, long> expiries,
+        HashSet<TKey> scheduled,
+        TKey key,
+        TimeSpan deadline)
+        where TKey : notnull
+    {
+        if (scheduled.Add(key))
+        {
+            EnqueueDeadline(expiries, key, deadline);
+        }
+    }
+
+    private static void EnqueueDeadline<TKey>(
+        PriorityQueue<DeadlineEntry<TKey>, long> expiries,
+        TKey key,
+        TimeSpan deadline)
+        where TKey : notnull =>
+        expiries.Enqueue(new(key, deadline), deadline.Ticks);
+
     private bool HandleExists(MediationHandle handle) =>
         _presenceHandles.ContainsKey(handle) || _attemptHandles.ContainsKey(handle);
 
@@ -1063,20 +1345,6 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
         IsCancelled = entry.IsCancelled,
     };
 
-    private static int RemoveExpired(Dictionary<string, TimeSpan> entries, TimeSpan now)
-    {
-        string[] expired = entries
-            .Where(item => item.Value <= now)
-            .Select(static item => item.Key)
-            .ToArray();
-        foreach (string key in expired)
-        {
-            entries.Remove(key);
-        }
-
-        return expired.Length;
-    }
-
     private static void ValidateListing(ListingDefinition listing)
     {
         ArgumentNullException.ThrowIfNull(listing);
@@ -1219,7 +1487,8 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
         public bool IsCancelled { get; set; }
     }
 
-    private readonly record struct AttemptExpiry(JoinAttemptId AttemptId, TimeSpan Deadline);
+    private readonly record struct DeadlineEntry<TKey>(TKey Key, TimeSpan Deadline)
+        where TKey : notnull;
 
     private sealed class OutcomeReportEntry(
         SessionListingId listingId,

tests/FinalFactory.Rendezvous.Capacity/CapacityOptions.cs

@@ -0,0 +1,91 @@
+namespace FinalFactory.Rendezvous.Capacity;
+
+internal sealed record CapacityOptions
+{
+    public required string Profile { get; init; }
+    public required int Listings { get; init; }
+    public required int Attempts { get; init; }
+    public required int Samples { get; init; }
+    public required int SoakCycles { get; init; }
+    public required int SoakSeconds { get; init; }
+    public string? OutputPath { get; init; }
+
+    public static CapacityOptions Parse(string[] args)
+    {
+        Dictionary<string, string> values = ParseArguments(args);
+        string profile = values.GetValueOrDefault("--profile") ?? "quick";
+        (int listings, int attempts, int samples, int soakCycles, int soakSeconds) = profile switch
+        {
+            "quick" => (1_000, 500, 100, 20, 0),
+            "candidate" => (25_000, 10_000, 1_000, 1_000, 300),
+            _ => throw new ArgumentException("--profile must be 'quick' or 'candidate'."),
+        };
+
+        return new()
+        {
+            Profile = profile,
+            Listings = ParsePositive(values.GetValueOrDefault("--listings"), listings, "--listings"),
+            Attempts = ParsePositive(values.GetValueOrDefault("--attempts"), attempts, "--attempts"),
+            Samples = ParsePositive(values.GetValueOrDefault("--samples"), samples, "--samples"),
+            SoakCycles = ParsePositive(
+                values.GetValueOrDefault("--soak-cycles"),
+                soakCycles,
+                "--soak-cycles"),
+            SoakSeconds = ParseNonNegative(
+                values.GetValueOrDefault("--soak-seconds"),
+                soakSeconds,
+                "--soak-seconds"),
+            OutputPath = values.GetValueOrDefault("--output"),
+        };
+    }
+
+    private static Dictionary<string, string> ParseArguments(string[] args)
+    {
+        HashSet<string> allowed =
+        [
+            "--profile",
+            "--listings",
+            "--attempts",
+            "--samples",
+            "--soak-cycles",
+            "--soak-seconds",
+            "--output",
+        ];
+        Dictionary<string, string> values = new(StringComparer.Ordinal);
+        for (int index = 0; index < args.Length; index += 2)
+        {
+            string option = args[index];
+            if (!allowed.Contains(option))
+            {
+                throw new ArgumentException($"Unknown option: {option}.");
+            }
+
+            if (index == args.Length - 1
+                || args[index + 1].StartsWith("--", StringComparison.Ordinal))
+            {
+                throw new ArgumentException($"{option} requires a value.");
+            }
+
+            if (!values.TryAdd(option, args[index + 1]))
+            {
+                throw new ArgumentException($"{option} may be supplied only once.");
+            }
+        }
+
+        return values;
+    }
+
+    private static int ParsePositive(string? value, int fallback, string option) =>
+        value is null
+            ? fallback
+            : int.TryParse(value, out int parsed) && parsed > 0
+                ? parsed
+                : throw new ArgumentException($"{option} must be a positive integer.");
+
+    private static int ParseNonNegative(string? value, int fallback, string option) =>
+        value is null
+            ? fallback
+            : int.TryParse(value, out int parsed) && parsed >= 0
+                ? parsed
+                : throw new ArgumentException($"{option} must be a non-negative integer.");
+}

tests/FinalFactory.Rendezvous.Capacity/CapacityReport.cs

@@ -0,0 +1,76 @@
+namespace FinalFactory.Rendezvous.Capacity;
+
+internal sealed record CapacityReport
+{
+    public required int SchemaVersion { get; init; }
+    public required string EvidenceVersion { get; init; }
+    public required DateTimeOffset GeneratedAt { get; init; }
+    public required string Profile { get; init; }
+    public required RuntimeEvidence Runtime { get; init; }
+    public required CapacityTargets Targets { get; init; }
+    public required IReadOnlyList<CapacityMeasurement> Measurements { get; init; }
+    public required StateEvidence State { get; init; }
+    public required IReadOnlyList<string> Failures { get; init; }
+    public required bool Passed { get; init; }
+}
+
+internal sealed record RuntimeEvidence(
+    string Framework,
+    string OperatingSystem,
+    string Kernel,
+    string Architecture,
+    string CpuModel,
+    int ProcessorCount,
+    string CpuAffinity,
+    string CpuQuota,
+    string MemoryLimit,
+    string GarbageCollector,
+    string CommitSha,
+    string TreeState,
+    string Command,
+    string ImageDigest,
+    string WorkloadSeed,
+    double CapacityPhaseAverageCpuPercent,
+    long PeakWorkingSetBytes,
+    long ManagedBytesAfterCleanup);
+
+internal sealed record CapacityTargets(
+    int VisibleListings,
+    int ActiveJoinAttempts,
+    int CoreControlOperationsPerSecond,
+    int CoreMediationOperationsPerSecond,
+    double CoreControlP95Milliseconds,
+    double CoreMediationP95Milliseconds,
+    double MaximumAverageCpuPercent,
+    long MaximumWorkingSetBytes,
+    int SoakCycles,
+    int SoakDurationSeconds);
+
+internal sealed record CapacityMeasurement(
+    string Operation,
+    int Samples,
+    double P50Milliseconds,
+    double P95Milliseconds,
+    double P99Milliseconds,
+    double OperationsPerSecond,
+    double MinimumOperationsPerSecond,
+    double BudgetMilliseconds,
+    bool Passed);
+
+internal sealed record StateEvidence(
+    int PeakListings,
+    int PeakAttempts,
+    int PeakReplayMarkers,
+    int FinalListings,
+    int FinalAttempts,
+    int FinalReplayMarkers,
+    long ExpiryChurn,
+    long MaintenanceSweeps,
+    int SoakCyclesCompleted,
+    double SoakDurationSeconds,
+    int SoakPeakScheduledExpiryEntries,
+    long SoakManagedGrowthBytes,
+    int SoakHandleGrowth,
+    bool RestartStartedEmpty,
+    bool OverloadWasTyped,
+    bool RecoverySucceeded);

tests/FinalFactory.Rendezvous.Capacity/CapacityRunner.cs

@@ -0,0 +1,580 @@
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Runtime;
+using System.Runtime.InteropServices;
+using FinalFactory.Rendezvous.Contracts;
+using FinalFactory.Rendezvous.Server.Observability;
+using FinalFactory.Rendezvous.Server.State;
+
+namespace FinalFactory.Rendezvous.Capacity;
+
+internal static class CapacityRunner
+{
+    private static readonly TenantScope Scope = new(new("space-game"), new("production"));
+    private const uint ProtocolVersion = 1;
+
+    public static Task<CapacityReport> RunAsync(CapacityOptions options)
+    {
+        ArgumentNullException.ThrowIfNull(options);
+        Process process = Process.GetCurrentProcess();
+        TimeSpan cpuBefore = process.TotalProcessorTime;
+        Stopwatch capacityPhaseTime = Stopwatch.StartNew();
+        List<string> failures = [];
+        List<CapacityMeasurement> measurements = [];
+        ManualClock clock = new();
+        InMemoryEphemeralRendezvousStore store = CreateStore(options, clock);
+        List<StoredListing> listings = new(options.Listings);
+        List<CreateJoinAttemptCommand> attempts = new(options.Attempts);
+        int registrationSamples = Math.Min(options.Samples, options.Listings);
+        int attemptSamples = Math.Min(options.Samples, options.Attempts);
+
+        for (int index = 0; index < options.Listings - registrationSamples; index++)
+        {
+            listings.Add(CreateVisibleListing(store, index));
+        }
+
+        measurements.Add(Measure(
+            "registration-and-presence",
+            registrationSamples,
+            budgetMilliseconds: 200,
+            minimumOperationsPerSecond: 200,
+            index => listings.Add(CreateVisibleListing(
+                store,
+                options.Listings - registrationSamples + index))));
+
+        measurements.Add(Measure(
+            "lease-renewal",
+            registrationSamples,
+            budgetMilliseconds: 200,
+            minimumOperationsPerSecond: 200,
+            index =>
+            {
+                StoredListing listing = listings[index];
+                StoreResult<StoredListing> renewed = store.RenewLease(new(
+                    listing.Definition.ListingId,
+                    listing.Definition.LeaseId,
+                    listing.Definition.LeaseFingerprint,
+                    listing.Definition.OwnerSubject,
+                    listing.Version));
+                RequireSuccess(renewed, "renewal");
+                listings[index] = renewed.Value!;
+            }));
+
+        int browseSamples = Math.Min(options.Samples, 250);
+        measurements.Add(Measure(
+            "visible-session-browse",
+            browseSamples,
+            budgetMilliseconds: 200,
+            minimumOperationsPerSecond: 200,
+            _ => RequireSuccess(
+                store.BrowseVisibleListings(new(
+                    Scope,
+                    ProtocolVersion,
+                    new RegionId("eu-central"),
+                    ContractLimits.BrowserPageMaxItems,
+                    ExcludeFull: true)),
+                "browse")));
+
+        for (int index = 0; index < options.Attempts - attemptSamples; index++)
+        {
+            CreateJoinAttemptCommand command = CreateAttempt(index, listings[index % listings.Count]);
+            RequireSuccess(store.CreateJoinAttempt(command), "join issuance");
+            attempts.Add(command);
+        }
+
+        measurements.Add(Measure(
+            "join-attempt-issuance",
+            attemptSamples,
+            budgetMilliseconds: 200,
+            minimumOperationsPerSecond: 200,
+            index =>
+            {
+                int sequence = options.Attempts - attemptSamples + index;
+                CreateJoinAttemptCommand command = CreateAttempt(
+                    sequence,
+                    listings[sequence % listings.Count]);
+                RequireSuccess(store.CreateJoinAttempt(command), "join issuance");
+                attempts.Add(command);
+            }));
+
+        int punchSamples = Math.Min(options.Samples, attempts.Count);
+        measurements.Add(MeasureConcurrentPunch(store, attempts, punchSamples));
+        EphemeralStoreSnapshot peak = store.GetSnapshot();
+
+        StoreResult<StoredJoinAttempt> overloaded = store.CreateJoinAttempt(
+            CreateAttempt(options.Attempts + 1, listings[^1]));
+        bool overloadWasTyped = peak.ActiveJoinAttempts == options.Attempts
+            && overloaded.Code == StoreResultCode.CapacityExceeded;
+        if (!overloadWasTyped)
+        {
+            failures.Add(
+                $"Expected typed CapacityExceeded at {options.Attempts} active attempts, "
+                + $"observed count={peak.ActiveJoinAttempts}, result={overloaded.Code}.");
+        }
+
+        int revocationSamples = Math.Min(Math.Max(1, options.Samples / 20), listings.Count / 2);
+        measurements.Add(Measure(
+            "principal-revocation",
+            revocationSamples,
+            budgetMilliseconds: 200,
+            minimumOperationsPerSecond: 50,
+            index => RequireSuccess(
+                store.RevokePrincipal(
+                    listings[index].Definition.OwnerSubject,
+                    TimeSpan.FromMinutes(1)),
+                "principal revocation")));
+
+        using (RendezvousTelemetry telemetry = new(store))
+        {
+            measurements.Add(Measure(
+                "telemetry-recording",
+                Math.Max(100, options.Samples),
+                budgetMilliseconds: 1,
+                minimumOperationsPerSecond: 10_000,
+                _ =>
+                {
+                    telemetry.RecordHttp("browse", 200, 1);
+                    telemetry.RecordUdp("contribution", "accepted", 1);
+                    telemetry.RecordPairingLatency(2);
+                }));
+        }
+
+        clock.Advance(TimeSpan.FromSeconds(61));
+        EphemeralStoreSnapshot? afterCoincidentExpiry = null;
+        measurements.Add(Measure(
+            "coincident-listing-attempt-expiry",
+            samples: 1,
+            budgetMilliseconds: 200,
+            minimumOperationsPerSecond: 0,
+            _ => afterCoincidentExpiry = store.GetSnapshot()));
+        if (afterCoincidentExpiry!.ActiveJoinAttempts != 0
+            || afterCoincidentExpiry.ActiveListings != 0)
+        {
+            failures.Add("Coincident 60-second cleanup retained expired listings or join attempts.");
+        }
+
+        clock.Advance(TimeSpan.FromSeconds(90));
+        _ = store.GetSnapshot();
+        StoredListing recoveryListing = CreateVisibleListing(store, options.Listings + 1);
+        StoreResult<StoredJoinAttempt> recovered = store.CreateJoinAttempt(
+            CreateAttempt(options.Attempts + 2, recoveryListing));
+        bool recoverySucceeded = recovered.Succeeded;
+        if (!recoverySucceeded)
+        {
+            failures.Add($"Store did not recover after attempt expiry: {recovered.Code}.");
+        }
+
+        clock.Advance(TimeSpan.FromSeconds(151));
+        EphemeralStoreSnapshot final = store.GetSnapshot();
+        if (final.ActiveListings != 0
+            || final.ActiveJoinAttempts != 0
+            || final.ReplayMarkers != 0
+            || final.IdempotencyEntries != 0
+            || final.RetainedOutcomeReports != 0)
+        {
+            failures.Add("Expiry cleanup left active or retained state after every configured deadline.");
+        }
+
+        capacityPhaseTime.Stop();
+        process.Refresh();
+        double capacityPhaseCpuPercent = 100
+            * (process.TotalProcessorTime - cpuBefore).TotalSeconds
+            / Math.Max(capacityPhaseTime.Elapsed.TotalSeconds * Environment.ProcessorCount, 0.000_001);
+        if (options.Profile == "candidate" && capacityPhaseCpuPercent > 70)
+        {
+            failures.Add(
+                $"Capacity-phase CPU {capacityPhaseCpuPercent:F1}% exceeded the 70% candidate budget.");
+        }
+
+        SoakEvidence soak = RunAcceleratedSoak(options, failures);
+        ManualClock restartClock = new();
+        EphemeralStoreSnapshot restarted = CreateStore(options, restartClock).GetSnapshot();
+        bool restartStartedEmpty = restarted.ActiveListings == 0
+            && restarted.ActiveJoinAttempts == 0
+            && restarted.ReplayMarkers == 0;
+        if (!restartStartedEmpty)
+        {
+            failures.Add("A restarted store did not begin empty.");
+        }
+
+        foreach (CapacityMeasurement measurement in measurements.Where(static item => !item.Passed))
+        {
+            failures.Add(
+                $"{measurement.Operation} missed its budget: p95={measurement.P95Milliseconds:F3} ms, "
+                + $"rate={measurement.OperationsPerSecond:F1}/s.");
+        }
+
+        GC.Collect();
+        GC.WaitForPendingFinalizers();
+        GC.Collect();
+        process.Refresh();
+        long managedAfterCleanup = GC.GetTotalMemory(forceFullCollection: true);
+        long memoryBudget = 1_610_612_736;
+        if (process.PeakWorkingSet64 > memoryBudget)
+        {
+            failures.Add(
+                $"Peak working set {process.PeakWorkingSet64} exceeded the 1.5 GiB profile budget.");
+        }
+
+        if (options.Profile == "candidate" && Environment.ProcessorCount != 2)
+        {
+            failures.Add(
+                $"Candidate evidence must expose exactly two CPUs; runtime exposed "
+                + $"{Environment.ProcessorCount}.");
+        }
+
+        CapacityReport report = new()
+        {
+            SchemaVersion = 2,
+            EvidenceVersion = "v2",
+            GeneratedAt = DateTimeOffset.UtcNow,
+            Profile = options.Profile,
+            Runtime = new(
+                RuntimeInformation.FrameworkDescription,
+                RuntimeInformation.OSDescription,
+                Environment.OSVersion.VersionString,
+                RuntimeInformation.ProcessArchitecture.ToString(),
+                ReadCpuModel(),
+                Environment.ProcessorCount,
+                Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_CPUSET") ?? "unrestricted",
+                ReadCgroupValue("/sys/fs/cgroup/cpu.max"),
+                ReadCgroupValue("/sys/fs/cgroup/memory.max"),
+                GCSettings.IsServerGC ? "server" : "workstation",
+                Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_COMMIT") ?? "unrecorded",
+                Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_TREE_STATE") ?? "unrecorded",
+                Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_COMMAND") ?? "unrecorded",
+                Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_IMAGE_DIGEST") ?? "not-containerized",
+                "fixed-sequences-random-identifiers",
+                capacityPhaseCpuPercent,
+                process.PeakWorkingSet64,
+                managedAfterCleanup),
+            Targets = new(
+                options.Listings,
+                options.Attempts,
+                200,
+                2_000,
+                200,
+                100,
+                70,
+                memoryBudget,
+                options.SoakCycles,
+                options.SoakSeconds),
+            Measurements = measurements,
+            State = new(
+                peak.ActiveListings,
+                peak.ActiveJoinAttempts,
+                peak.ReplayMarkers,
+                final.ActiveListings,
+                final.ActiveJoinAttempts,
+                final.ReplayMarkers,
+                final.ExpiryChurn,
+                final.MaintenanceSweeps,
+                soak.Cycles,
+                soak.Duration.TotalSeconds,
+                soak.PeakScheduledExpiryEntries,
+                soak.ManagedGrowthBytes,
+                soak.HandleGrowth,
+                restartStartedEmpty,
+                overloadWasTyped,
+                recoverySucceeded),
+            Failures = failures,
+            Passed = failures.Count == 0,
+        };
+        return Task.FromResult(report);
+    }
+
+    private static InMemoryEphemeralRendezvousStore CreateStore(
+        CapacityOptions options,
+        ManualClock clock) => new(
+        new EphemeralStoreOptions
+        {
+            MaxListings = options.Listings,
+            MaxPresenceBindings = options.Listings,
+            MaxJoinAttempts = options.Attempts,
+            MaxOutcomeReports = options.Attempts,
+            MaxIdempotencyEntries = options.Listings + options.Attempts + 1,
+        },
+        clock,
+        clock);
+
+    private static StoredListing CreateVisibleListing(
+        InMemoryEphemeralRendezvousStore store,
+        int sequence)
+    {
+        string owner = $"publisher-{sequence}";
+        SecretFingerprint leaseFingerprint = new($"lease-{sequence}");
+        SecretFingerprint presenceFingerprint = new($"presence-{sequence}");
+        ListingDefinition definition = new()
+        {
+            ListingId = new(Guid.NewGuid()),
+            LeaseId = new(Guid.NewGuid()),
+            Scope = Scope,
+            OwnerSubject = owner,
+            RegionId = new("eu-central"),
+            ProtocolVersion = ProtocolVersion,
+            BuildVersion = "1.0.0",
+            DisplayName = $"Capacity host {sequence}",
+            Visibility = ListingVisibility.Public,
+            TrustMode = PublisherTrustMode.ManagedDedicated,
+            CurrentPlayers = 1,
+            MaximumPlayers = 8,
+            Metadata = new Dictionary<string, string>(StringComparer.Ordinal),
+            LeaseFingerprint = leaseFingerprint,
+            HostPresenceHandle = new(Guid.NewGuid()),
+            HostPresenceFingerprint = presenceFingerprint,
+            CapabilityDerivationSalt = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+        };
+        StoreResult<StoredListing> created = store.CreateListing(new(
+            $"register-{sequence}",
+            $"register-request-{sequence}",
+            definition));
+        RequireSuccess(created, "registration");
+        StoreResult<StoredListing> bound = store.BindHostPresence(new(
+            definition.HostPresenceHandle,
+            presenceFingerprint,
+            PublicEndpoint(10_000 + sequence % 50_000),
+            null));
+        RequireSuccess(bound, "host presence");
+        return bound.Value!;
+    }
+
+    private static CreateJoinAttemptCommand CreateAttempt(int sequence, StoredListing listing) => new()
+    {
+        IdempotencyOwner = $"client-{sequence}",
+        IdempotencyKey = $"join-{sequence}",
+        RequestFingerprint = $"join-request-{sequence}",
+        ClientSubject = $"client-{sequence}",
+        AttemptId = new(Guid.NewGuid()),
+        MediationHandle = new(Guid.NewGuid()),
+        Scope = Scope,
+        ListingId = listing.Definition.ListingId,
+        ProtocolVersion = ProtocolVersion,
+        HostCapabilityFingerprint = new($"host-capability-{sequence}"),
+        ClientCapabilityFingerprint = new($"client-capability-{sequence}"),
+        ConnectionTicketFingerprint = new($"ticket-{sequence}"),
+        CapabilityDerivationSalt = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+    };
+
+    private static CapacityMeasurement MeasureConcurrentPunch(
+        InMemoryEphemeralRendezvousStore store,
+        List<CreateJoinAttemptCommand> attempts,
+        int samples)
+    {
+        ConcurrentBag<double> latencies = [];
+        Stopwatch total = Stopwatch.StartNew();
+        Parallel.ForEach(
+            Enumerable.Range(0, samples),
+            new ParallelOptions { MaxDegreeOfParallelism = Math.Min(64, Environment.ProcessorCount * 4) },
+            index =>
+            {
+                CreateJoinAttemptCommand attempt = attempts[index];
+                Stopwatch elapsed = Stopwatch.StartNew();
+                RequireSuccess(store.BindAttemptEndpoint(new(
+                    attempt.MediationHandle,
+                    AttemptPeerRole.Host,
+                    attempt.HostCapabilityFingerprint,
+                    PublicEndpoint(20_000 + index % 20_000),
+                    null)), "host punch");
+                RequireSuccess(store.BindAttemptEndpoint(new(
+                    attempt.MediationHandle,
+                    AttemptPeerRole.Client,
+                    attempt.ClientCapabilityFingerprint,
+                    PublicEndpoint(40_000 + index % 20_000),
+                    null)), "client punch");
+                RequireSuccess(store.ConsumeIntroduction(attempt.MediationHandle), "introduction");
+                latencies.Add(elapsed.Elapsed.TotalMilliseconds);
+            });
+        total.Stop();
+        return BuildMeasurement(
+            "simultaneous-punch-pairing",
+            latencies.ToArray(),
+            total.Elapsed,
+            budgetMilliseconds: 100,
+            minimumOperationsPerSecond: 2_000);
+    }
+
+    private static CapacityMeasurement Measure(
+        string operation,
+        int samples,
+        double budgetMilliseconds,
+        double minimumOperationsPerSecond,
+        Action<int> action)
+    {
+        double[] latencies = new double[samples];
+        Stopwatch total = Stopwatch.StartNew();
+        for (int index = 0; index < samples; index++)
+        {
+            long started = Stopwatch.GetTimestamp();
+            action(index);
+            latencies[index] = Stopwatch.GetElapsedTime(started).TotalMilliseconds;
+        }
+
+        total.Stop();
+        return BuildMeasurement(
+            operation,
+            latencies,
+            total.Elapsed,
+            budgetMilliseconds,
+            minimumOperationsPerSecond);
+    }
+
+    private static CapacityMeasurement BuildMeasurement(
+        string operation,
+        double[] latencies,
+        TimeSpan elapsed,
+        double budgetMilliseconds,
+        double minimumOperationsPerSecond)
+    {
+        Array.Sort(latencies);
+        double operationsPerSecond = latencies.Length / Math.Max(elapsed.TotalSeconds, 0.000_001);
+        double p95 = Percentile(latencies, 0.95);
+        return new(
+            operation,
+            latencies.Length,
+            Percentile(latencies, 0.50),
+            p95,
+            Percentile(latencies, 0.99),
+            operationsPerSecond,
+            minimumOperationsPerSecond,
+            budgetMilliseconds,
+            p95 <= budgetMilliseconds && operationsPerSecond >= minimumOperationsPerSecond);
+    }
+
+    private static double Percentile(double[] sorted, double percentile)
+    {
+        int index = Math.Clamp((int)Math.Ceiling(sorted.Length * percentile) - 1, 0, sorted.Length - 1);
+        return sorted[index];
+    }
+
+    private static SoakEvidence RunAcceleratedSoak(
+        CapacityOptions options,
+        List<string> failures)
+    {
+        GC.Collect();
+        GC.WaitForPendingFinalizers();
+        GC.Collect();
+        long managedBefore = GC.GetTotalMemory(forceFullCollection: true);
+        int handlesBefore = Process.GetCurrentProcess().HandleCount;
+        ManualClock clock = new();
+        CapacityOptions soakOptions = options with { Listings = 100, Attempts = 100 };
+        InMemoryEphemeralRendezvousStore store = CreateStore(soakOptions, clock);
+        Stopwatch elapsed = Stopwatch.StartNew();
+        int cycle = 0;
+        int peakScheduledExpiryEntries = 0;
+        while (cycle < options.SoakCycles
+               || elapsed.Elapsed < TimeSpan.FromSeconds(options.SoakSeconds))
+        {
+            StoredListing listing = CreateVisibleListing(store, cycle);
+            for (int refresh = 0; refresh < 10; refresh++)
+            {
+                clock.Advance(TimeSpan.FromTicks(1));
+                listing = RequireSuccess(store.RenewLease(new(
+                    listing.Definition.ListingId,
+                    listing.Definition.LeaseId,
+                    listing.Definition.LeaseFingerprint,
+                    listing.Definition.OwnerSubject,
+                    listing.Version)), "soak lease refresh");
+                listing = RequireSuccess(store.BindHostPresence(new(
+                    listing.Definition.HostPresenceHandle,
+                    listing.Definition.HostPresenceFingerprint,
+                    PublicEndpoint(10_000 + cycle % 50_000),
+                    null)), "soak presence refresh");
+            }
+
+            CreateJoinAttemptCommand attempt = CreateAttempt(cycle, listing);
+            RequireSuccess(store.CreateJoinAttempt(attempt), "soak join issuance");
+            RequireSuccess(store.ConsumeReplay(new("capacity-soak", $"replay-{cycle}")), "soak replay");
+            peakScheduledExpiryEntries = Math.Max(
+                peakScheduledExpiryEntries,
+                store.ScheduledExpiryEntryCount);
+            if (store.ScheduledExpiryEntryCount > 7)
+            {
+                failures.Add(
+                    $"Mutable deadline refresh grew the expiry queue to "
+                    + $"{store.ScheduledExpiryEntryCount} entries for one lifecycle.");
+                break;
+            }
+
+            clock.Advance(TimeSpan.FromSeconds(151));
+            EphemeralStoreSnapshot snapshot = store.GetSnapshot();
+            if (snapshot.ActiveListings != 0
+                || snapshot.ActiveJoinAttempts != 0
+                || snapshot.ReplayMarkers != 0
+                || snapshot.IdempotencyEntries != 0
+                || snapshot.RetainedOutcomeReports != 0)
+            {
+                failures.Add($"Accelerated soak retained state after cycle {cycle}.");
+                break;
+            }
+
+            cycle++;
+        }
+
+        elapsed.Stop();
+        GC.Collect();
+        GC.WaitForPendingFinalizers();
+        GC.Collect();
+        long managedGrowth = GC.GetTotalMemory(forceFullCollection: true) - managedBefore;
+        int handleGrowth = Process.GetCurrentProcess().HandleCount - handlesBefore;
+        if (managedGrowth > 67_108_864)
+        {
+            failures.Add($"Soak retained {managedGrowth} managed bytes; budget is 64 MiB.");
+        }
+
+        if (handleGrowth > 8)
+        {
+            failures.Add($"Soak retained {handleGrowth} process handles; budget is 8.");
+        }
+
+        return new(cycle, elapsed.Elapsed, peakScheduledExpiryEntries, managedGrowth, handleGrowth);
+    }
+
+    private static ObservedEndpoint PublicEndpoint(int port) =>
+        new(AddressFamilyKind.Ipv4, "203.0.113.10", port);
+
+    private static T RequireSuccess<T>(StoreResult<T> result, string operation)
+    {
+        if (!result.Succeeded)
+        {
+            throw new InvalidOperationException($"{operation} failed with {result.Code}.");
+        }
+
+        return result.Value!;
+    }
+
+    private static string ReadCpuModel()
+    {
+        const string cpuInfoPath = "/proc/cpuinfo";
+        if (!File.Exists(cpuInfoPath))
+        {
+            return "unavailable";
+        }
+
+        string? model = File.ReadLines(cpuInfoPath)
+            .FirstOrDefault(static line => line.StartsWith("model name", StringComparison.Ordinal));
+        int separator = model?.IndexOf(':') ?? -1;
+        return separator >= 0 ? model![(separator + 1)..].Trim() : "unavailable";
+    }
+
+    private static string ReadCgroupValue(string path) =>
+        File.Exists(path) ? File.ReadAllText(path).Trim() : "not-enforced";
+
+    private sealed class ManualClock : IWallClock, IMonotonicClock
+    {
+        public DateTimeOffset UtcNow { get; private set; } = DateTimeOffset.UtcNow;
+        public TimeSpan Elapsed { get; private set; }
+
+        public void Advance(TimeSpan duration)
+        {
+            UtcNow += duration;
+            Elapsed += duration;
+        }
+    }
+
+    private readonly record struct SoakEvidence(
+        int Cycles,
+        TimeSpan Duration,
+        int PeakScheduledExpiryEntries,
+        long ManagedGrowthBytes,
+        int HandleGrowth);
+}

tests/FinalFactory.Rendezvous.Capacity/FinalFactory.Rendezvous.Capacity.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <AssemblyName>FinalFactory.Rendezvous.Capacity</AssemblyName>
+    <RootNamespace>FinalFactory.Rendezvous.Capacity</RootNamespace>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="../../src/FinalFactory.Rendezvous.Contracts/FinalFactory.Rendezvous.Contracts.csproj" />
+    <ProjectReference Include="../../src/FinalFactory.Rendezvous.Server/FinalFactory.Rendezvous.Server.csproj" />
+  </ItemGroup>
+</Project>

tests/FinalFactory.Rendezvous.Capacity/Program.cs

@@ -0,0 +1,37 @@
+using System.Text.Json;
+
+namespace FinalFactory.Rendezvous.Capacity;
+
+internal static class Program
+{
+    private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
+    {
+        WriteIndented = true,
+    };
+
+    public static async Task<int> Main(string[] args)
+    {
+        CapacityOptions options;
+        try
+        {
+            options = CapacityOptions.Parse(args);
+        }
+        catch (ArgumentException exception)
+        {
+            Console.Error.WriteLine(exception.Message);
+            return 2;
+        }
+
+        CapacityReport report = await CapacityRunner.RunAsync(options).ConfigureAwait(false);
+        string json = JsonSerializer.Serialize(report, JsonOptions);
+        Console.WriteLine(json);
+        if (options.OutputPath is not null)
+        {
+            string fullPath = Path.GetFullPath(options.OutputPath);
+            Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!);
+            await File.WriteAllTextAsync(fullPath, json + Environment.NewLine).ConfigureAwait(false);
+        }
+
+        return report.Passed ? 0 : 1;
+    }
+}

tests/FinalFactory.Rendezvous.Capacity/packages.lock.json

@@ -0,0 +1,39 @@
+{
+  "version": 2,
+  "dependencies": {
+    "net10.0": {
+      "finalfactory.rendezvous.contracts": {
+        "type": "Project"
+      },
+      "finalfactory.rendezvous.server": {
+        "type": "Project",
+        "dependencies": {
+          "FinalFactory.Rendezvous.Contracts": "[1.0.0, )",
+          "LiteNetLib": "[2.1.4, )",
+          "Microsoft.AspNetCore.OpenApi": "[10.0.9, )"
+        }
+      },
+      "LiteNetLib": {
+        "type": "CentralTransitive",
+        "requested": "[2.1.4, )",
+        "resolved": "2.1.4",
+        "contentHash": "KWlxvMw3Urpqj9joD96LRiK+LC62pQNs/zkXRJc+rHnxgkGp+vV703xzDrxRmv+V1YhCFfIGzs5nrVWtREIlyA=="
+      },
+      "Microsoft.AspNetCore.OpenApi": {
+        "type": "CentralTransitive",
+        "requested": "[10.0.9, )",
+        "resolved": "10.0.9",
+        "contentHash": "1ihb8FO9cGgEK1/m3CTtT/SfnynwmiZib0W2pcDVj3KSWk/Sca4VOXEtaptKQc582zpFrzTFiwkGRCglt6H+WQ==",
+        "dependencies": {
+          "Microsoft.OpenApi": "2.0.0"
+        }
+      },
+      "Microsoft.OpenApi": {
+        "type": "CentralTransitive",
+        "requested": "[2.7.5, )",
+        "resolved": "2.7.5",
+        "contentHash": "0FA67RSnRM4tcBKqiqVu/HPdZ9+QOKbmeRjxRUGTCjPU4C0bmUhd97Dso7Yild5P7nOV6GxJ2xrK0Kv/O9xp0w=="
+      }
+    }
+  }
+}

tests/FinalFactory.Rendezvous.Tests/Client/RendezvousClientIntegrationTests.cs

@@ -1,3 +1,6 @@
+using System.Diagnostics;
+using System.Net;
+using System.Net.Sockets;
 using FinalFactory.Rendezvous.Client;
 using FinalFactory.Rendezvous.Contracts;
 using FinalFactory.Rendezvous.Server.Abuse;
@@ -19,6 +22,49 @@ namespace FinalFactory.Rendezvous.Tests.Client;
 
 public sealed class RendezvousClientIntegrationTests
 {
+    [Fact]
+    public async Task RestartReturnsTypedUnavailabilityThenAllowsHostReregistration()
+    {
+        int port = ReserveTcpPort();
+        string address = $"http://127.0.0.1:{port}";
+        using HttpClient client = new() { BaseAddress = new Uri(address) };
+        RendezvousClientOptions noRetry = new()
+        {
+            MaximumSafeRetries = 0,
+            RequestTimeout = TimeSpan.FromSeconds(1),
+        };
+        ClientTestHost first = await ClientTestHost.StartAsync(address);
+        RendezvousPublisherClient publisher = new(client, noRetry);
+        RendezvousClientResult<PublishedSession> registered = await publisher.RegisterAsync(
+            CreateRegistration(100),
+            first.PublisherCredential);
+        PublishedSession initialSession = AssertSuccess(registered);
+        BindPresence(first, initialSession, 41_100);
+        RendezvousSessionBrowserClient browser = new(client, noRetry);
+        BrowseSessionsResponse beforeRestart = AssertSuccess(await browser.BrowseAsync(BrowseRequest()));
+        Assert.Equal(initialSession.ListingId, Assert.Single(beforeRestart.Items).ListingId);
+        Stopwatch restart = Stopwatch.StartNew();
+        await first.DisposeAsync();
+
+        RendezvousClientResult<BrowseSessionsResponse> unavailable = await browser.BrowseAsync(BrowseRequest());
+        Assert.False(unavailable.IsSuccess);
+        Assert.Equal(RendezvousErrorCode.ServiceUnavailable, unavailable.Error);
+
+        await using ClientTestHost second = await ClientTestHost.StartAsync(address);
+        RendezvousClientResult<PublishedSession> reregistered = await publisher.RegisterAsync(
+            CreateRegistration(101),
+            second.PublisherCredential);
+        PublishedSession replacementSession = AssertSuccess(reregistered);
+        Assert.NotEqual(initialSession.ListingId, replacementSession.ListingId);
+        BindPresence(second, replacementSession, 41_101);
+        BrowseSessionsResponse afterRestart = AssertSuccess(await browser.BrowseAsync(BrowseRequest()));
+        Assert.Equal(replacementSession.ListingId, Assert.Single(afterRestart.Items).ListingId);
+        Assert.DoesNotContain(afterRestart.Items, item => item.ListingId == initialSession.ListingId);
+        Assert.True(
+            restart.Elapsed < TimeSpan.FromSeconds(5),
+            $"Local restart and host re-registration took {restart.Elapsed}.");
+    }
+
     [Fact]
     public async Task PublisherAndBrowserClientsCompleteTheRealSessionLifecycleAndPaging()
     {
@@ -102,6 +148,27 @@ public sealed class RendezvousClientIntegrationTests
         return Assert.IsAssignableFrom<T>(result.Value);
     }
 
+    private static void BindPresence(ClientTestHost host, PublishedSession session, int port)
+    {
+        Assert.True(host.Capabilities.TryFingerprint(
+            session.HostPresenceCapability,
+            out SecretFingerprint fingerprint));
+        Assert.Equal(StoreResultCode.Success, host.Store.BindHostPresence(new(
+            session.HostPresenceHandle,
+            fingerprint,
+            new(AddressFamilyKind.Ipv4, "203.0.113.80", port),
+            null)).Code);
+    }
+
+    private static BrowseSessionsRequest BrowseRequest() => new()
+    {
+        GameId = new("space-game"),
+        EnvironmentId = new("production"),
+        RegionId = new("eu-central"),
+        ProtocolVersion = 7,
+        PageSize = 10,
+    };
+
     private static RegisterSessionRequest CreateRegistration(int index) => new()
     {
         IdempotencyKey = $"sdk-integration-{index}",
@@ -139,7 +206,7 @@ public sealed class RendezvousClientIntegrationTests
         internal EphemeralCapabilityIssuer Capabilities { get; }
         internal string PublisherCredential { get; }
 
-        internal static async Task<ClientTestHost> StartAsync()
+        internal static async Task<ClientTestHost> StartAsync(string? bindAddress = null)
         {
             ManualRendezvousClock clock = new(ProvisioningTestData.Now);
             EphemeralStoreOptions stateOptions = new();
@@ -153,7 +220,7 @@ public sealed class RendezvousClientIntegrationTests
             string credential = provisioning.Credentials.Issue(principal, clock.UtcNow);
 
             WebApplicationBuilder builder = WebApplication.CreateBuilder();
-            builder.WebHost.UseUrls("http://127.0.0.1:0");
+            builder.WebHost.UseUrls(bindAddress ?? "http://127.0.0.1:0");
             builder.Services.ConfigureHttpJsonOptions(static options =>
                 ContractJson.Configure(options.SerializerOptions));
             builder.Services.Configure<RouteHandlerOptions>(static options =>
@@ -180,10 +247,10 @@ public sealed class RendezvousClientIntegrationTests
             app.MapRendezvousContractEndpoints();
             await app.StartAsync();
             IServer server = app.Services.GetRequiredService<IServer>();
-            string address = Assert.Single(server.Features.Get<IServerAddressesFeature>()!.Addresses);
+            string serviceAddress = Assert.Single(server.Features.Get<IServerAddressesFeature>()!.Addresses);
             return new(
                 app,
-                new HttpClient { BaseAddress = new Uri(address) },
+                new HttpClient { BaseAddress = new Uri(serviceAddress) },
                 store,
                 capabilities,
                 credential);
@@ -196,4 +263,13 @@ public sealed class RendezvousClientIntegrationTests
             await _application.DisposeAsync();
         }
     }
+
+    private static int ReserveTcpPort()
+    {
+        TcpListener listener = new(IPAddress.Loopback, 0);
+        listener.Start();
+        int port = ((IPEndPoint)listener.LocalEndpoint).Port;
+        listener.Stop();
+        return port;
+    }
 }

tests/FinalFactory.Rendezvous.Tests/Deployment/ProductionProcessTests.cs

@@ -61,6 +61,32 @@ public sealed class ProductionProcessTests
             Assert.InRange(shutdown.Elapsed, TimeSpan.FromMilliseconds(700), TimeSpan.FromSeconds(5));
             AssertTcpPortIsReleased(httpPort);
             AssertUdpPortIsReleased(udpPort);
+
+            process.Dispose();
+            process = null;
+            Stopwatch replacementReady = Stopwatch.StartNew();
+            ProcessStartInfo replacementInfo = CreateStartInfo(httpPort, udpPort, secretPath);
+            process = Process.Start(replacementInfo)
+                ?? throw new InvalidOperationException("The replacement production process did not start.");
+            Task<string> replacementOutput = process.StandardOutput.ReadToEndAsync();
+            Task<string> replacementError = process.StandardError.ReadToEndAsync();
+            await WaitForReadyAsync(httpPort, process, TimeSpan.FromSeconds(15));
+            Assert.True(
+                replacementReady.Elapsed < TimeSpan.FromSeconds(15),
+                $"Replacement readiness took {replacementReady.Elapsed}.");
+            AssertUdpPortIsBound(udpPort);
+
+            await SendSigtermAsync(process);
+            using CancellationTokenSource replacementTimeout = new(TimeSpan.FromSeconds(6));
+            await process.WaitForExitAsync(replacementTimeout.Token);
+            string replacementFinalOutput = await replacementOutput;
+            string replacementFinalError = await replacementError;
+            Assert.True(
+                process.ExitCode == 0,
+                $"Replacement exited with {process.ExitCode}. "
+                + $"stdout: {replacementFinalOutput} stderr: {replacementFinalError}");
+            AssertTcpPortIsReleased(httpPort);
+            AssertUdpPortIsReleased(udpPort);
         }
         finally
         {
@@ -79,6 +105,97 @@ public sealed class ProductionProcessTests
         }
     }
 
+    [Fact]
+    public async Task ProductionTransportSoakKeepsHandlesMemoryAndSocketsBounded()
+    {
+        if (!OperatingSystem.IsLinux())
+        {
+            return;
+        }
+
+        int httpPort = ReserveTcpPort();
+        int udpPort = ReserveUdpPort();
+        string secretPath = Path.Combine(
+            Path.GetTempPath(),
+            $"rendezvous-transport-soak-secret-{Guid.NewGuid():N}");
+        await File.WriteAllBytesAsync(secretPath, RandomNumberGenerator.GetBytes(32));
+        Process? process = null;
+        try
+        {
+            process = Process.Start(CreateStartInfo(httpPort, udpPort, secretPath))
+                ?? throw new InvalidOperationException("The production soak process did not start.");
+            Task<string> standardOutput = process.StandardOutput.ReadToEndAsync();
+            Task<string> standardError = process.StandardError.ReadToEndAsync();
+            await WaitForReadyAsync(httpPort, process, TimeSpan.FromSeconds(10));
+            process.Refresh();
+            int baselineHandles = process.HandleCount;
+            long baselineWorkingSet = process.WorkingSet64;
+            int peakHandles = baselineHandles;
+            byte[] invalidDatagram = RandomNumberGenerator.GetBytes(64);
+            IPEndPoint udpEndpoint = new(IPAddress.Loopback, udpPort);
+            using HttpClient client = new() { Timeout = TimeSpan.FromSeconds(1) };
+            using UdpClient udp = new();
+            Stopwatch soak = Stopwatch.StartNew();
+            int cycles = 0;
+            int accepted = 0;
+            int shed = 0;
+            while (soak.Elapsed < TimeSpan.FromSeconds(10))
+            {
+                using HttpResponseMessage response = await client.GetAsync(
+                    $"http://127.0.0.1:{httpPort}/health/live");
+                if (response.StatusCode == HttpStatusCode.OK)
+                {
+                    accepted++;
+                }
+                else
+                {
+                    Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode);
+                    shed++;
+                }
+
+                await udp.SendAsync(invalidDatagram, udpEndpoint);
+                cycles++;
+                if (cycles % 100 == 0)
+                {
+                    process.Refresh();
+                    peakHandles = Math.Max(peakHandles, process.HandleCount);
+                }
+            }
+
+            Assert.True(cycles >= 100, $"Transport soak completed only {cycles} cycles.");
+            Assert.True(accepted > 0, "Transport soak never admitted a health request.");
+            Assert.True(shed > 0, "Transport soak never exercised typed HTTP load shedding.");
+            await Task.Delay(TimeSpan.FromSeconds(2));
+            using (HttpResponseMessage recovered = await client.GetAsync(
+                       $"http://127.0.0.1:{httpPort}/health/live"))
+            {
+                Assert.Equal(HttpStatusCode.OK, recovered.StatusCode);
+            }
+
+            process.Refresh();
+            Assert.InRange(peakHandles, 0, baselineHandles + 32);
+            Assert.InRange(process.HandleCount, 0, baselineHandles + 16);
+            Assert.InRange(process.WorkingSet64, 0, baselineWorkingSet + 67_108_864);
+            AssertUdpPortIsBound(udpPort);
+
+            await SendSigtermAsync(process);
+            using CancellationTokenSource shutdownTimeout = new(TimeSpan.FromSeconds(6));
+            await process.WaitForExitAsync(shutdownTimeout.Token);
+            string output = await standardOutput;
+            string error = await standardError;
+            Assert.True(
+                process.ExitCode == 0,
+                $"Transport soak process failed. stdout: {output} stderr: {error}");
+            AssertTcpPortIsReleased(httpPort);
+            AssertUdpPortIsReleased(udpPort);
+        }
+        finally
+        {
+            await StopProcessTreeAsync(process);
+            File.Delete(secretPath);
+        }
+    }
+
     [Fact]
     public async Task DocumentedSmokeScriptReachesHttpAndAuthenticatedUdpFlow()
     {

tests/FinalFactory.Rendezvous.Tests/State/InMemoryEphemeralRendezvousStoreTests.cs

@@ -411,6 +411,92 @@ public sealed class InMemoryEphemeralRendezvousStoreTests
         Assert.True(fixture.Store.CreateJoinAttempt(attempt).Succeeded);
         Assert.Equal(StoreResultCode.CapacityExceeded, fixture.Store.CreateJoinAttempt(
             fixture.AttemptCommand(listing, "client-quota-2") with { ScopeAttemptLimit = 1 }).Code);
+
+        fixture.Clock.Advance(TimeSpan.FromSeconds(30));
+        Assert.True(fixture.Store.BindHostPresence(new(
+            first.Listing.HostPresenceHandle,
+            first.Listing.HostPresenceFingerprint,
+            EphemeralStateFixture.PublicEndpoint(42_101),
+            null)).Succeeded);
+        Assert.True(fixture.Store.CreateJoinAttempt(
+            fixture.AttemptCommand(listing, "client-quota-3") with { ScopeAttemptLimit = 1 }).Succeeded);
+    }
+
+    [Fact]
+    public void RemovingAListingReleasesItsConstantTimeOwnerQuota()
+    {
+        EphemeralStateFixture fixture = new();
+        CreateListingCommand first = fixture.ListingCommand(owner: "publisher-quota") with
+        {
+            OwnerListingLimit = 1,
+        };
+        CreateListingCommand second = fixture.ListingCommand(owner: "publisher-quota") with
+        {
+            OwnerListingLimit = 1,
+        };
+        StoreResult<StoredListing> created = fixture.Store.CreateListing(first);
+        Assert.True(created.Succeeded);
+        Assert.Equal(StoreResultCode.CapacityExceeded, fixture.Store.CreateListing(second).Code);
+
+        Assert.True(fixture.Store.DeleteListing(new(
+            first.Listing.ListingId,
+            first.Listing.LeaseId,
+            first.Listing.LeaseFingerprint,
+            first.Listing.OwnerSubject)).Succeeded);
+        Assert.True(fixture.Store.CreateListing(second).Succeeded);
+    }
+
+    [Fact]
+    public void RepeatedMutableDeadlineRefreshesKeepOneScheduledEntryPerKey()
+    {
+        EphemeralStateFixture fixture = new();
+        StoredListing listing = fixture.CreateVisibleListing(out _);
+
+        for (int iteration = 0; iteration < 10_000; iteration++)
+        {
+            fixture.Clock.Advance(TimeSpan.FromTicks(1));
+            StoreResult<StoredListing> renewed = fixture.Store.RenewLease(new(
+                listing.Definition.ListingId,
+                listing.Definition.LeaseId,
+                listing.Definition.LeaseFingerprint,
+                listing.Definition.OwnerSubject,
+                listing.Version));
+            Assert.True(renewed.Succeeded, $"Renewal {iteration} failed with {renewed.Code}.");
+            listing = renewed.Value!;
+            StoreResult<StoredListing> presence = fixture.Store.BindHostPresence(new(
+                listing.Definition.HostPresenceHandle,
+                listing.Definition.HostPresenceFingerprint,
+                EphemeralStateFixture.PublicEndpoint(42_200),
+                null));
+            Assert.True(presence.Succeeded, $"Presence {iteration} failed with {presence.Code}.");
+            StoreResult<int> revoked = fixture.Store.RevokePrincipal(
+                "repeated-revocation",
+                TimeSpan.FromMinutes(1));
+            Assert.True(revoked.Succeeded, $"Revocation {iteration} failed with {revoked.Code}.");
+        }
+
+        Assert.Equal(4, fixture.Store.ScheduledExpiryEntryCount);
+        fixture.Clock.Advance(TimeSpan.FromSeconds(19));
+        Assert.True(fixture.Store.GetListing(listing.Definition.ListingId, true).Succeeded);
+        Assert.Equal(4, fixture.Store.ScheduledExpiryEntryCount);
+    }
+
+    [Fact]
+    public void RepeatedPrincipalRevocationCanExtendButCannotShortenProtection()
+    {
+        EphemeralStateFixture fixture = new();
+        const string subject = "protected-publisher";
+
+        Assert.True(fixture.Store.RevokePrincipal(subject, TimeSpan.FromMinutes(10)).Succeeded);
+        fixture.Clock.Advance(TimeSpan.FromSeconds(1));
+        Assert.True(fixture.Store.RevokePrincipal(subject, TimeSpan.FromSeconds(1)).Succeeded);
+        fixture.Clock.Advance(TimeSpan.FromSeconds(2));
+        Assert.Equal(
+            StoreResultCode.Revoked,
+            fixture.Store.CreateListing(fixture.ListingCommand(owner: subject)).Code);
+
+        fixture.Clock.Advance(TimeSpan.FromSeconds(597));
+        Assert.True(fixture.Store.CreateListing(fixture.ListingCommand(owner: subject)).Succeeded);
     }
 
     [Fact]