feat(operations): add capacity and resilience gates (#18)
This commit is contained in:
@@ -36,6 +36,9 @@ jobs:
|
||||
- name: Test
|
||||
run: dotnet test Rendezvous.slnx --configuration Release --no-build
|
||||
|
||||
- name: Run quick capacity and resilience gate
|
||||
run: ./scripts/run-capacity-gate.sh
|
||||
|
||||
- name: Test privileged Linux namespace topology when available
|
||||
shell: bash
|
||||
run: |
|
||||
|
||||
@@ -7,5 +7,6 @@ TestResults/
|
||||
*.user
|
||||
*.userosscache
|
||||
deploy/compose/.smoke.env
|
||||
artifacts/
|
||||
deploy/compose/secrets/*
|
||||
!deploy/compose/secrets/.gitignore
|
||||
|
||||
@@ -83,9 +83,9 @@ Rendezvous is under active roadmap development. The versioned contracts,
|
||||
directory leases, authenticated join attempts, LiteNetLib mediator, caller-owned
|
||||
SDK coordination, typed connection outcomes, thin public-SDK diagnostic client,
|
||||
deterministic NAT topology harness, hostile-input controls,
|
||||
observability/operator surface, and secure single-active Linux deployment are
|
||||
implemented. Capacity, resilience, packaging, and final production-readiness
|
||||
gates remain in progress;
|
||||
observability/operator surface, secure single-active Linux deployment, and
|
||||
numeric capacity/resilience gates are implemented. Packaging, consumer pilots,
|
||||
and final production-readiness gates remain in progress;
|
||||
participating games must not treat the current repository as a finished production
|
||||
service until those gates land.
|
||||
|
||||
@@ -103,6 +103,9 @@ operator controls are defined in the
|
||||
The pinned non-root container, production topology, graceful drain, Linux
|
||||
hardening, smoke procedure, and recovery lifecycle are documented in
|
||||
[secure single-active Linux deployment](docs/deployment/linux.md).
|
||||
The numeric core-state candidate profile, public launch objectives, accelerated
|
||||
soak, resilience matrix, and single-active scaling decision are recorded in
|
||||
[capacity and resilience gates](docs/operations/capacity-and-resilience.md).
|
||||
The scriptable host/browser/join diagnostic and its stable automation contract are
|
||||
documented in the [TestClient integration guide](docs/integration/test-client.md).
|
||||
The always-on three-party scenarios, optional Linux namespace topology, and
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
<Project Path="src/FinalFactory.Rendezvous.TestClient/FinalFactory.Rendezvous.TestClient.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/tests/">
|
||||
<Project Path="tests/FinalFactory.Rendezvous.Capacity/FinalFactory.Rendezvous.Capacity.csproj" />
|
||||
<Project Path="tests/FinalFactory.Rendezvous.Tests/FinalFactory.Rendezvous.Tests.csproj" />
|
||||
</Folder>
|
||||
</Solution>
|
||||
|
||||
@@ -40,7 +40,8 @@ LockPersonality=true
|
||||
SystemCallArchitectures=native
|
||||
UMask=0077
|
||||
LimitNOFILE=4096
|
||||
MemoryMax=512M
|
||||
CPUQuota=200%
|
||||
MemoryMax=2G
|
||||
TasksMax=128
|
||||
|
||||
[Install]
|
||||
|
||||
@@ -129,9 +129,18 @@ until the owner records:
|
||||
- which games may enable anonymous unlisted player hosting;
|
||||
- deployment regions, data-processing jurisdiction, and approval of the stated
|
||||
30-day audit/13-month aggregate retention periods;
|
||||
- the per-game dedicated fallback endpoint policy;
|
||||
- the measured supported profile and whether the 99.5% single-active objective
|
||||
is sufficient or shared-state/high-availability work must be brought forward.
|
||||
- the per-game dedicated fallback endpoint policy.
|
||||
|
||||
Issue #18 measured and ratified the original 2-vCPU/2-GiB, 25,000-listing,
|
||||
10,000-attempt core-state candidate profile and retained the 99.5% single-active
|
||||
topology. It does not claim that core measurements prove public HTTP/UDP SLOs.
|
||||
The versioned evidence, RTO, failure domains, and explicit signals that trigger
|
||||
shared-state/high-availability work are recorded in the
|
||||
[capacity and resilience gate](../operations/capacity-and-resilience.md). The
|
||||
real-network canary in #23 must confirm that the proposed regional launch load
|
||||
fits this profile and validate the public SLOs; it may lower the launch cap but
|
||||
may not silently enable a
|
||||
second active instance.
|
||||
|
||||
These are configuration and launch decisions, not permission to weaken the
|
||||
tenant, replay, endpoint-verification, or secret-handling controls.
|
||||
|
||||
@@ -118,8 +118,11 @@ Keep the host clock synchronized with authenticated NTP. Credential and key
|
||||
windows use wall time; lease, timeout, drain, and rate-limit deadlines use a
|
||||
monotonic clock. Alert on clock synchronization loss before rotating keys.
|
||||
|
||||
Start with the Compose limits (one CPU, 512 MiB, 128 PIDs, 4096 descriptors),
|
||||
measure real traffic, then change the limits and the server budgets together.
|
||||
The checked-in Compose limits (one CPU and 512 MiB) are for its isolated smoke
|
||||
profile, not a production capacity claim. The measured core-state candidate
|
||||
uses 2 vCPU and 2 GiB with the same 128-PID/4096-descriptor ceilings; see
|
||||
the [capacity and resilience gate](../operations/capacity-and-resilience.md).
|
||||
Measure real traffic, then change resource limits and server budgets together.
|
||||
Memory pressure or CPU throttling must not extend orchestrator termination past
|
||||
`DrainDeadlineSeconds` plus five seconds.
|
||||
|
||||
@@ -157,8 +160,8 @@ sudo systemctl enable --now rendezvous.service
|
||||
Create the dedicated `rendezvous` user without a login shell. Keep
|
||||
`/opt/rendezvous` and `/etc/rendezvous` root-owned and non-writable by that user;
|
||||
install each required key with `root:rendezvous` ownership and mode `0440`. The
|
||||
unit applies the same resource, filesystem, privilege, network-family, and
|
||||
shutdown constraints as Compose.
|
||||
unit applies the measured 2-vCPU/2-GiB core-state candidate profile plus the
|
||||
same filesystem, privilege, network-family, and shutdown hardening as Compose.
|
||||
|
||||
## HTTP and UDP smoke
|
||||
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
# Capacity, resilience, and availability gate
|
||||
|
||||
Tracking: #18
|
||||
|
||||
This gate turns the v1 budgets in ADR 0003 into a repeatable release decision.
|
||||
It does not turn Rendezvous into a horizontally scalable service: v1 remains one
|
||||
active process with bounded in-memory state. A second process may be a cold
|
||||
standby, but it must not accept traffic until the first process has stopped and
|
||||
released the public HTTP and UDP endpoints.
|
||||
|
||||
## Launch envelope and approved core-state profile
|
||||
|
||||
The approved core-state profile is one Linux process limited to 2 vCPU and
|
||||
2 GiB RAM. Public HTTP/UDP numbers are launch objectives that require the #23
|
||||
real-network canary before they become a supported service claim:
|
||||
|
||||
| Dimension | Value | Evidence status |
|
||||
| --- | --- | --- |
|
||||
| Visible listings | 25,000 | Enforced and measured here |
|
||||
| Active join attempts | 10,000 | Enforced and measured here |
|
||||
| Core control path | 200 operations/second; p95 at most 200 ms | Measured here |
|
||||
| Core mediation path | 2,000 pairings/second; p95 at most 100 ms | Measured here |
|
||||
| Sustained HTTP demand | 200 requests/second | #23 launch objective; not yet a supported claim |
|
||||
| Sustained UDP demand | 2,000 datagrams/second | #23 launch objective; not yet a supported claim |
|
||||
| Public HTTP/UDP latency | p95 at most 200 ms / 100 ms | #23 launch objective; not yet a supported claim |
|
||||
| Capacity-phase average CPU / peak working memory | below 70% / below 1.5 GiB | Measured for the core candidate |
|
||||
| Valid in-profile monthly availability | 99.5%, excluding announced maintenance | Operational objective |
|
||||
| Process-ready RTO / host-visible recovery | 15 seconds / 90 seconds | 15 seconds automated; 90-second deployment drill required |
|
||||
|
||||
The proposed public-network mix is 20% registration/update, 30% lease-critical
|
||||
renew/delete, 30% browse, and 20% join authorization for HTTP. The UDP mix is
|
||||
60% authenticated host-presence refresh, 30% attempt contributions, and 10%
|
||||
invalid or duplicate traffic that must be dropped early. A deployment may use a
|
||||
lower per-game profile, but must not claim a higher one without new versioned
|
||||
evidence.
|
||||
|
||||
The capacity harness fills the complete state ceilings, then measures
|
||||
registration plus presence, renewal, a 100-item compatible browse, join
|
||||
issuance, simultaneous two-peer pairing, principal revocation, and telemetry.
|
||||
It applies 200/100 ms guardrails and minimum 200 control / 2,000 mediation
|
||||
operations per second to the core hot path. Those measurements deliberately
|
||||
exclude Kestrel, LiteNetLib, TLS, JSON, socket scheduling, and the documented
|
||||
mixed traffic shape. The #23 real-network canary must exercise those layers,
|
||||
rate-shape the mix, record errors and shedding, and meet the public objectives
|
||||
before launch; a core result is not a public-network latency or throughput claim.
|
||||
|
||||
## Reproduce the evidence
|
||||
|
||||
Every push runs the quick profile and the selected fault matrix:
|
||||
|
||||
```bash
|
||||
./scripts/run-capacity-gate.sh
|
||||
```
|
||||
|
||||
Run the production candidate on an otherwise idle Linux host and restrict the
|
||||
runtime to two logical CPUs. The default candidate includes a five-minute,
|
||||
high-intensity expiry soak; use 3,600 seconds for a release-candidate endurance
|
||||
run:
|
||||
|
||||
```bash
|
||||
export RENDEZVOUS_CAPACITY_PROFILE=candidate
|
||||
export RENDEZVOUS_CAPACITY_CPUSET=0,1
|
||||
export RENDEZVOUS_CAPACITY_OUTPUT="$PWD/artifacts/capacity/candidate.json"
|
||||
./scripts/run-capacity-gate.sh
|
||||
|
||||
# Release-candidate endurance override:
|
||||
dotnet run --project tests/FinalFactory.Rendezvous.Capacity \
|
||||
--configuration Release --no-build -- \
|
||||
--profile candidate --soak-seconds 3600 \
|
||||
--output artifacts/capacity/candidate-endurance.json
|
||||
```
|
||||
|
||||
The machine must have at least 2 GiB available to the process. For formal
|
||||
deployment evidence, run inside the same cgroup/container shape as production.
|
||||
The v2 JSON embeds the commit and tree state, command, image context, CPU model,
|
||||
kernel, affinity, cgroup quota/limit, collector mode, and workload seed. Supply
|
||||
`RENDEZVOUS_EVIDENCE_IMAGE_DIGEST` when running a release image. Do not compare
|
||||
results collected under a debugger,
|
||||
concurrent build, thermal throttling, or oversubscribed CI host.
|
||||
|
||||
The checked-in baseline is
|
||||
[`candidate-2cpu.json`](../evidence/capacity/v2/candidate-2cpu.json). It was
|
||||
produced on .NET 10.0.9/Linux x64 with CPU affinity restricted to two logical
|
||||
CPUs. It filled 25,000 listings and 10,000 attempts, peaked at about 162 MiB,
|
||||
and cleared all active/retained state. The five-minute baseline supersedes any
|
||||
earlier local probe when its timestamp and target duration differ.
|
||||
|
||||
## Soak and bounded-state interpretation
|
||||
|
||||
Each soak cycle creates a listing, repeatedly renews its lease and refreshes
|
||||
presence, creates a join attempt, replay marker, and retained outcome, checks
|
||||
that scheduled expiry entries remain proportional to live keys, then advances
|
||||
the injected monotonic clock beyond all
|
||||
deadlines, and verifies that listings, attempts, replay, idempotency, and outcome
|
||||
state return to zero. The candidate also measures managed-memory and process
|
||||
handle deltas after full collection. Failure is any retained state, more than
|
||||
64 MiB retained managed memory, more than eight retained handles, a working set
|
||||
above 1.5 GiB, an untyped capacity result, or failure to admit work after expiry.
|
||||
|
||||
This accelerated soak intentionally executes far more state lifecycle/cleanup
|
||||
events than wall-clock traffic would permit. It catches stale deadline-queue
|
||||
entries, cache growth, replay/idempotency retention, and cleanup cost. Because
|
||||
it does not open Kestrel/LiteNetLib connections, its process-handle delta is only
|
||||
a harness guard and is not evidence of transport stability by itself. The
|
||||
selected production-process gate adds a ten-second real HTTP/UDP transport soak,
|
||||
samples child-process handles and RSS, asserts bounded growth, then verifies a
|
||||
clean SIGTERM and socket release. #23 must extend that into the full rate-shaped
|
||||
multi-client canary while sampling queues, managed memory, and state
|
||||
cardinalities. A one-hour core override remains required before tagging a
|
||||
production release.
|
||||
|
||||
## Fault and recovery matrix
|
||||
|
||||
`run-capacity-gate.sh` runs these deterministic production paths before the
|
||||
numeric profile:
|
||||
|
||||
| Fault | Required result |
|
||||
| --- | --- |
|
||||
| HTTP/UDP overload and tracker exhaustion | Typed HTTP `429`/`CapacityExceeded`, silent UDP drop, bounded tracker keys, recovery after the window |
|
||||
| Optional traffic saturation | Lease-critical renew/update/delete capacity remains available |
|
||||
| Store/dependency unavailable | Readiness fails; new authorization returns typed `ServiceUnavailable`; liveness remains independent |
|
||||
| Graceful drain/SIGTERM | New work returns `Draining`; existing pairing may finish; process exits 0 and releases TCP/UDP before the deadline |
|
||||
| Hard restart | In-flight state is lost; SDK reports typed `ServiceUnavailable`; a host re-registers, rebinds presence, and becomes the only browser-visible replacement |
|
||||
| UDP listener bind/restart | Readiness stays false without the required listener; rebinding the advertised port restores native LiteNetLib pairing |
|
||||
| Wall-clock jump/skew | Monotonic lease/attempt authority is neither shortened nor extended; credential skew remains capped at 30 seconds |
|
||||
| Signing-secret rotation | New key signs, overlap verifies, retired/revoked key rejects, missing material fails startup |
|
||||
| Principal revocation | Listing, presence, attempts, and outcome paths are removed atomically within the latency budget |
|
||||
|
||||
No external database exists in v1, so “dependency/store failure” means the
|
||||
process-local atomic store is marked unavailable or a required listener/key is
|
||||
unready. The service fails closed rather than pretending a degraded writable
|
||||
mode exists.
|
||||
|
||||
## Bandwidth and amplification
|
||||
|
||||
- Accepted application datagrams are at most 1,200 bytes.
|
||||
- Malformed, oversized, unauthenticated, stale, replayed, wrong-role, and
|
||||
rate-limited traffic receives zero response bytes.
|
||||
- A completing authenticated contribution produces at most one introduction to
|
||||
each observed peer, and the combined response is at most 2.0 times that
|
||||
contribution's bytes.
|
||||
- The frozen-envelope and native LiteNetLib socket tests measure this on the real
|
||||
UDP listener; the hostile corpus and allocation gate exercise 10,000+ inputs
|
||||
without input-sized logs, tasks, or queues.
|
||||
|
||||
Bandwidth planning must therefore reserve ingress for the configured 2,000
|
||||
datagrams/second plus edge overhead and egress for a worst-case verified 2.0
|
||||
amplification. Actual successful pairs normally use two contributions and two
|
||||
introductions; normal gameplay leaves Rendezvous entirely.
|
||||
|
||||
## Availability decision
|
||||
|
||||
Single-active remains the v1 topology. The measured core profile proves bounded
|
||||
state and substantial core-path headroom, while public launch capacity remains
|
||||
conditional on #23. The service has a bounded stop-before-start restart path.
|
||||
Its failure domain is deliberately
|
||||
one process/node/public UDP endpoint: node, kernel, host network, DNS/TLS edge,
|
||||
secret configuration, or operator error can remove all readiness until the cold
|
||||
replacement owns the same source-preserving endpoint.
|
||||
|
||||
The 99.5% objective permits about 216 minutes of unannounced downtime in a
|
||||
30-day month. Operations must target process readiness within 15 seconds and
|
||||
host-visible re-registration within 90 seconds, page when no ready instance
|
||||
exists, and include detection plus recovery in the monthly budget. The current
|
||||
in-process test validates typed downtime, same-port HTTP restart, fresh
|
||||
registration, presence rebinding, and browser visibility in under five seconds;
|
||||
the production-process test separately validates graceful termination, TCP/UDP
|
||||
release, replacement startup on the same endpoints, UDP readiness, and the
|
||||
15-second process-ready RTO. Cold-standby activation policy and the 90-second
|
||||
operator-to-host recovery objective still require a deployment drill before
|
||||
release. Rollout and rollback use the
|
||||
deployment runbook's drain, stop, socket-release, start, smoke sequence; never
|
||||
overlap old and new active processes.
|
||||
|
||||
Bring shared TTL/CAS state and deterministic mediator routing forward before
|
||||
enabling two active instances if any of these occurs:
|
||||
|
||||
- one node cannot sustain 150% of the measured 30-day peak while meeting SLOs;
|
||||
- CPU stays above 70%, memory above 75%, attempt depth above 70%, or limiter
|
||||
drops/latency remain elevated after abusive traffic is excluded;
|
||||
- the availability target rises above 99.5% or planned maintenance must preserve
|
||||
listings; or
|
||||
- one region requires multiple simultaneously active mediator endpoints.
|
||||
|
||||
Rendezvous makes no multi-instance claim today, so a two-node atomic-pairing
|
||||
test is intentionally not applicable. It becomes a hard release gate with the
|
||||
shared-state/routing implementation; until then `SingleActiveInstance=false`
|
||||
fails production startup. Multi-region and relay remain separate evidence-driven
|
||||
decisions.
|
||||
@@ -115,7 +115,9 @@ permission denial is audited with actor and target fingerprints.
|
||||
Key revocation is process-local in the current single-instance store. Apply the
|
||||
same revocation to every instance, then replace configuration before restarting;
|
||||
a restart reconstructs the configured key ring. Principal revocation is bounded
|
||||
to ten minutes and removes that principal's active listings and attempts. Use
|
||||
to ten minutes and removes that principal's active listings and attempts. A
|
||||
repeat action may extend an active revocation but never shortens it; wait for its
|
||||
original deadline rather than treating a shorter repeat as an un-revoke. Use
|
||||
listing revocation for one targeted session and drain before planned shutdown.
|
||||
|
||||
## Audit retention and incident handling
|
||||
|
||||
Executable
+55
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PROFILE="${RENDEZVOUS_CAPACITY_PROFILE:-quick}"
|
||||
OUTPUT="${RENDEZVOUS_CAPACITY_OUTPUT:-$ROOT/artifacts/capacity/rendezvous-capacity-v2.json}"
|
||||
CPUSET="${RENDEZVOUS_CAPACITY_CPUSET:-}"
|
||||
PROJECT="$ROOT/tests/FinalFactory.Rendezvous.Capacity/FinalFactory.Rendezvous.Capacity.csproj"
|
||||
TESTS="$ROOT/tests/FinalFactory.Rendezvous.Tests/FinalFactory.Rendezvous.Tests.csproj"
|
||||
|
||||
if [[ "$PROFILE" != quick && "$PROFILE" != candidate ]]; then
|
||||
printf 'RENDEZVOUS_CAPACITY_PROFILE must be quick or candidate.\n' >&2
|
||||
exit 2
|
||||
fi
|
||||
command -v dotnet >/dev/null || {
|
||||
printf 'Missing required command: dotnet\n' >&2
|
||||
exit 2
|
||||
}
|
||||
if [[ -n "$CPUSET" ]]; then
|
||||
command -v taskset >/dev/null || {
|
||||
printf 'taskset is required when RENDEZVOUS_CAPACITY_CPUSET is set.\n' >&2
|
||||
exit 2
|
||||
}
|
||||
fi
|
||||
|
||||
cd "$ROOT"
|
||||
dotnet restore "$ROOT/Rendezvous.slnx" --locked-mode
|
||||
dotnet build "$ROOT/Rendezvous.slnx" --configuration Release --no-restore
|
||||
|
||||
filter='FullyQualifiedName~TrackerCapacityFailsClosedWithoutGrowingAndAWindowResetRecovers|FullyQualifiedName~OptionalTrafficCannotConsumeTheLeaseOperationReserve|FullyQualifiedName~ConcurrentAbusiveBurstStaysBoundedAndCannotBlockCriticalHttp|FullyQualifiedName~HttpOverloadIsTypedAndOversizedBodiesAreRejectedBeforeDispatch|FullyQualifiedName~WallClockMovementDoesNotExpireOrExtendLease|FullyQualifiedName~RepeatedMutableDeadlineRefreshesKeepOneScheduledEntryPerKey|FullyQualifiedName~RepeatedPrincipalRevocationCanExtendButCannotShortenProtection|FullyQualifiedName~RestartHasNewGenerationAndNoEphemeralState|FullyQualifiedName~RestartReturnsTypedUnavailabilityThenAllowsHostReregistration|FullyQualifiedName~DrainRejectsNewWorkAllowsInflightCompletionThenClearsState|FullyQualifiedName~UnavailableStoreFailsNewAuthorizationClosedAndErasesActiveState|FullyQualifiedName~KeyRotationHonorsOverlapAndRejectsRetiredKeys|FullyQualifiedName~OperatorSurfaceSeparatesAuthenticationConfirmsActionsAndRedactsInspection|FullyQualifiedName~SigtermDrainsThenReleasesHttpAndUdpSockets|FullyQualifiedName~ProductionTransportSoakKeepsHandlesMemoryAndSocketsBounded|FullyQualifiedName~NativeLiteNetLibRequestsIntroduceTheAuthorizedPair'
|
||||
dotnet test "$TESTS" --configuration Release --no-build --filter "$filter" \
|
||||
--logger 'console;verbosity=minimal'
|
||||
|
||||
mkdir -p "$(dirname "$OUTPUT")"
|
||||
arguments=(
|
||||
dotnet run --project "$PROJECT" --configuration Release --no-build --
|
||||
--profile "$PROFILE" --output "$OUTPUT"
|
||||
)
|
||||
export RENDEZVOUS_EVIDENCE_COMMIT="${RENDEZVOUS_EVIDENCE_COMMIT:-$(git rev-parse HEAD)}"
|
||||
if [[ -z "${RENDEZVOUS_EVIDENCE_TREE_STATE:-}" ]]; then
|
||||
if [[ -n "$(git status --porcelain)" ]]; then
|
||||
export RENDEZVOUS_EVIDENCE_TREE_STATE=dirty
|
||||
else
|
||||
export RENDEZVOUS_EVIDENCE_TREE_STATE=clean
|
||||
fi
|
||||
fi
|
||||
export RENDEZVOUS_EVIDENCE_CPUSET="${CPUSET:-unrestricted}"
|
||||
export RENDEZVOUS_EVIDENCE_COMMAND="RENDEZVOUS_CAPACITY_PROFILE=$PROFILE RENDEZVOUS_CAPACITY_CPUSET=${CPUSET:-unrestricted} ./scripts/run-capacity-gate.sh"
|
||||
if [[ -n "$CPUSET" ]]; then
|
||||
taskset -c "$CPUSET" "${arguments[@]}"
|
||||
else
|
||||
"${arguments[@]}"
|
||||
fi
|
||||
|
||||
printf 'Capacity and resilience gate passed; evidence: %s\n' "$OUTPUT"
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("FinalFactory.Rendezvous.Tests")]
|
||||
[assembly: InternalsVisibleTo("FinalFactory.Rendezvous.Capacity")]
|
||||
|
||||
@@ -11,16 +11,29 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
||||
private readonly DateTimeOffset _wallOrigin;
|
||||
private readonly TimeSpan _monotonicOrigin;
|
||||
private readonly Dictionary<SessionListingId, ListingEntry> _listings = [];
|
||||
private readonly Dictionary<string, int> _listingCountsByOwner = new(StringComparer.Ordinal);
|
||||
private readonly PriorityQueue<DeadlineEntry<SessionListingId>, long> _listingExpiries = new();
|
||||
private readonly HashSet<SessionListingId> _scheduledListingExpiries = [];
|
||||
private readonly Dictionary<LeaseId, SessionListingId> _leases = [];
|
||||
private readonly Dictionary<MediationHandle, SessionListingId> _presenceHandles = [];
|
||||
private readonly Dictionary<MediationHandle, PresenceEntry> _presence = [];
|
||||
private readonly PriorityQueue<DeadlineEntry<MediationHandle>, long> _presenceExpiries = new();
|
||||
private readonly HashSet<MediationHandle> _scheduledPresenceExpiries = [];
|
||||
private readonly Dictionary<JoinAttemptId, AttemptEntry> _attempts = [];
|
||||
private readonly PriorityQueue<AttemptExpiry, long> _attemptExpiries = new();
|
||||
private readonly Dictionary<TenantScope, int> _attemptCountsByScope = [];
|
||||
private readonly Dictionary<SessionListingId, HashSet<JoinAttemptId>> _attemptsByListing = [];
|
||||
private readonly PriorityQueue<DeadlineEntry<JoinAttemptId>, long> _attemptExpiries = new();
|
||||
private readonly Dictionary<JoinAttemptId, OutcomeReportEntry> _outcomeReports = [];
|
||||
private readonly Dictionary<SessionListingId, HashSet<JoinAttemptId>> _outcomesByListing = [];
|
||||
private readonly PriorityQueue<DeadlineEntry<JoinAttemptId>, long> _outcomeExpiries = new();
|
||||
private readonly Dictionary<MediationHandle, JoinAttemptId> _attemptHandles = [];
|
||||
private readonly Dictionary<string, IdempotencyEntry> _idempotency = new(StringComparer.Ordinal);
|
||||
private readonly PriorityQueue<DeadlineEntry<string>, long> _idempotencyExpiries = new();
|
||||
private readonly Dictionary<string, TimeSpan> _replay = new(StringComparer.Ordinal);
|
||||
private readonly PriorityQueue<DeadlineEntry<string>, long> _replayExpiries = new();
|
||||
private readonly Dictionary<string, TimeSpan> _revocations = new(StringComparer.Ordinal);
|
||||
private readonly PriorityQueue<DeadlineEntry<string>, long> _revocationExpiries = new();
|
||||
private readonly HashSet<string> _scheduledRevocationExpiries = new(StringComparer.Ordinal);
|
||||
private TimeSpan? _drainDeadline;
|
||||
private TimeSpan _nextUdpMaintenance;
|
||||
private long _maintenanceSweepCount;
|
||||
@@ -47,6 +60,22 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
||||
|
||||
public Guid InstanceId { get; }
|
||||
internal long MaintenanceSweepCount => Interlocked.Read(ref _maintenanceSweepCount);
|
||||
internal int ScheduledExpiryEntryCount
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _listingExpiries.Count
|
||||
+ _presenceExpiries.Count
|
||||
+ _attemptExpiries.Count
|
||||
+ _outcomeExpiries.Count
|
||||
+ _idempotencyExpiries.Count
|
||||
+ _replayExpiries.Count
|
||||
+ _revocationExpiries.Count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public bool IsAvailable
|
||||
{
|
||||
@@ -161,10 +190,8 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
||||
|
||||
if (_listings.Count >= _options.MaxListings
|
||||
|| _idempotency.Count >= _options.MaxIdempotencyEntries
|
||||
|| _listings.Values.Count(entry => string.Equals(
|
||||
entry.Definition.OwnerSubject,
|
||||
command.Listing.OwnerSubject,
|
||||
StringComparison.Ordinal)) >= command.OwnerListingLimit)
|
||||
|| _listingCountsByOwner.GetValueOrDefault(command.Listing.OwnerSubject)
|
||||
>= command.OwnerListingLimit)
|
||||
{
|
||||
return new(StoreResultCode.CapacityExceeded);
|
||||
}
|
||||
@@ -183,12 +210,21 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
||||
WallDeadline(now, _options.LeaseLifetime),
|
||||
version: 1);
|
||||
_listings.Add(frozen.ListingId, entry);
|
||||
ScheduleMutableDeadline(
|
||||
_listingExpiries,
|
||||
_scheduledListingExpiries,
|
||||
frozen.ListingId,
|
||||
entry.LeaseDeadline);
|
||||
_listingCountsByOwner[frozen.OwnerSubject] =
|
||||
_listingCountsByOwner.GetValueOrDefault(frozen.OwnerSubject) + 1;
|
||||
_leases.Add(frozen.LeaseId, frozen.ListingId);
|
||||
_presenceHandles.Add(frozen.HostPresenceHandle, frozen.ListingId);
|
||||
_idempotency.Add(idempotencyKey, new(
|
||||
IdempotencyEntry idempotency = new(
|
||||
command.RequestFingerprint,
|
||||
frozen.ListingId,
|
||||
now + _options.IdempotencyLifetime));
|
||||
now + _options.IdempotencyLifetime);
|
||||
_idempotency.Add(idempotencyKey, idempotency);
|
||||
EnqueueDeadline(_idempotencyExpiries, idempotencyKey, idempotency.Deadline);
|
||||
return new(StoreResultCode.Success, Snapshot(entry));
|
||||
}, cancellationToken);
|
||||
|
||||
@@ -346,10 +382,20 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
||||
return new(StoreResultCode.CapacityExceeded);
|
||||
}
|
||||
|
||||
_presence[command.Handle] = new(
|
||||
bool isNewPresence = !_presence.ContainsKey(command.Handle);
|
||||
PresenceEntry presence = new(
|
||||
command.PublicEndpoint,
|
||||
command.LocalEndpoint,
|
||||
now + _options.PresenceLifetime);
|
||||
_presence[command.Handle] = presence;
|
||||
if (isNewPresence)
|
||||
{
|
||||
ScheduleMutableDeadline(
|
||||
_presenceExpiries,
|
||||
_scheduledPresenceExpiries,
|
||||
command.Handle,
|
||||
presence.Deadline);
|
||||
}
|
||||
return new(StoreResultCode.Success, Snapshot(entry));
|
||||
}, cancellationToken, eagerCleanup: false);
|
||||
|
||||
@@ -444,8 +490,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
||||
if (_attempts.Count >= _options.MaxJoinAttempts
|
||||
|| _outcomeReports.Count >= _options.MaxOutcomeReports
|
||||
|| _idempotency.Count >= _options.MaxIdempotencyEntries
|
||||
|| _attempts.Values.Count(entry => entry.Command.Scope == command.Scope)
|
||||
>= command.ScopeAttemptLimit)
|
||||
|| _attemptCountsByScope.GetValueOrDefault(command.Scope) >= command.ScopeAttemptLimit)
|
||||
{
|
||||
return new(StoreResultCode.CapacityExceeded);
|
||||
}
|
||||
@@ -461,19 +506,25 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
||||
now + _options.JoinAttemptLifetime,
|
||||
WallDeadline(now, _options.JoinAttemptLifetime));
|
||||
_attempts.Add(command.AttemptId, attempt);
|
||||
_attemptExpiries.Enqueue(
|
||||
new AttemptExpiry(command.AttemptId, attempt.Deadline),
|
||||
attempt.Deadline.Ticks);
|
||||
_outcomeReports.Add(command.AttemptId, new(
|
||||
AddToIndex(_attemptsByListing, command.ListingId, command.AttemptId);
|
||||
_attemptCountsByScope[command.Scope] =
|
||||
_attemptCountsByScope.GetValueOrDefault(command.Scope) + 1;
|
||||
EnqueueDeadline(_attemptExpiries, command.AttemptId, attempt.Deadline);
|
||||
OutcomeReportEntry outcome = new(
|
||||
command.ListingId,
|
||||
command.ClientSubject,
|
||||
command.ClientCapabilityFingerprint,
|
||||
now + _options.JoinAttemptLifetime + _options.IdempotencyLifetime));
|
||||
now + _options.JoinAttemptLifetime + _options.IdempotencyLifetime);
|
||||
_outcomeReports.Add(command.AttemptId, outcome);
|
||||
AddToIndex(_outcomesByListing, command.ListingId, command.AttemptId);
|
||||
EnqueueDeadline(_outcomeExpiries, command.AttemptId, outcome.Deadline);
|
||||
_attemptHandles.Add(command.MediationHandle, command.AttemptId);
|
||||
_idempotency.Add(idempotencyKey, new(
|
||||
IdempotencyEntry idempotency = new(
|
||||
command.RequestFingerprint,
|
||||
command.AttemptId,
|
||||
now + _options.IdempotencyLifetime));
|
||||
now + _options.IdempotencyLifetime);
|
||||
_idempotency.Add(idempotencyKey, idempotency);
|
||||
EnqueueDeadline(_idempotencyExpiries, idempotencyKey, idempotency.Deadline);
|
||||
return new(StoreResultCode.Success, Snapshot(attempt));
|
||||
}, cancellationToken);
|
||||
|
||||
@@ -757,7 +808,9 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
||||
throw new ArgumentOutOfRangeException(nameof(consumption), "Replay lifetime exceeds the configured ceiling.");
|
||||
}
|
||||
|
||||
_replay.Add(key, now + lifetime);
|
||||
TimeSpan deadline = now + lifetime;
|
||||
_replay.Add(key, deadline);
|
||||
EnqueueDeadline(_replayExpiries, key, deadline);
|
||||
return new(StoreResultCode.Success, true);
|
||||
}, cancellationToken);
|
||||
|
||||
@@ -794,7 +847,24 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
||||
+ _presence.Count
|
||||
+ _attempts.Count
|
||||
+ _outcomeReports.Count;
|
||||
_revocations[subject] = now + lifetime;
|
||||
TimeSpan deadline = now + lifetime;
|
||||
bool isNewRevocation = !_revocations.TryGetValue(subject, out TimeSpan existingDeadline);
|
||||
if (!isNewRevocation && existingDeadline > deadline)
|
||||
{
|
||||
// A repeated operator action may extend protection but cannot silently
|
||||
// shorten an already-authoritative security revocation.
|
||||
deadline = existingDeadline;
|
||||
}
|
||||
|
||||
_revocations[subject] = deadline;
|
||||
if (isNewRevocation)
|
||||
{
|
||||
ScheduleMutableDeadline(
|
||||
_revocationExpiries,
|
||||
_scheduledRevocationExpiries,
|
||||
subject,
|
||||
deadline);
|
||||
}
|
||||
SessionListingId[] listings = _listings
|
||||
.Where(item => string.Equals(item.Value.Definition.OwnerSubject, subject, StringComparison.Ordinal))
|
||||
.Select(static item => item.Key)
|
||||
@@ -818,7 +888,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
||||
}
|
||||
foreach (JoinAttemptId attemptId in outcomeReports)
|
||||
{
|
||||
_outcomeReports.Remove(attemptId);
|
||||
RemoveOutcome(attemptId);
|
||||
}
|
||||
|
||||
int activeResourcesAfter = _listings.Count
|
||||
@@ -911,63 +981,42 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
||||
ClearActiveState();
|
||||
}
|
||||
|
||||
_expiryChurn += RemoveExpired(_revocations, now);
|
||||
_expiryChurn += RemoveExpired(_replay, now);
|
||||
string[] expiredIdempotency = _idempotency
|
||||
.Where(item => item.Value.Deadline <= now)
|
||||
.Select(static item => item.Key)
|
||||
.ToArray();
|
||||
_expiryChurn += expiredIdempotency.Length;
|
||||
foreach (string key in expiredIdempotency)
|
||||
{
|
||||
_idempotency.Remove(key);
|
||||
}
|
||||
|
||||
MediationHandle[] expiredPresence = _presence
|
||||
.Where(item => item.Value.Deadline <= now)
|
||||
.Select(static item => item.Key)
|
||||
.ToArray();
|
||||
_expiryChurn += expiredPresence.Length;
|
||||
foreach (MediationHandle handle in expiredPresence)
|
||||
{
|
||||
_presence.Remove(handle);
|
||||
}
|
||||
|
||||
_expiryChurn += RemoveExpiredMutableDeadlines(
|
||||
_revocations,
|
||||
_revocationExpiries,
|
||||
_scheduledRevocationExpiries,
|
||||
now);
|
||||
_expiryChurn += RemoveExpiredDeadlines(_replay, _replayExpiries, now);
|
||||
_expiryChurn += RemoveExpiredIdempotency(now);
|
||||
_expiryChurn += RemoveExpiredPresence(now);
|
||||
_expiryChurn += RemoveExpiredAttempts(now);
|
||||
|
||||
JoinAttemptId[] expiredOutcomes = _outcomeReports
|
||||
.Where(item => item.Value.Deadline <= now)
|
||||
.Select(static item => item.Key)
|
||||
.ToArray();
|
||||
_expiryChurn += expiredOutcomes.Length;
|
||||
foreach (JoinAttemptId attemptId in expiredOutcomes)
|
||||
{
|
||||
_outcomeReports.Remove(attemptId);
|
||||
}
|
||||
|
||||
SessionListingId[] expiredListings = _listings
|
||||
.Where(item => item.Value.LeaseDeadline <= now)
|
||||
.Select(static item => item.Key)
|
||||
.ToArray();
|
||||
_expiryChurn += expiredListings.Length;
|
||||
foreach (SessionListingId listingId in expiredListings)
|
||||
{
|
||||
RemoveListing(listingId);
|
||||
}
|
||||
_expiryChurn += RemoveExpiredOutcomes(now);
|
||||
_expiryChurn += RemoveExpiredListings(now);
|
||||
}
|
||||
|
||||
private void ClearActiveState()
|
||||
{
|
||||
_listings.Clear();
|
||||
_listingCountsByOwner.Clear();
|
||||
_listingExpiries.Clear();
|
||||
_scheduledListingExpiries.Clear();
|
||||
_leases.Clear();
|
||||
_presenceHandles.Clear();
|
||||
_presence.Clear();
|
||||
_presenceExpiries.Clear();
|
||||
_scheduledPresenceExpiries.Clear();
|
||||
_attempts.Clear();
|
||||
_attemptCountsByScope.Clear();
|
||||
_attemptsByListing.Clear();
|
||||
_attemptExpiries.Clear();
|
||||
_outcomeReports.Clear();
|
||||
_outcomesByListing.Clear();
|
||||
_outcomeExpiries.Clear();
|
||||
_attemptHandles.Clear();
|
||||
_idempotency.Clear();
|
||||
_idempotencyExpiries.Clear();
|
||||
_replay.Clear();
|
||||
_replayExpiries.Clear();
|
||||
}
|
||||
|
||||
private void RemoveListing(SessionListingId listingId)
|
||||
@@ -978,23 +1027,23 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
||||
}
|
||||
|
||||
_leases.Remove(listing.Definition.LeaseId);
|
||||
DecrementCount(_listingCountsByOwner, listing.Definition.OwnerSubject);
|
||||
_presenceHandles.Remove(listing.Definition.HostPresenceHandle);
|
||||
_presence.Remove(listing.Definition.HostPresenceHandle);
|
||||
foreach (JoinAttemptId attemptId in _attempts
|
||||
.Where(item => item.Value.Command.ListingId == listingId)
|
||||
.Select(static item => item.Key)
|
||||
.ToArray())
|
||||
if (_attemptsByListing.TryGetValue(listingId, out HashSet<JoinAttemptId>? attempts))
|
||||
{
|
||||
foreach (JoinAttemptId attemptId in attempts.ToArray())
|
||||
{
|
||||
RemoveAttempt(attemptId);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
foreach (JoinAttemptId attemptId in _outcomeReports
|
||||
.Where(item => item.Value.ListingId == listingId)
|
||||
.Select(static item => item.Key)
|
||||
.ToArray())
|
||||
if (_outcomesByListing.TryGetValue(listingId, out HashSet<JoinAttemptId>? outcomes))
|
||||
{
|
||||
_outcomeReports.Remove(attemptId);
|
||||
foreach (JoinAttemptId attemptId in outcomes.ToArray())
|
||||
{
|
||||
RemoveOutcome(attemptId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1003,20 +1052,75 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
||||
if (_attempts.Remove(attemptId, out AttemptEntry? attempt))
|
||||
{
|
||||
_attemptHandles.Remove(attempt.Command.MediationHandle);
|
||||
DecrementCount(_attemptCountsByScope, attempt.Command.Scope);
|
||||
RemoveFromIndex(_attemptsByListing, attempt.Command.ListingId, attemptId);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveOutcome(JoinAttemptId attemptId)
|
||||
{
|
||||
if (_outcomeReports.Remove(attemptId, out OutcomeReportEntry? outcome))
|
||||
{
|
||||
RemoveFromIndex(_outcomesByListing, outcome.ListingId, attemptId);
|
||||
}
|
||||
}
|
||||
|
||||
private static void AddToIndex<TKey>(
|
||||
Dictionary<TKey, HashSet<JoinAttemptId>> index,
|
||||
TKey key,
|
||||
JoinAttemptId attemptId)
|
||||
where TKey : notnull
|
||||
{
|
||||
if (!index.TryGetValue(key, out HashSet<JoinAttemptId>? values))
|
||||
{
|
||||
values = [];
|
||||
index.Add(key, values);
|
||||
}
|
||||
|
||||
values.Add(attemptId);
|
||||
}
|
||||
|
||||
private static void RemoveFromIndex<TKey>(
|
||||
Dictionary<TKey, HashSet<JoinAttemptId>> index,
|
||||
TKey key,
|
||||
JoinAttemptId attemptId)
|
||||
where TKey : notnull
|
||||
{
|
||||
if (index.TryGetValue(key, out HashSet<JoinAttemptId>? values)
|
||||
&& values.Remove(attemptId)
|
||||
&& values.Count == 0)
|
||||
{
|
||||
index.Remove(key);
|
||||
}
|
||||
}
|
||||
|
||||
private static void DecrementCount<TKey>(Dictionary<TKey, int> counts, TKey key)
|
||||
where TKey : notnull
|
||||
{
|
||||
int remaining = counts[key] - 1;
|
||||
if (remaining == 0)
|
||||
{
|
||||
counts.Remove(key);
|
||||
}
|
||||
else
|
||||
{
|
||||
counts[key] = remaining;
|
||||
}
|
||||
}
|
||||
|
||||
private int RemoveExpiredAttempts(TimeSpan now)
|
||||
{
|
||||
int removed = 0;
|
||||
while (_attemptExpiries.TryPeek(out AttemptExpiry candidate, out long deadlineTicks)
|
||||
while (_attemptExpiries.TryPeek(
|
||||
out DeadlineEntry<JoinAttemptId> candidate,
|
||||
out long deadlineTicks)
|
||||
&& deadlineTicks <= now.Ticks)
|
||||
{
|
||||
_attemptExpiries.Dequeue();
|
||||
if (_attempts.TryGetValue(candidate.AttemptId, out AttemptEntry? current)
|
||||
if (_attempts.TryGetValue(candidate.Key, out AttemptEntry? current)
|
||||
&& current.Deadline == candidate.Deadline)
|
||||
{
|
||||
RemoveAttempt(candidate.AttemptId);
|
||||
RemoveAttempt(candidate.Key);
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
@@ -1024,6 +1128,184 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
||||
return removed;
|
||||
}
|
||||
|
||||
private int RemoveExpiredListings(TimeSpan now)
|
||||
{
|
||||
int removed = 0;
|
||||
while (_listingExpiries.TryPeek(
|
||||
out DeadlineEntry<SessionListingId> candidate,
|
||||
out long deadlineTicks)
|
||||
&& deadlineTicks <= now.Ticks)
|
||||
{
|
||||
_listingExpiries.Dequeue();
|
||||
_scheduledListingExpiries.Remove(candidate.Key);
|
||||
if (!_listings.TryGetValue(candidate.Key, out ListingEntry? current))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (current.LeaseDeadline > now)
|
||||
{
|
||||
ScheduleMutableDeadline(
|
||||
_listingExpiries,
|
||||
_scheduledListingExpiries,
|
||||
candidate.Key,
|
||||
current.LeaseDeadline);
|
||||
}
|
||||
else
|
||||
{
|
||||
RemoveListing(candidate.Key);
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
private int RemoveExpiredPresence(TimeSpan now)
|
||||
{
|
||||
int removed = 0;
|
||||
while (_presenceExpiries.TryPeek(
|
||||
out DeadlineEntry<MediationHandle> candidate,
|
||||
out long deadlineTicks)
|
||||
&& deadlineTicks <= now.Ticks)
|
||||
{
|
||||
_presenceExpiries.Dequeue();
|
||||
_scheduledPresenceExpiries.Remove(candidate.Key);
|
||||
if (!_presence.TryGetValue(candidate.Key, out PresenceEntry? current))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (current.Deadline > now)
|
||||
{
|
||||
ScheduleMutableDeadline(
|
||||
_presenceExpiries,
|
||||
_scheduledPresenceExpiries,
|
||||
candidate.Key,
|
||||
current.Deadline);
|
||||
}
|
||||
else if (_presence.Remove(candidate.Key))
|
||||
{
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
private int RemoveExpiredOutcomes(TimeSpan now)
|
||||
{
|
||||
int removed = 0;
|
||||
while (_outcomeExpiries.TryPeek(
|
||||
out DeadlineEntry<JoinAttemptId> candidate,
|
||||
out long deadlineTicks)
|
||||
&& deadlineTicks <= now.Ticks)
|
||||
{
|
||||
_outcomeExpiries.Dequeue();
|
||||
if (_outcomeReports.TryGetValue(candidate.Key, out OutcomeReportEntry? current)
|
||||
&& current.Deadline == candidate.Deadline)
|
||||
{
|
||||
RemoveOutcome(candidate.Key);
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
private int RemoveExpiredIdempotency(TimeSpan now)
|
||||
{
|
||||
int removed = 0;
|
||||
while (_idempotencyExpiries.TryPeek(
|
||||
out DeadlineEntry<string> candidate,
|
||||
out long deadlineTicks)
|
||||
&& deadlineTicks <= now.Ticks)
|
||||
{
|
||||
_idempotencyExpiries.Dequeue();
|
||||
if (_idempotency.TryGetValue(candidate.Key, out IdempotencyEntry? current)
|
||||
&& current.Deadline == candidate.Deadline
|
||||
&& _idempotency.Remove(candidate.Key))
|
||||
{
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
private static int RemoveExpiredDeadlines<TKey>(
|
||||
Dictionary<TKey, TimeSpan> entries,
|
||||
PriorityQueue<DeadlineEntry<TKey>, long> expiries,
|
||||
TimeSpan now)
|
||||
where TKey : notnull
|
||||
{
|
||||
int removed = 0;
|
||||
while (expiries.TryPeek(out DeadlineEntry<TKey> candidate, out long deadlineTicks)
|
||||
&& deadlineTicks <= now.Ticks)
|
||||
{
|
||||
expiries.Dequeue();
|
||||
if (entries.TryGetValue(candidate.Key, out TimeSpan current)
|
||||
&& current == candidate.Deadline
|
||||
&& entries.Remove(candidate.Key))
|
||||
{
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
private static int RemoveExpiredMutableDeadlines<TKey>(
|
||||
Dictionary<TKey, TimeSpan> entries,
|
||||
PriorityQueue<DeadlineEntry<TKey>, long> expiries,
|
||||
HashSet<TKey> scheduled,
|
||||
TimeSpan now)
|
||||
where TKey : notnull
|
||||
{
|
||||
int removed = 0;
|
||||
while (expiries.TryPeek(out DeadlineEntry<TKey> candidate, out long deadlineTicks)
|
||||
&& deadlineTicks <= now.Ticks)
|
||||
{
|
||||
expiries.Dequeue();
|
||||
scheduled.Remove(candidate.Key);
|
||||
if (!entries.TryGetValue(candidate.Key, out TimeSpan current))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (current > now)
|
||||
{
|
||||
ScheduleMutableDeadline(expiries, scheduled, candidate.Key, current);
|
||||
}
|
||||
else if (entries.Remove(candidate.Key))
|
||||
{
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
private static void ScheduleMutableDeadline<TKey>(
|
||||
PriorityQueue<DeadlineEntry<TKey>, long> expiries,
|
||||
HashSet<TKey> scheduled,
|
||||
TKey key,
|
||||
TimeSpan deadline)
|
||||
where TKey : notnull
|
||||
{
|
||||
if (scheduled.Add(key))
|
||||
{
|
||||
EnqueueDeadline(expiries, key, deadline);
|
||||
}
|
||||
}
|
||||
|
||||
private static void EnqueueDeadline<TKey>(
|
||||
PriorityQueue<DeadlineEntry<TKey>, long> expiries,
|
||||
TKey key,
|
||||
TimeSpan deadline)
|
||||
where TKey : notnull =>
|
||||
expiries.Enqueue(new(key, deadline), deadline.Ticks);
|
||||
|
||||
private bool HandleExists(MediationHandle handle) =>
|
||||
_presenceHandles.ContainsKey(handle) || _attemptHandles.ContainsKey(handle);
|
||||
|
||||
@@ -1063,20 +1345,6 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
||||
IsCancelled = entry.IsCancelled,
|
||||
};
|
||||
|
||||
private static int RemoveExpired(Dictionary<string, TimeSpan> entries, TimeSpan now)
|
||||
{
|
||||
string[] expired = entries
|
||||
.Where(item => item.Value <= now)
|
||||
.Select(static item => item.Key)
|
||||
.ToArray();
|
||||
foreach (string key in expired)
|
||||
{
|
||||
entries.Remove(key);
|
||||
}
|
||||
|
||||
return expired.Length;
|
||||
}
|
||||
|
||||
private static void ValidateListing(ListingDefinition listing)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(listing);
|
||||
@@ -1219,7 +1487,8 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
||||
public bool IsCancelled { get; set; }
|
||||
}
|
||||
|
||||
private readonly record struct AttemptExpiry(JoinAttemptId AttemptId, TimeSpan Deadline);
|
||||
private readonly record struct DeadlineEntry<TKey>(TKey Key, TimeSpan Deadline)
|
||||
where TKey : notnull;
|
||||
|
||||
private sealed class OutcomeReportEntry(
|
||||
SessionListingId listingId,
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
namespace FinalFactory.Rendezvous.Capacity;
|
||||
|
||||
internal sealed record CapacityOptions
|
||||
{
|
||||
public required string Profile { get; init; }
|
||||
public required int Listings { get; init; }
|
||||
public required int Attempts { get; init; }
|
||||
public required int Samples { get; init; }
|
||||
public required int SoakCycles { get; init; }
|
||||
public required int SoakSeconds { get; init; }
|
||||
public string? OutputPath { get; init; }
|
||||
|
||||
public static CapacityOptions Parse(string[] args)
|
||||
{
|
||||
Dictionary<string, string> values = ParseArguments(args);
|
||||
string profile = values.GetValueOrDefault("--profile") ?? "quick";
|
||||
(int listings, int attempts, int samples, int soakCycles, int soakSeconds) = profile switch
|
||||
{
|
||||
"quick" => (1_000, 500, 100, 20, 0),
|
||||
"candidate" => (25_000, 10_000, 1_000, 1_000, 300),
|
||||
_ => throw new ArgumentException("--profile must be 'quick' or 'candidate'."),
|
||||
};
|
||||
|
||||
return new()
|
||||
{
|
||||
Profile = profile,
|
||||
Listings = ParsePositive(values.GetValueOrDefault("--listings"), listings, "--listings"),
|
||||
Attempts = ParsePositive(values.GetValueOrDefault("--attempts"), attempts, "--attempts"),
|
||||
Samples = ParsePositive(values.GetValueOrDefault("--samples"), samples, "--samples"),
|
||||
SoakCycles = ParsePositive(
|
||||
values.GetValueOrDefault("--soak-cycles"),
|
||||
soakCycles,
|
||||
"--soak-cycles"),
|
||||
SoakSeconds = ParseNonNegative(
|
||||
values.GetValueOrDefault("--soak-seconds"),
|
||||
soakSeconds,
|
||||
"--soak-seconds"),
|
||||
OutputPath = values.GetValueOrDefault("--output"),
|
||||
};
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> ParseArguments(string[] args)
|
||||
{
|
||||
HashSet<string> allowed =
|
||||
[
|
||||
"--profile",
|
||||
"--listings",
|
||||
"--attempts",
|
||||
"--samples",
|
||||
"--soak-cycles",
|
||||
"--soak-seconds",
|
||||
"--output",
|
||||
];
|
||||
Dictionary<string, string> values = new(StringComparer.Ordinal);
|
||||
for (int index = 0; index < args.Length; index += 2)
|
||||
{
|
||||
string option = args[index];
|
||||
if (!allowed.Contains(option))
|
||||
{
|
||||
throw new ArgumentException($"Unknown option: {option}.");
|
||||
}
|
||||
|
||||
if (index == args.Length - 1
|
||||
|| args[index + 1].StartsWith("--", StringComparison.Ordinal))
|
||||
{
|
||||
throw new ArgumentException($"{option} requires a value.");
|
||||
}
|
||||
|
||||
if (!values.TryAdd(option, args[index + 1]))
|
||||
{
|
||||
throw new ArgumentException($"{option} may be supplied only once.");
|
||||
}
|
||||
}
|
||||
|
||||
return values;
|
||||
}
|
||||
|
||||
private static int ParsePositive(string? value, int fallback, string option) =>
|
||||
value is null
|
||||
? fallback
|
||||
: int.TryParse(value, out int parsed) && parsed > 0
|
||||
? parsed
|
||||
: throw new ArgumentException($"{option} must be a positive integer.");
|
||||
|
||||
private static int ParseNonNegative(string? value, int fallback, string option) =>
|
||||
value is null
|
||||
? fallback
|
||||
: int.TryParse(value, out int parsed) && parsed >= 0
|
||||
? parsed
|
||||
: throw new ArgumentException($"{option} must be a non-negative integer.");
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
namespace FinalFactory.Rendezvous.Capacity;
|
||||
|
||||
internal sealed record CapacityReport
|
||||
{
|
||||
public required int SchemaVersion { get; init; }
|
||||
public required string EvidenceVersion { get; init; }
|
||||
public required DateTimeOffset GeneratedAt { get; init; }
|
||||
public required string Profile { get; init; }
|
||||
public required RuntimeEvidence Runtime { get; init; }
|
||||
public required CapacityTargets Targets { get; init; }
|
||||
public required IReadOnlyList<CapacityMeasurement> Measurements { get; init; }
|
||||
public required StateEvidence State { get; init; }
|
||||
public required IReadOnlyList<string> Failures { get; init; }
|
||||
public required bool Passed { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record RuntimeEvidence(
|
||||
string Framework,
|
||||
string OperatingSystem,
|
||||
string Kernel,
|
||||
string Architecture,
|
||||
string CpuModel,
|
||||
int ProcessorCount,
|
||||
string CpuAffinity,
|
||||
string CpuQuota,
|
||||
string MemoryLimit,
|
||||
string GarbageCollector,
|
||||
string CommitSha,
|
||||
string TreeState,
|
||||
string Command,
|
||||
string ImageDigest,
|
||||
string WorkloadSeed,
|
||||
double CapacityPhaseAverageCpuPercent,
|
||||
long PeakWorkingSetBytes,
|
||||
long ManagedBytesAfterCleanup);
|
||||
|
||||
internal sealed record CapacityTargets(
|
||||
int VisibleListings,
|
||||
int ActiveJoinAttempts,
|
||||
int CoreControlOperationsPerSecond,
|
||||
int CoreMediationOperationsPerSecond,
|
||||
double CoreControlP95Milliseconds,
|
||||
double CoreMediationP95Milliseconds,
|
||||
double MaximumAverageCpuPercent,
|
||||
long MaximumWorkingSetBytes,
|
||||
int SoakCycles,
|
||||
int SoakDurationSeconds);
|
||||
|
||||
internal sealed record CapacityMeasurement(
|
||||
string Operation,
|
||||
int Samples,
|
||||
double P50Milliseconds,
|
||||
double P95Milliseconds,
|
||||
double P99Milliseconds,
|
||||
double OperationsPerSecond,
|
||||
double MinimumOperationsPerSecond,
|
||||
double BudgetMilliseconds,
|
||||
bool Passed);
|
||||
|
||||
internal sealed record StateEvidence(
|
||||
int PeakListings,
|
||||
int PeakAttempts,
|
||||
int PeakReplayMarkers,
|
||||
int FinalListings,
|
||||
int FinalAttempts,
|
||||
int FinalReplayMarkers,
|
||||
long ExpiryChurn,
|
||||
long MaintenanceSweeps,
|
||||
int SoakCyclesCompleted,
|
||||
double SoakDurationSeconds,
|
||||
int SoakPeakScheduledExpiryEntries,
|
||||
long SoakManagedGrowthBytes,
|
||||
int SoakHandleGrowth,
|
||||
bool RestartStartedEmpty,
|
||||
bool OverloadWasTyped,
|
||||
bool RecoverySucceeded);
|
||||
@@ -0,0 +1,580 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.Runtime;
|
||||
using System.Runtime.InteropServices;
|
||||
using FinalFactory.Rendezvous.Contracts;
|
||||
using FinalFactory.Rendezvous.Server.Observability;
|
||||
using FinalFactory.Rendezvous.Server.State;
|
||||
|
||||
namespace FinalFactory.Rendezvous.Capacity;
|
||||
|
||||
internal static class CapacityRunner
|
||||
{
|
||||
private static readonly TenantScope Scope = new(new("space-game"), new("production"));
|
||||
private const uint ProtocolVersion = 1;
|
||||
|
||||
public static Task<CapacityReport> RunAsync(CapacityOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
Process process = Process.GetCurrentProcess();
|
||||
TimeSpan cpuBefore = process.TotalProcessorTime;
|
||||
Stopwatch capacityPhaseTime = Stopwatch.StartNew();
|
||||
List<string> failures = [];
|
||||
List<CapacityMeasurement> measurements = [];
|
||||
ManualClock clock = new();
|
||||
InMemoryEphemeralRendezvousStore store = CreateStore(options, clock);
|
||||
List<StoredListing> listings = new(options.Listings);
|
||||
List<CreateJoinAttemptCommand> attempts = new(options.Attempts);
|
||||
int registrationSamples = Math.Min(options.Samples, options.Listings);
|
||||
int attemptSamples = Math.Min(options.Samples, options.Attempts);
|
||||
|
||||
for (int index = 0; index < options.Listings - registrationSamples; index++)
|
||||
{
|
||||
listings.Add(CreateVisibleListing(store, index));
|
||||
}
|
||||
|
||||
measurements.Add(Measure(
|
||||
"registration-and-presence",
|
||||
registrationSamples,
|
||||
budgetMilliseconds: 200,
|
||||
minimumOperationsPerSecond: 200,
|
||||
index => listings.Add(CreateVisibleListing(
|
||||
store,
|
||||
options.Listings - registrationSamples + index))));
|
||||
|
||||
measurements.Add(Measure(
|
||||
"lease-renewal",
|
||||
registrationSamples,
|
||||
budgetMilliseconds: 200,
|
||||
minimumOperationsPerSecond: 200,
|
||||
index =>
|
||||
{
|
||||
StoredListing listing = listings[index];
|
||||
StoreResult<StoredListing> renewed = store.RenewLease(new(
|
||||
listing.Definition.ListingId,
|
||||
listing.Definition.LeaseId,
|
||||
listing.Definition.LeaseFingerprint,
|
||||
listing.Definition.OwnerSubject,
|
||||
listing.Version));
|
||||
RequireSuccess(renewed, "renewal");
|
||||
listings[index] = renewed.Value!;
|
||||
}));
|
||||
|
||||
int browseSamples = Math.Min(options.Samples, 250);
|
||||
measurements.Add(Measure(
|
||||
"visible-session-browse",
|
||||
browseSamples,
|
||||
budgetMilliseconds: 200,
|
||||
minimumOperationsPerSecond: 200,
|
||||
_ => RequireSuccess(
|
||||
store.BrowseVisibleListings(new(
|
||||
Scope,
|
||||
ProtocolVersion,
|
||||
new RegionId("eu-central"),
|
||||
ContractLimits.BrowserPageMaxItems,
|
||||
ExcludeFull: true)),
|
||||
"browse")));
|
||||
|
||||
for (int index = 0; index < options.Attempts - attemptSamples; index++)
|
||||
{
|
||||
CreateJoinAttemptCommand command = CreateAttempt(index, listings[index % listings.Count]);
|
||||
RequireSuccess(store.CreateJoinAttempt(command), "join issuance");
|
||||
attempts.Add(command);
|
||||
}
|
||||
|
||||
measurements.Add(Measure(
|
||||
"join-attempt-issuance",
|
||||
attemptSamples,
|
||||
budgetMilliseconds: 200,
|
||||
minimumOperationsPerSecond: 200,
|
||||
index =>
|
||||
{
|
||||
int sequence = options.Attempts - attemptSamples + index;
|
||||
CreateJoinAttemptCommand command = CreateAttempt(
|
||||
sequence,
|
||||
listings[sequence % listings.Count]);
|
||||
RequireSuccess(store.CreateJoinAttempt(command), "join issuance");
|
||||
attempts.Add(command);
|
||||
}));
|
||||
|
||||
int punchSamples = Math.Min(options.Samples, attempts.Count);
|
||||
measurements.Add(MeasureConcurrentPunch(store, attempts, punchSamples));
|
||||
EphemeralStoreSnapshot peak = store.GetSnapshot();
|
||||
|
||||
StoreResult<StoredJoinAttempt> overloaded = store.CreateJoinAttempt(
|
||||
CreateAttempt(options.Attempts + 1, listings[^1]));
|
||||
bool overloadWasTyped = peak.ActiveJoinAttempts == options.Attempts
|
||||
&& overloaded.Code == StoreResultCode.CapacityExceeded;
|
||||
if (!overloadWasTyped)
|
||||
{
|
||||
failures.Add(
|
||||
$"Expected typed CapacityExceeded at {options.Attempts} active attempts, "
|
||||
+ $"observed count={peak.ActiveJoinAttempts}, result={overloaded.Code}.");
|
||||
}
|
||||
|
||||
int revocationSamples = Math.Min(Math.Max(1, options.Samples / 20), listings.Count / 2);
|
||||
measurements.Add(Measure(
|
||||
"principal-revocation",
|
||||
revocationSamples,
|
||||
budgetMilliseconds: 200,
|
||||
minimumOperationsPerSecond: 50,
|
||||
index => RequireSuccess(
|
||||
store.RevokePrincipal(
|
||||
listings[index].Definition.OwnerSubject,
|
||||
TimeSpan.FromMinutes(1)),
|
||||
"principal revocation")));
|
||||
|
||||
using (RendezvousTelemetry telemetry = new(store))
|
||||
{
|
||||
measurements.Add(Measure(
|
||||
"telemetry-recording",
|
||||
Math.Max(100, options.Samples),
|
||||
budgetMilliseconds: 1,
|
||||
minimumOperationsPerSecond: 10_000,
|
||||
_ =>
|
||||
{
|
||||
telemetry.RecordHttp("browse", 200, 1);
|
||||
telemetry.RecordUdp("contribution", "accepted", 1);
|
||||
telemetry.RecordPairingLatency(2);
|
||||
}));
|
||||
}
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(61));
|
||||
EphemeralStoreSnapshot? afterCoincidentExpiry = null;
|
||||
measurements.Add(Measure(
|
||||
"coincident-listing-attempt-expiry",
|
||||
samples: 1,
|
||||
budgetMilliseconds: 200,
|
||||
minimumOperationsPerSecond: 0,
|
||||
_ => afterCoincidentExpiry = store.GetSnapshot()));
|
||||
if (afterCoincidentExpiry!.ActiveJoinAttempts != 0
|
||||
|| afterCoincidentExpiry.ActiveListings != 0)
|
||||
{
|
||||
failures.Add("Coincident 60-second cleanup retained expired listings or join attempts.");
|
||||
}
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(90));
|
||||
_ = store.GetSnapshot();
|
||||
StoredListing recoveryListing = CreateVisibleListing(store, options.Listings + 1);
|
||||
StoreResult<StoredJoinAttempt> recovered = store.CreateJoinAttempt(
|
||||
CreateAttempt(options.Attempts + 2, recoveryListing));
|
||||
bool recoverySucceeded = recovered.Succeeded;
|
||||
if (!recoverySucceeded)
|
||||
{
|
||||
failures.Add($"Store did not recover after attempt expiry: {recovered.Code}.");
|
||||
}
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(151));
|
||||
EphemeralStoreSnapshot final = store.GetSnapshot();
|
||||
if (final.ActiveListings != 0
|
||||
|| final.ActiveJoinAttempts != 0
|
||||
|| final.ReplayMarkers != 0
|
||||
|| final.IdempotencyEntries != 0
|
||||
|| final.RetainedOutcomeReports != 0)
|
||||
{
|
||||
failures.Add("Expiry cleanup left active or retained state after every configured deadline.");
|
||||
}
|
||||
|
||||
capacityPhaseTime.Stop();
|
||||
process.Refresh();
|
||||
double capacityPhaseCpuPercent = 100
|
||||
* (process.TotalProcessorTime - cpuBefore).TotalSeconds
|
||||
/ Math.Max(capacityPhaseTime.Elapsed.TotalSeconds * Environment.ProcessorCount, 0.000_001);
|
||||
if (options.Profile == "candidate" && capacityPhaseCpuPercent > 70)
|
||||
{
|
||||
failures.Add(
|
||||
$"Capacity-phase CPU {capacityPhaseCpuPercent:F1}% exceeded the 70% candidate budget.");
|
||||
}
|
||||
|
||||
SoakEvidence soak = RunAcceleratedSoak(options, failures);
|
||||
ManualClock restartClock = new();
|
||||
EphemeralStoreSnapshot restarted = CreateStore(options, restartClock).GetSnapshot();
|
||||
bool restartStartedEmpty = restarted.ActiveListings == 0
|
||||
&& restarted.ActiveJoinAttempts == 0
|
||||
&& restarted.ReplayMarkers == 0;
|
||||
if (!restartStartedEmpty)
|
||||
{
|
||||
failures.Add("A restarted store did not begin empty.");
|
||||
}
|
||||
|
||||
foreach (CapacityMeasurement measurement in measurements.Where(static item => !item.Passed))
|
||||
{
|
||||
failures.Add(
|
||||
$"{measurement.Operation} missed its budget: p95={measurement.P95Milliseconds:F3} ms, "
|
||||
+ $"rate={measurement.OperationsPerSecond:F1}/s.");
|
||||
}
|
||||
|
||||
GC.Collect();
|
||||
GC.WaitForPendingFinalizers();
|
||||
GC.Collect();
|
||||
process.Refresh();
|
||||
long managedAfterCleanup = GC.GetTotalMemory(forceFullCollection: true);
|
||||
long memoryBudget = 1_610_612_736;
|
||||
if (process.PeakWorkingSet64 > memoryBudget)
|
||||
{
|
||||
failures.Add(
|
||||
$"Peak working set {process.PeakWorkingSet64} exceeded the 1.5 GiB profile budget.");
|
||||
}
|
||||
|
||||
if (options.Profile == "candidate" && Environment.ProcessorCount != 2)
|
||||
{
|
||||
failures.Add(
|
||||
$"Candidate evidence must expose exactly two CPUs; runtime exposed "
|
||||
+ $"{Environment.ProcessorCount}.");
|
||||
}
|
||||
|
||||
CapacityReport report = new()
|
||||
{
|
||||
SchemaVersion = 2,
|
||||
EvidenceVersion = "v2",
|
||||
GeneratedAt = DateTimeOffset.UtcNow,
|
||||
Profile = options.Profile,
|
||||
Runtime = new(
|
||||
RuntimeInformation.FrameworkDescription,
|
||||
RuntimeInformation.OSDescription,
|
||||
Environment.OSVersion.VersionString,
|
||||
RuntimeInformation.ProcessArchitecture.ToString(),
|
||||
ReadCpuModel(),
|
||||
Environment.ProcessorCount,
|
||||
Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_CPUSET") ?? "unrestricted",
|
||||
ReadCgroupValue("/sys/fs/cgroup/cpu.max"),
|
||||
ReadCgroupValue("/sys/fs/cgroup/memory.max"),
|
||||
GCSettings.IsServerGC ? "server" : "workstation",
|
||||
Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_COMMIT") ?? "unrecorded",
|
||||
Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_TREE_STATE") ?? "unrecorded",
|
||||
Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_COMMAND") ?? "unrecorded",
|
||||
Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_IMAGE_DIGEST") ?? "not-containerized",
|
||||
"fixed-sequences-random-identifiers",
|
||||
capacityPhaseCpuPercent,
|
||||
process.PeakWorkingSet64,
|
||||
managedAfterCleanup),
|
||||
Targets = new(
|
||||
options.Listings,
|
||||
options.Attempts,
|
||||
200,
|
||||
2_000,
|
||||
200,
|
||||
100,
|
||||
70,
|
||||
memoryBudget,
|
||||
options.SoakCycles,
|
||||
options.SoakSeconds),
|
||||
Measurements = measurements,
|
||||
State = new(
|
||||
peak.ActiveListings,
|
||||
peak.ActiveJoinAttempts,
|
||||
peak.ReplayMarkers,
|
||||
final.ActiveListings,
|
||||
final.ActiveJoinAttempts,
|
||||
final.ReplayMarkers,
|
||||
final.ExpiryChurn,
|
||||
final.MaintenanceSweeps,
|
||||
soak.Cycles,
|
||||
soak.Duration.TotalSeconds,
|
||||
soak.PeakScheduledExpiryEntries,
|
||||
soak.ManagedGrowthBytes,
|
||||
soak.HandleGrowth,
|
||||
restartStartedEmpty,
|
||||
overloadWasTyped,
|
||||
recoverySucceeded),
|
||||
Failures = failures,
|
||||
Passed = failures.Count == 0,
|
||||
};
|
||||
return Task.FromResult(report);
|
||||
}
|
||||
|
||||
private static InMemoryEphemeralRendezvousStore CreateStore(
|
||||
CapacityOptions options,
|
||||
ManualClock clock) => new(
|
||||
new EphemeralStoreOptions
|
||||
{
|
||||
MaxListings = options.Listings,
|
||||
MaxPresenceBindings = options.Listings,
|
||||
MaxJoinAttempts = options.Attempts,
|
||||
MaxOutcomeReports = options.Attempts,
|
||||
MaxIdempotencyEntries = options.Listings + options.Attempts + 1,
|
||||
},
|
||||
clock,
|
||||
clock);
|
||||
|
||||
private static StoredListing CreateVisibleListing(
|
||||
InMemoryEphemeralRendezvousStore store,
|
||||
int sequence)
|
||||
{
|
||||
string owner = $"publisher-{sequence}";
|
||||
SecretFingerprint leaseFingerprint = new($"lease-{sequence}");
|
||||
SecretFingerprint presenceFingerprint = new($"presence-{sequence}");
|
||||
ListingDefinition definition = new()
|
||||
{
|
||||
ListingId = new(Guid.NewGuid()),
|
||||
LeaseId = new(Guid.NewGuid()),
|
||||
Scope = Scope,
|
||||
OwnerSubject = owner,
|
||||
RegionId = new("eu-central"),
|
||||
ProtocolVersion = ProtocolVersion,
|
||||
BuildVersion = "1.0.0",
|
||||
DisplayName = $"Capacity host {sequence}",
|
||||
Visibility = ListingVisibility.Public,
|
||||
TrustMode = PublisherTrustMode.ManagedDedicated,
|
||||
CurrentPlayers = 1,
|
||||
MaximumPlayers = 8,
|
||||
Metadata = new Dictionary<string, string>(StringComparer.Ordinal),
|
||||
LeaseFingerprint = leaseFingerprint,
|
||||
HostPresenceHandle = new(Guid.NewGuid()),
|
||||
HostPresenceFingerprint = presenceFingerprint,
|
||||
CapabilityDerivationSalt = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
||||
};
|
||||
StoreResult<StoredListing> created = store.CreateListing(new(
|
||||
$"register-{sequence}",
|
||||
$"register-request-{sequence}",
|
||||
definition));
|
||||
RequireSuccess(created, "registration");
|
||||
StoreResult<StoredListing> bound = store.BindHostPresence(new(
|
||||
definition.HostPresenceHandle,
|
||||
presenceFingerprint,
|
||||
PublicEndpoint(10_000 + sequence % 50_000),
|
||||
null));
|
||||
RequireSuccess(bound, "host presence");
|
||||
return bound.Value!;
|
||||
}
|
||||
|
||||
private static CreateJoinAttemptCommand CreateAttempt(int sequence, StoredListing listing) => new()
|
||||
{
|
||||
IdempotencyOwner = $"client-{sequence}",
|
||||
IdempotencyKey = $"join-{sequence}",
|
||||
RequestFingerprint = $"join-request-{sequence}",
|
||||
ClientSubject = $"client-{sequence}",
|
||||
AttemptId = new(Guid.NewGuid()),
|
||||
MediationHandle = new(Guid.NewGuid()),
|
||||
Scope = Scope,
|
||||
ListingId = listing.Definition.ListingId,
|
||||
ProtocolVersion = ProtocolVersion,
|
||||
HostCapabilityFingerprint = new($"host-capability-{sequence}"),
|
||||
ClientCapabilityFingerprint = new($"client-capability-{sequence}"),
|
||||
ConnectionTicketFingerprint = new($"ticket-{sequence}"),
|
||||
CapabilityDerivationSalt = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
||||
};
|
||||
|
||||
private static CapacityMeasurement MeasureConcurrentPunch(
|
||||
InMemoryEphemeralRendezvousStore store,
|
||||
List<CreateJoinAttemptCommand> attempts,
|
||||
int samples)
|
||||
{
|
||||
ConcurrentBag<double> latencies = [];
|
||||
Stopwatch total = Stopwatch.StartNew();
|
||||
Parallel.ForEach(
|
||||
Enumerable.Range(0, samples),
|
||||
new ParallelOptions { MaxDegreeOfParallelism = Math.Min(64, Environment.ProcessorCount * 4) },
|
||||
index =>
|
||||
{
|
||||
CreateJoinAttemptCommand attempt = attempts[index];
|
||||
Stopwatch elapsed = Stopwatch.StartNew();
|
||||
RequireSuccess(store.BindAttemptEndpoint(new(
|
||||
attempt.MediationHandle,
|
||||
AttemptPeerRole.Host,
|
||||
attempt.HostCapabilityFingerprint,
|
||||
PublicEndpoint(20_000 + index % 20_000),
|
||||
null)), "host punch");
|
||||
RequireSuccess(store.BindAttemptEndpoint(new(
|
||||
attempt.MediationHandle,
|
||||
AttemptPeerRole.Client,
|
||||
attempt.ClientCapabilityFingerprint,
|
||||
PublicEndpoint(40_000 + index % 20_000),
|
||||
null)), "client punch");
|
||||
RequireSuccess(store.ConsumeIntroduction(attempt.MediationHandle), "introduction");
|
||||
latencies.Add(elapsed.Elapsed.TotalMilliseconds);
|
||||
});
|
||||
total.Stop();
|
||||
return BuildMeasurement(
|
||||
"simultaneous-punch-pairing",
|
||||
latencies.ToArray(),
|
||||
total.Elapsed,
|
||||
budgetMilliseconds: 100,
|
||||
minimumOperationsPerSecond: 2_000);
|
||||
}
|
||||
|
||||
private static CapacityMeasurement Measure(
|
||||
string operation,
|
||||
int samples,
|
||||
double budgetMilliseconds,
|
||||
double minimumOperationsPerSecond,
|
||||
Action<int> action)
|
||||
{
|
||||
double[] latencies = new double[samples];
|
||||
Stopwatch total = Stopwatch.StartNew();
|
||||
for (int index = 0; index < samples; index++)
|
||||
{
|
||||
long started = Stopwatch.GetTimestamp();
|
||||
action(index);
|
||||
latencies[index] = Stopwatch.GetElapsedTime(started).TotalMilliseconds;
|
||||
}
|
||||
|
||||
total.Stop();
|
||||
return BuildMeasurement(
|
||||
operation,
|
||||
latencies,
|
||||
total.Elapsed,
|
||||
budgetMilliseconds,
|
||||
minimumOperationsPerSecond);
|
||||
}
|
||||
|
||||
private static CapacityMeasurement BuildMeasurement(
|
||||
string operation,
|
||||
double[] latencies,
|
||||
TimeSpan elapsed,
|
||||
double budgetMilliseconds,
|
||||
double minimumOperationsPerSecond)
|
||||
{
|
||||
Array.Sort(latencies);
|
||||
double operationsPerSecond = latencies.Length / Math.Max(elapsed.TotalSeconds, 0.000_001);
|
||||
double p95 = Percentile(latencies, 0.95);
|
||||
return new(
|
||||
operation,
|
||||
latencies.Length,
|
||||
Percentile(latencies, 0.50),
|
||||
p95,
|
||||
Percentile(latencies, 0.99),
|
||||
operationsPerSecond,
|
||||
minimumOperationsPerSecond,
|
||||
budgetMilliseconds,
|
||||
p95 <= budgetMilliseconds && operationsPerSecond >= minimumOperationsPerSecond);
|
||||
}
|
||||
|
||||
private static double Percentile(double[] sorted, double percentile)
|
||||
{
|
||||
int index = Math.Clamp((int)Math.Ceiling(sorted.Length * percentile) - 1, 0, sorted.Length - 1);
|
||||
return sorted[index];
|
||||
}
|
||||
|
||||
private static SoakEvidence RunAcceleratedSoak(
|
||||
CapacityOptions options,
|
||||
List<string> failures)
|
||||
{
|
||||
GC.Collect();
|
||||
GC.WaitForPendingFinalizers();
|
||||
GC.Collect();
|
||||
long managedBefore = GC.GetTotalMemory(forceFullCollection: true);
|
||||
int handlesBefore = Process.GetCurrentProcess().HandleCount;
|
||||
ManualClock clock = new();
|
||||
CapacityOptions soakOptions = options with { Listings = 100, Attempts = 100 };
|
||||
InMemoryEphemeralRendezvousStore store = CreateStore(soakOptions, clock);
|
||||
Stopwatch elapsed = Stopwatch.StartNew();
|
||||
int cycle = 0;
|
||||
int peakScheduledExpiryEntries = 0;
|
||||
while (cycle < options.SoakCycles
|
||||
|| elapsed.Elapsed < TimeSpan.FromSeconds(options.SoakSeconds))
|
||||
{
|
||||
StoredListing listing = CreateVisibleListing(store, cycle);
|
||||
for (int refresh = 0; refresh < 10; refresh++)
|
||||
{
|
||||
clock.Advance(TimeSpan.FromTicks(1));
|
||||
listing = RequireSuccess(store.RenewLease(new(
|
||||
listing.Definition.ListingId,
|
||||
listing.Definition.LeaseId,
|
||||
listing.Definition.LeaseFingerprint,
|
||||
listing.Definition.OwnerSubject,
|
||||
listing.Version)), "soak lease refresh");
|
||||
listing = RequireSuccess(store.BindHostPresence(new(
|
||||
listing.Definition.HostPresenceHandle,
|
||||
listing.Definition.HostPresenceFingerprint,
|
||||
PublicEndpoint(10_000 + cycle % 50_000),
|
||||
null)), "soak presence refresh");
|
||||
}
|
||||
|
||||
CreateJoinAttemptCommand attempt = CreateAttempt(cycle, listing);
|
||||
RequireSuccess(store.CreateJoinAttempt(attempt), "soak join issuance");
|
||||
RequireSuccess(store.ConsumeReplay(new("capacity-soak", $"replay-{cycle}")), "soak replay");
|
||||
peakScheduledExpiryEntries = Math.Max(
|
||||
peakScheduledExpiryEntries,
|
||||
store.ScheduledExpiryEntryCount);
|
||||
if (store.ScheduledExpiryEntryCount > 7)
|
||||
{
|
||||
failures.Add(
|
||||
$"Mutable deadline refresh grew the expiry queue to "
|
||||
+ $"{store.ScheduledExpiryEntryCount} entries for one lifecycle.");
|
||||
break;
|
||||
}
|
||||
|
||||
clock.Advance(TimeSpan.FromSeconds(151));
|
||||
EphemeralStoreSnapshot snapshot = store.GetSnapshot();
|
||||
if (snapshot.ActiveListings != 0
|
||||
|| snapshot.ActiveJoinAttempts != 0
|
||||
|| snapshot.ReplayMarkers != 0
|
||||
|| snapshot.IdempotencyEntries != 0
|
||||
|| snapshot.RetainedOutcomeReports != 0)
|
||||
{
|
||||
failures.Add($"Accelerated soak retained state after cycle {cycle}.");
|
||||
break;
|
||||
}
|
||||
|
||||
cycle++;
|
||||
}
|
||||
|
||||
elapsed.Stop();
|
||||
GC.Collect();
|
||||
GC.WaitForPendingFinalizers();
|
||||
GC.Collect();
|
||||
long managedGrowth = GC.GetTotalMemory(forceFullCollection: true) - managedBefore;
|
||||
int handleGrowth = Process.GetCurrentProcess().HandleCount - handlesBefore;
|
||||
if (managedGrowth > 67_108_864)
|
||||
{
|
||||
failures.Add($"Soak retained {managedGrowth} managed bytes; budget is 64 MiB.");
|
||||
}
|
||||
|
||||
if (handleGrowth > 8)
|
||||
{
|
||||
failures.Add($"Soak retained {handleGrowth} process handles; budget is 8.");
|
||||
}
|
||||
|
||||
return new(cycle, elapsed.Elapsed, peakScheduledExpiryEntries, managedGrowth, handleGrowth);
|
||||
}
|
||||
|
||||
private static ObservedEndpoint PublicEndpoint(int port) =>
|
||||
new(AddressFamilyKind.Ipv4, "203.0.113.10", port);
|
||||
|
||||
private static T RequireSuccess<T>(StoreResult<T> result, string operation)
|
||||
{
|
||||
if (!result.Succeeded)
|
||||
{
|
||||
throw new InvalidOperationException($"{operation} failed with {result.Code}.");
|
||||
}
|
||||
|
||||
return result.Value!;
|
||||
}
|
||||
|
||||
private static string ReadCpuModel()
|
||||
{
|
||||
const string cpuInfoPath = "/proc/cpuinfo";
|
||||
if (!File.Exists(cpuInfoPath))
|
||||
{
|
||||
return "unavailable";
|
||||
}
|
||||
|
||||
string? model = File.ReadLines(cpuInfoPath)
|
||||
.FirstOrDefault(static line => line.StartsWith("model name", StringComparison.Ordinal));
|
||||
int separator = model?.IndexOf(':') ?? -1;
|
||||
return separator >= 0 ? model![(separator + 1)..].Trim() : "unavailable";
|
||||
}
|
||||
|
||||
private static string ReadCgroupValue(string path) =>
|
||||
File.Exists(path) ? File.ReadAllText(path).Trim() : "not-enforced";
|
||||
|
||||
private sealed class ManualClock : IWallClock, IMonotonicClock
|
||||
{
|
||||
public DateTimeOffset UtcNow { get; private set; } = DateTimeOffset.UtcNow;
|
||||
public TimeSpan Elapsed { get; private set; }
|
||||
|
||||
public void Advance(TimeSpan duration)
|
||||
{
|
||||
UtcNow += duration;
|
||||
Elapsed += duration;
|
||||
}
|
||||
}
|
||||
|
||||
private readonly record struct SoakEvidence(
|
||||
int Cycles,
|
||||
TimeSpan Duration,
|
||||
int PeakScheduledExpiryEntries,
|
||||
long ManagedGrowthBytes,
|
||||
int HandleGrowth);
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<AssemblyName>FinalFactory.Rendezvous.Capacity</AssemblyName>
|
||||
<RootNamespace>FinalFactory.Rendezvous.Capacity</RootNamespace>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../src/FinalFactory.Rendezvous.Contracts/FinalFactory.Rendezvous.Contracts.csproj" />
|
||||
<ProjectReference Include="../../src/FinalFactory.Rendezvous.Server/FinalFactory.Rendezvous.Server.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,37 @@
|
||||
using System.Text.Json;
|
||||
|
||||
namespace FinalFactory.Rendezvous.Capacity;
|
||||
|
||||
internal static class Program
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = true,
|
||||
};
|
||||
|
||||
public static async Task<int> Main(string[] args)
|
||||
{
|
||||
CapacityOptions options;
|
||||
try
|
||||
{
|
||||
options = CapacityOptions.Parse(args);
|
||||
}
|
||||
catch (ArgumentException exception)
|
||||
{
|
||||
Console.Error.WriteLine(exception.Message);
|
||||
return 2;
|
||||
}
|
||||
|
||||
CapacityReport report = await CapacityRunner.RunAsync(options).ConfigureAwait(false);
|
||||
string json = JsonSerializer.Serialize(report, JsonOptions);
|
||||
Console.WriteLine(json);
|
||||
if (options.OutputPath is not null)
|
||||
{
|
||||
string fullPath = Path.GetFullPath(options.OutputPath);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!);
|
||||
await File.WriteAllTextAsync(fullPath, json + Environment.NewLine).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return report.Passed ? 0 : 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"version": 2,
|
||||
"dependencies": {
|
||||
"net10.0": {
|
||||
"finalfactory.rendezvous.contracts": {
|
||||
"type": "Project"
|
||||
},
|
||||
"finalfactory.rendezvous.server": {
|
||||
"type": "Project",
|
||||
"dependencies": {
|
||||
"FinalFactory.Rendezvous.Contracts": "[1.0.0, )",
|
||||
"LiteNetLib": "[2.1.4, )",
|
||||
"Microsoft.AspNetCore.OpenApi": "[10.0.9, )"
|
||||
}
|
||||
},
|
||||
"LiteNetLib": {
|
||||
"type": "CentralTransitive",
|
||||
"requested": "[2.1.4, )",
|
||||
"resolved": "2.1.4",
|
||||
"contentHash": "KWlxvMw3Urpqj9joD96LRiK+LC62pQNs/zkXRJc+rHnxgkGp+vV703xzDrxRmv+V1YhCFfIGzs5nrVWtREIlyA=="
|
||||
},
|
||||
"Microsoft.AspNetCore.OpenApi": {
|
||||
"type": "CentralTransitive",
|
||||
"requested": "[10.0.9, )",
|
||||
"resolved": "10.0.9",
|
||||
"contentHash": "1ihb8FO9cGgEK1/m3CTtT/SfnynwmiZib0W2pcDVj3KSWk/Sca4VOXEtaptKQc582zpFrzTFiwkGRCglt6H+WQ==",
|
||||
"dependencies": {
|
||||
"Microsoft.OpenApi": "2.0.0"
|
||||
}
|
||||
},
|
||||
"Microsoft.OpenApi": {
|
||||
"type": "CentralTransitive",
|
||||
"requested": "[2.7.5, )",
|
||||
"resolved": "2.7.5",
|
||||
"contentHash": "0FA67RSnRM4tcBKqiqVu/HPdZ9+QOKbmeRjxRUGTCjPU4C0bmUhd97Dso7Yild5P7nOV6GxJ2xrK0Kv/O9xp0w=="
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,6 @@
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using FinalFactory.Rendezvous.Client;
|
||||
using FinalFactory.Rendezvous.Contracts;
|
||||
using FinalFactory.Rendezvous.Server.Abuse;
|
||||
@@ -19,6 +22,49 @@ namespace FinalFactory.Rendezvous.Tests.Client;
|
||||
|
||||
public sealed class RendezvousClientIntegrationTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task RestartReturnsTypedUnavailabilityThenAllowsHostReregistration()
|
||||
{
|
||||
int port = ReserveTcpPort();
|
||||
string address = $"http://127.0.0.1:{port}";
|
||||
using HttpClient client = new() { BaseAddress = new Uri(address) };
|
||||
RendezvousClientOptions noRetry = new()
|
||||
{
|
||||
MaximumSafeRetries = 0,
|
||||
RequestTimeout = TimeSpan.FromSeconds(1),
|
||||
};
|
||||
ClientTestHost first = await ClientTestHost.StartAsync(address);
|
||||
RendezvousPublisherClient publisher = new(client, noRetry);
|
||||
RendezvousClientResult<PublishedSession> registered = await publisher.RegisterAsync(
|
||||
CreateRegistration(100),
|
||||
first.PublisherCredential);
|
||||
PublishedSession initialSession = AssertSuccess(registered);
|
||||
BindPresence(first, initialSession, 41_100);
|
||||
RendezvousSessionBrowserClient browser = new(client, noRetry);
|
||||
BrowseSessionsResponse beforeRestart = AssertSuccess(await browser.BrowseAsync(BrowseRequest()));
|
||||
Assert.Equal(initialSession.ListingId, Assert.Single(beforeRestart.Items).ListingId);
|
||||
Stopwatch restart = Stopwatch.StartNew();
|
||||
await first.DisposeAsync();
|
||||
|
||||
RendezvousClientResult<BrowseSessionsResponse> unavailable = await browser.BrowseAsync(BrowseRequest());
|
||||
Assert.False(unavailable.IsSuccess);
|
||||
Assert.Equal(RendezvousErrorCode.ServiceUnavailable, unavailable.Error);
|
||||
|
||||
await using ClientTestHost second = await ClientTestHost.StartAsync(address);
|
||||
RendezvousClientResult<PublishedSession> reregistered = await publisher.RegisterAsync(
|
||||
CreateRegistration(101),
|
||||
second.PublisherCredential);
|
||||
PublishedSession replacementSession = AssertSuccess(reregistered);
|
||||
Assert.NotEqual(initialSession.ListingId, replacementSession.ListingId);
|
||||
BindPresence(second, replacementSession, 41_101);
|
||||
BrowseSessionsResponse afterRestart = AssertSuccess(await browser.BrowseAsync(BrowseRequest()));
|
||||
Assert.Equal(replacementSession.ListingId, Assert.Single(afterRestart.Items).ListingId);
|
||||
Assert.DoesNotContain(afterRestart.Items, item => item.ListingId == initialSession.ListingId);
|
||||
Assert.True(
|
||||
restart.Elapsed < TimeSpan.FromSeconds(5),
|
||||
$"Local restart and host re-registration took {restart.Elapsed}.");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PublisherAndBrowserClientsCompleteTheRealSessionLifecycleAndPaging()
|
||||
{
|
||||
@@ -102,6 +148,27 @@ public sealed class RendezvousClientIntegrationTests
|
||||
return Assert.IsAssignableFrom<T>(result.Value);
|
||||
}
|
||||
|
||||
private static void BindPresence(ClientTestHost host, PublishedSession session, int port)
|
||||
{
|
||||
Assert.True(host.Capabilities.TryFingerprint(
|
||||
session.HostPresenceCapability,
|
||||
out SecretFingerprint fingerprint));
|
||||
Assert.Equal(StoreResultCode.Success, host.Store.BindHostPresence(new(
|
||||
session.HostPresenceHandle,
|
||||
fingerprint,
|
||||
new(AddressFamilyKind.Ipv4, "203.0.113.80", port),
|
||||
null)).Code);
|
||||
}
|
||||
|
||||
private static BrowseSessionsRequest BrowseRequest() => new()
|
||||
{
|
||||
GameId = new("space-game"),
|
||||
EnvironmentId = new("production"),
|
||||
RegionId = new("eu-central"),
|
||||
ProtocolVersion = 7,
|
||||
PageSize = 10,
|
||||
};
|
||||
|
||||
private static RegisterSessionRequest CreateRegistration(int index) => new()
|
||||
{
|
||||
IdempotencyKey = $"sdk-integration-{index}",
|
||||
@@ -139,7 +206,7 @@ public sealed class RendezvousClientIntegrationTests
|
||||
internal EphemeralCapabilityIssuer Capabilities { get; }
|
||||
internal string PublisherCredential { get; }
|
||||
|
||||
internal static async Task<ClientTestHost> StartAsync()
|
||||
internal static async Task<ClientTestHost> StartAsync(string? bindAddress = null)
|
||||
{
|
||||
ManualRendezvousClock clock = new(ProvisioningTestData.Now);
|
||||
EphemeralStoreOptions stateOptions = new();
|
||||
@@ -153,7 +220,7 @@ public sealed class RendezvousClientIntegrationTests
|
||||
string credential = provisioning.Credentials.Issue(principal, clock.UtcNow);
|
||||
|
||||
WebApplicationBuilder builder = WebApplication.CreateBuilder();
|
||||
builder.WebHost.UseUrls("http://127.0.0.1:0");
|
||||
builder.WebHost.UseUrls(bindAddress ?? "http://127.0.0.1:0");
|
||||
builder.Services.ConfigureHttpJsonOptions(static options =>
|
||||
ContractJson.Configure(options.SerializerOptions));
|
||||
builder.Services.Configure<RouteHandlerOptions>(static options =>
|
||||
@@ -180,10 +247,10 @@ public sealed class RendezvousClientIntegrationTests
|
||||
app.MapRendezvousContractEndpoints();
|
||||
await app.StartAsync();
|
||||
IServer server = app.Services.GetRequiredService<IServer>();
|
||||
string address = Assert.Single(server.Features.Get<IServerAddressesFeature>()!.Addresses);
|
||||
string serviceAddress = Assert.Single(server.Features.Get<IServerAddressesFeature>()!.Addresses);
|
||||
return new(
|
||||
app,
|
||||
new HttpClient { BaseAddress = new Uri(address) },
|
||||
new HttpClient { BaseAddress = new Uri(serviceAddress) },
|
||||
store,
|
||||
capabilities,
|
||||
credential);
|
||||
@@ -196,4 +263,13 @@ public sealed class RendezvousClientIntegrationTests
|
||||
await _application.DisposeAsync();
|
||||
}
|
||||
}
|
||||
|
||||
private static int ReserveTcpPort()
|
||||
{
|
||||
TcpListener listener = new(IPAddress.Loopback, 0);
|
||||
listener.Start();
|
||||
int port = ((IPEndPoint)listener.LocalEndpoint).Port;
|
||||
listener.Stop();
|
||||
return port;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,6 +61,32 @@ public sealed class ProductionProcessTests
|
||||
Assert.InRange(shutdown.Elapsed, TimeSpan.FromMilliseconds(700), TimeSpan.FromSeconds(5));
|
||||
AssertTcpPortIsReleased(httpPort);
|
||||
AssertUdpPortIsReleased(udpPort);
|
||||
|
||||
process.Dispose();
|
||||
process = null;
|
||||
Stopwatch replacementReady = Stopwatch.StartNew();
|
||||
ProcessStartInfo replacementInfo = CreateStartInfo(httpPort, udpPort, secretPath);
|
||||
process = Process.Start(replacementInfo)
|
||||
?? throw new InvalidOperationException("The replacement production process did not start.");
|
||||
Task<string> replacementOutput = process.StandardOutput.ReadToEndAsync();
|
||||
Task<string> replacementError = process.StandardError.ReadToEndAsync();
|
||||
await WaitForReadyAsync(httpPort, process, TimeSpan.FromSeconds(15));
|
||||
Assert.True(
|
||||
replacementReady.Elapsed < TimeSpan.FromSeconds(15),
|
||||
$"Replacement readiness took {replacementReady.Elapsed}.");
|
||||
AssertUdpPortIsBound(udpPort);
|
||||
|
||||
await SendSigtermAsync(process);
|
||||
using CancellationTokenSource replacementTimeout = new(TimeSpan.FromSeconds(6));
|
||||
await process.WaitForExitAsync(replacementTimeout.Token);
|
||||
string replacementFinalOutput = await replacementOutput;
|
||||
string replacementFinalError = await replacementError;
|
||||
Assert.True(
|
||||
process.ExitCode == 0,
|
||||
$"Replacement exited with {process.ExitCode}. "
|
||||
+ $"stdout: {replacementFinalOutput} stderr: {replacementFinalError}");
|
||||
AssertTcpPortIsReleased(httpPort);
|
||||
AssertUdpPortIsReleased(udpPort);
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -79,6 +105,97 @@ public sealed class ProductionProcessTests
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ProductionTransportSoakKeepsHandlesMemoryAndSocketsBounded()
|
||||
{
|
||||
if (!OperatingSystem.IsLinux())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
int httpPort = ReserveTcpPort();
|
||||
int udpPort = ReserveUdpPort();
|
||||
string secretPath = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
$"rendezvous-transport-soak-secret-{Guid.NewGuid():N}");
|
||||
await File.WriteAllBytesAsync(secretPath, RandomNumberGenerator.GetBytes(32));
|
||||
Process? process = null;
|
||||
try
|
||||
{
|
||||
process = Process.Start(CreateStartInfo(httpPort, udpPort, secretPath))
|
||||
?? throw new InvalidOperationException("The production soak process did not start.");
|
||||
Task<string> standardOutput = process.StandardOutput.ReadToEndAsync();
|
||||
Task<string> standardError = process.StandardError.ReadToEndAsync();
|
||||
await WaitForReadyAsync(httpPort, process, TimeSpan.FromSeconds(10));
|
||||
process.Refresh();
|
||||
int baselineHandles = process.HandleCount;
|
||||
long baselineWorkingSet = process.WorkingSet64;
|
||||
int peakHandles = baselineHandles;
|
||||
byte[] invalidDatagram = RandomNumberGenerator.GetBytes(64);
|
||||
IPEndPoint udpEndpoint = new(IPAddress.Loopback, udpPort);
|
||||
using HttpClient client = new() { Timeout = TimeSpan.FromSeconds(1) };
|
||||
using UdpClient udp = new();
|
||||
Stopwatch soak = Stopwatch.StartNew();
|
||||
int cycles = 0;
|
||||
int accepted = 0;
|
||||
int shed = 0;
|
||||
while (soak.Elapsed < TimeSpan.FromSeconds(10))
|
||||
{
|
||||
using HttpResponseMessage response = await client.GetAsync(
|
||||
$"http://127.0.0.1:{httpPort}/health/live");
|
||||
if (response.StatusCode == HttpStatusCode.OK)
|
||||
{
|
||||
accepted++;
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode);
|
||||
shed++;
|
||||
}
|
||||
|
||||
await udp.SendAsync(invalidDatagram, udpEndpoint);
|
||||
cycles++;
|
||||
if (cycles % 100 == 0)
|
||||
{
|
||||
process.Refresh();
|
||||
peakHandles = Math.Max(peakHandles, process.HandleCount);
|
||||
}
|
||||
}
|
||||
|
||||
Assert.True(cycles >= 100, $"Transport soak completed only {cycles} cycles.");
|
||||
Assert.True(accepted > 0, "Transport soak never admitted a health request.");
|
||||
Assert.True(shed > 0, "Transport soak never exercised typed HTTP load shedding.");
|
||||
await Task.Delay(TimeSpan.FromSeconds(2));
|
||||
using (HttpResponseMessage recovered = await client.GetAsync(
|
||||
$"http://127.0.0.1:{httpPort}/health/live"))
|
||||
{
|
||||
Assert.Equal(HttpStatusCode.OK, recovered.StatusCode);
|
||||
}
|
||||
|
||||
process.Refresh();
|
||||
Assert.InRange(peakHandles, 0, baselineHandles + 32);
|
||||
Assert.InRange(process.HandleCount, 0, baselineHandles + 16);
|
||||
Assert.InRange(process.WorkingSet64, 0, baselineWorkingSet + 67_108_864);
|
||||
AssertUdpPortIsBound(udpPort);
|
||||
|
||||
await SendSigtermAsync(process);
|
||||
using CancellationTokenSource shutdownTimeout = new(TimeSpan.FromSeconds(6));
|
||||
await process.WaitForExitAsync(shutdownTimeout.Token);
|
||||
string output = await standardOutput;
|
||||
string error = await standardError;
|
||||
Assert.True(
|
||||
process.ExitCode == 0,
|
||||
$"Transport soak process failed. stdout: {output} stderr: {error}");
|
||||
AssertTcpPortIsReleased(httpPort);
|
||||
AssertUdpPortIsReleased(udpPort);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await StopProcessTreeAsync(process);
|
||||
File.Delete(secretPath);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DocumentedSmokeScriptReachesHttpAndAuthenticatedUdpFlow()
|
||||
{
|
||||
|
||||
@@ -411,6 +411,92 @@ public sealed class InMemoryEphemeralRendezvousStoreTests
|
||||
Assert.True(fixture.Store.CreateJoinAttempt(attempt).Succeeded);
|
||||
Assert.Equal(StoreResultCode.CapacityExceeded, fixture.Store.CreateJoinAttempt(
|
||||
fixture.AttemptCommand(listing, "client-quota-2") with { ScopeAttemptLimit = 1 }).Code);
|
||||
|
||||
fixture.Clock.Advance(TimeSpan.FromSeconds(30));
|
||||
Assert.True(fixture.Store.BindHostPresence(new(
|
||||
first.Listing.HostPresenceHandle,
|
||||
first.Listing.HostPresenceFingerprint,
|
||||
EphemeralStateFixture.PublicEndpoint(42_101),
|
||||
null)).Succeeded);
|
||||
Assert.True(fixture.Store.CreateJoinAttempt(
|
||||
fixture.AttemptCommand(listing, "client-quota-3") with { ScopeAttemptLimit = 1 }).Succeeded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RemovingAListingReleasesItsConstantTimeOwnerQuota()
|
||||
{
|
||||
EphemeralStateFixture fixture = new();
|
||||
CreateListingCommand first = fixture.ListingCommand(owner: "publisher-quota") with
|
||||
{
|
||||
OwnerListingLimit = 1,
|
||||
};
|
||||
CreateListingCommand second = fixture.ListingCommand(owner: "publisher-quota") with
|
||||
{
|
||||
OwnerListingLimit = 1,
|
||||
};
|
||||
StoreResult<StoredListing> created = fixture.Store.CreateListing(first);
|
||||
Assert.True(created.Succeeded);
|
||||
Assert.Equal(StoreResultCode.CapacityExceeded, fixture.Store.CreateListing(second).Code);
|
||||
|
||||
Assert.True(fixture.Store.DeleteListing(new(
|
||||
first.Listing.ListingId,
|
||||
first.Listing.LeaseId,
|
||||
first.Listing.LeaseFingerprint,
|
||||
first.Listing.OwnerSubject)).Succeeded);
|
||||
Assert.True(fixture.Store.CreateListing(second).Succeeded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RepeatedMutableDeadlineRefreshesKeepOneScheduledEntryPerKey()
|
||||
{
|
||||
EphemeralStateFixture fixture = new();
|
||||
StoredListing listing = fixture.CreateVisibleListing(out _);
|
||||
|
||||
for (int iteration = 0; iteration < 10_000; iteration++)
|
||||
{
|
||||
fixture.Clock.Advance(TimeSpan.FromTicks(1));
|
||||
StoreResult<StoredListing> renewed = fixture.Store.RenewLease(new(
|
||||
listing.Definition.ListingId,
|
||||
listing.Definition.LeaseId,
|
||||
listing.Definition.LeaseFingerprint,
|
||||
listing.Definition.OwnerSubject,
|
||||
listing.Version));
|
||||
Assert.True(renewed.Succeeded, $"Renewal {iteration} failed with {renewed.Code}.");
|
||||
listing = renewed.Value!;
|
||||
StoreResult<StoredListing> presence = fixture.Store.BindHostPresence(new(
|
||||
listing.Definition.HostPresenceHandle,
|
||||
listing.Definition.HostPresenceFingerprint,
|
||||
EphemeralStateFixture.PublicEndpoint(42_200),
|
||||
null));
|
||||
Assert.True(presence.Succeeded, $"Presence {iteration} failed with {presence.Code}.");
|
||||
StoreResult<int> revoked = fixture.Store.RevokePrincipal(
|
||||
"repeated-revocation",
|
||||
TimeSpan.FromMinutes(1));
|
||||
Assert.True(revoked.Succeeded, $"Revocation {iteration} failed with {revoked.Code}.");
|
||||
}
|
||||
|
||||
Assert.Equal(4, fixture.Store.ScheduledExpiryEntryCount);
|
||||
fixture.Clock.Advance(TimeSpan.FromSeconds(19));
|
||||
Assert.True(fixture.Store.GetListing(listing.Definition.ListingId, true).Succeeded);
|
||||
Assert.Equal(4, fixture.Store.ScheduledExpiryEntryCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RepeatedPrincipalRevocationCanExtendButCannotShortenProtection()
|
||||
{
|
||||
EphemeralStateFixture fixture = new();
|
||||
const string subject = "protected-publisher";
|
||||
|
||||
Assert.True(fixture.Store.RevokePrincipal(subject, TimeSpan.FromMinutes(10)).Succeeded);
|
||||
fixture.Clock.Advance(TimeSpan.FromSeconds(1));
|
||||
Assert.True(fixture.Store.RevokePrincipal(subject, TimeSpan.FromSeconds(1)).Succeeded);
|
||||
fixture.Clock.Advance(TimeSpan.FromSeconds(2));
|
||||
Assert.Equal(
|
||||
StoreResultCode.Revoked,
|
||||
fixture.Store.CreateListing(fixture.ListingCommand(owner: subject)).Code);
|
||||
|
||||
fixture.Clock.Advance(TimeSpan.FromSeconds(597));
|
||||
Assert.True(fixture.Store.CreateListing(fixture.ListingCommand(owner: subject)).Succeeded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
Reference in New Issue
Block a user