feat(operations): add capacity and resilience gates (#18)
This commit is contained in:
@@ -36,6 +36,9 @@ jobs:
|
|||||||
- name: Test
|
- name: Test
|
||||||
run: dotnet test Rendezvous.slnx --configuration Release --no-build
|
run: dotnet test Rendezvous.slnx --configuration Release --no-build
|
||||||
|
|
||||||
|
- name: Run quick capacity and resilience gate
|
||||||
|
run: ./scripts/run-capacity-gate.sh
|
||||||
|
|
||||||
- name: Test privileged Linux namespace topology when available
|
- name: Test privileged Linux namespace topology when available
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
@@ -7,5 +7,6 @@ TestResults/
|
|||||||
*.user
|
*.user
|
||||||
*.userosscache
|
*.userosscache
|
||||||
deploy/compose/.smoke.env
|
deploy/compose/.smoke.env
|
||||||
|
artifacts/
|
||||||
deploy/compose/secrets/*
|
deploy/compose/secrets/*
|
||||||
!deploy/compose/secrets/.gitignore
|
!deploy/compose/secrets/.gitignore
|
||||||
|
|||||||
@@ -83,9 +83,9 @@ Rendezvous is under active roadmap development. The versioned contracts,
|
|||||||
directory leases, authenticated join attempts, LiteNetLib mediator, caller-owned
|
directory leases, authenticated join attempts, LiteNetLib mediator, caller-owned
|
||||||
SDK coordination, typed connection outcomes, thin public-SDK diagnostic client,
|
SDK coordination, typed connection outcomes, thin public-SDK diagnostic client,
|
||||||
deterministic NAT topology harness, hostile-input controls,
|
deterministic NAT topology harness, hostile-input controls,
|
||||||
observability/operator surface, and secure single-active Linux deployment are
|
observability/operator surface, secure single-active Linux deployment, and
|
||||||
implemented. Capacity, resilience, packaging, and final production-readiness
|
numeric capacity/resilience gates are implemented. Packaging, consumer pilots,
|
||||||
gates remain in progress;
|
and final production-readiness gates remain in progress;
|
||||||
participating games must not treat the current repository as a finished production
|
participating games must not treat the current repository as a finished production
|
||||||
service until those gates land.
|
service until those gates land.
|
||||||
|
|
||||||
@@ -103,6 +103,9 @@ operator controls are defined in the
|
|||||||
The pinned non-root container, production topology, graceful drain, Linux
|
The pinned non-root container, production topology, graceful drain, Linux
|
||||||
hardening, smoke procedure, and recovery lifecycle are documented in
|
hardening, smoke procedure, and recovery lifecycle are documented in
|
||||||
[secure single-active Linux deployment](docs/deployment/linux.md).
|
[secure single-active Linux deployment](docs/deployment/linux.md).
|
||||||
|
The numeric core-state candidate profile, public launch objectives, accelerated
|
||||||
|
soak, resilience matrix, and single-active scaling decision are recorded in
|
||||||
|
[capacity and resilience gates](docs/operations/capacity-and-resilience.md).
|
||||||
The scriptable host/browser/join diagnostic and its stable automation contract are
|
The scriptable host/browser/join diagnostic and its stable automation contract are
|
||||||
documented in the [TestClient integration guide](docs/integration/test-client.md).
|
documented in the [TestClient integration guide](docs/integration/test-client.md).
|
||||||
The always-on three-party scenarios, optional Linux namespace topology, and
|
The always-on three-party scenarios, optional Linux namespace topology, and
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
<Project Path="src/FinalFactory.Rendezvous.TestClient/FinalFactory.Rendezvous.TestClient.csproj" />
|
<Project Path="src/FinalFactory.Rendezvous.TestClient/FinalFactory.Rendezvous.TestClient.csproj" />
|
||||||
</Folder>
|
</Folder>
|
||||||
<Folder Name="/tests/">
|
<Folder Name="/tests/">
|
||||||
|
<Project Path="tests/FinalFactory.Rendezvous.Capacity/FinalFactory.Rendezvous.Capacity.csproj" />
|
||||||
<Project Path="tests/FinalFactory.Rendezvous.Tests/FinalFactory.Rendezvous.Tests.csproj" />
|
<Project Path="tests/FinalFactory.Rendezvous.Tests/FinalFactory.Rendezvous.Tests.csproj" />
|
||||||
</Folder>
|
</Folder>
|
||||||
</Solution>
|
</Solution>
|
||||||
|
|||||||
@@ -40,7 +40,8 @@ LockPersonality=true
|
|||||||
SystemCallArchitectures=native
|
SystemCallArchitectures=native
|
||||||
UMask=0077
|
UMask=0077
|
||||||
LimitNOFILE=4096
|
LimitNOFILE=4096
|
||||||
MemoryMax=512M
|
CPUQuota=200%
|
||||||
|
MemoryMax=2G
|
||||||
TasksMax=128
|
TasksMax=128
|
||||||
|
|
||||||
[Install]
|
[Install]
|
||||||
|
|||||||
@@ -129,9 +129,18 @@ until the owner records:
|
|||||||
- which games may enable anonymous unlisted player hosting;
|
- which games may enable anonymous unlisted player hosting;
|
||||||
- deployment regions, data-processing jurisdiction, and approval of the stated
|
- deployment regions, data-processing jurisdiction, and approval of the stated
|
||||||
30-day audit/13-month aggregate retention periods;
|
30-day audit/13-month aggregate retention periods;
|
||||||
- the per-game dedicated fallback endpoint policy;
|
- the per-game dedicated fallback endpoint policy.
|
||||||
- the measured supported profile and whether the 99.5% single-active objective
|
|
||||||
is sufficient or shared-state/high-availability work must be brought forward.
|
Issue #18 measured and ratified the original 2-vCPU/2-GiB, 25,000-listing,
|
||||||
|
10,000-attempt core-state candidate profile and retained the 99.5% single-active
|
||||||
|
topology. It does not claim that core measurements prove public HTTP/UDP SLOs.
|
||||||
|
The versioned evidence, RTO, failure domains, and explicit signals that trigger
|
||||||
|
shared-state/high-availability work are recorded in the
|
||||||
|
[capacity and resilience gate](../operations/capacity-and-resilience.md). The
|
||||||
|
real-network canary in #23 must confirm that the proposed regional launch load
|
||||||
|
fits this profile and validate the public SLOs; it may lower the launch cap but
|
||||||
|
may not silently enable a
|
||||||
|
second active instance.
|
||||||
|
|
||||||
These are configuration and launch decisions, not permission to weaken the
|
These are configuration and launch decisions, not permission to weaken the
|
||||||
tenant, replay, endpoint-verification, or secret-handling controls.
|
tenant, replay, endpoint-verification, or secret-handling controls.
|
||||||
|
|||||||
@@ -118,8 +118,11 @@ Keep the host clock synchronized with authenticated NTP. Credential and key
|
|||||||
windows use wall time; lease, timeout, drain, and rate-limit deadlines use a
|
windows use wall time; lease, timeout, drain, and rate-limit deadlines use a
|
||||||
monotonic clock. Alert on clock synchronization loss before rotating keys.
|
monotonic clock. Alert on clock synchronization loss before rotating keys.
|
||||||
|
|
||||||
Start with the Compose limits (one CPU, 512 MiB, 128 PIDs, 4096 descriptors),
|
The checked-in Compose limits (one CPU and 512 MiB) are for its isolated smoke
|
||||||
measure real traffic, then change the limits and the server budgets together.
|
profile, not a production capacity claim. The measured core-state candidate
|
||||||
|
uses 2 vCPU and 2 GiB with the same 128-PID/4096-descriptor ceilings; see
|
||||||
|
the [capacity and resilience gate](../operations/capacity-and-resilience.md).
|
||||||
|
Measure real traffic, then change resource limits and server budgets together.
|
||||||
Memory pressure or CPU throttling must not extend orchestrator termination past
|
Memory pressure or CPU throttling must not extend orchestrator termination past
|
||||||
`DrainDeadlineSeconds` plus five seconds.
|
`DrainDeadlineSeconds` plus five seconds.
|
||||||
|
|
||||||
@@ -157,8 +160,8 @@ sudo systemctl enable --now rendezvous.service
|
|||||||
Create the dedicated `rendezvous` user without a login shell. Keep
|
Create the dedicated `rendezvous` user without a login shell. Keep
|
||||||
`/opt/rendezvous` and `/etc/rendezvous` root-owned and non-writable by that user;
|
`/opt/rendezvous` and `/etc/rendezvous` root-owned and non-writable by that user;
|
||||||
install each required key with `root:rendezvous` ownership and mode `0440`. The
|
install each required key with `root:rendezvous` ownership and mode `0440`. The
|
||||||
unit applies the same resource, filesystem, privilege, network-family, and
|
unit applies the measured 2-vCPU/2-GiB core-state candidate profile plus the
|
||||||
shutdown constraints as Compose.
|
same filesystem, privilege, network-family, and shutdown hardening as Compose.
|
||||||
|
|
||||||
## HTTP and UDP smoke
|
## HTTP and UDP smoke
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,189 @@
|
|||||||
|
# Capacity, resilience, and availability gate
|
||||||
|
|
||||||
|
Tracking: #18
|
||||||
|
|
||||||
|
This gate turns the v1 budgets in ADR 0003 into a repeatable release decision.
|
||||||
|
It does not turn Rendezvous into a horizontally scalable service: v1 remains one
|
||||||
|
active process with bounded in-memory state. A second process may be a cold
|
||||||
|
standby, but it must not accept traffic until the first process has stopped and
|
||||||
|
released the public HTTP and UDP endpoints.
|
||||||
|
|
||||||
|
## Launch envelope and approved core-state profile
|
||||||
|
|
||||||
|
The approved core-state profile is one Linux process limited to 2 vCPU and
|
||||||
|
2 GiB RAM. Public HTTP/UDP numbers are launch objectives that require the #23
|
||||||
|
real-network canary before they become a supported service claim:
|
||||||
|
|
||||||
|
| Dimension | Value | Evidence status |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Visible listings | 25,000 | Enforced and measured here |
|
||||||
|
| Active join attempts | 10,000 | Enforced and measured here |
|
||||||
|
| Core control path | 200 operations/second; p95 at most 200 ms | Measured here |
|
||||||
|
| Core mediation path | 2,000 pairings/second; p95 at most 100 ms | Measured here |
|
||||||
|
| Sustained HTTP demand | 200 requests/second | #23 launch objective; not yet a supported claim |
|
||||||
|
| Sustained UDP demand | 2,000 datagrams/second | #23 launch objective; not yet a supported claim |
|
||||||
|
| Public HTTP/UDP latency | p95 at most 200 ms / 100 ms | #23 launch objective; not yet a supported claim |
|
||||||
|
| Capacity-phase average CPU / peak working memory | below 70% / below 1.5 GiB | Measured for the core candidate |
|
||||||
|
| Valid in-profile monthly availability | 99.5%, excluding announced maintenance | Operational objective |
|
||||||
|
| Process-ready RTO / host-visible recovery | 15 seconds / 90 seconds | 15 seconds automated; 90-second deployment drill required |
|
||||||
|
|
||||||
|
The proposed public-network mix is 20% registration/update, 30% lease-critical
|
||||||
|
renew/delete, 30% browse, and 20% join authorization for HTTP. The UDP mix is
|
||||||
|
60% authenticated host-presence refresh, 30% attempt contributions, and 10%
|
||||||
|
invalid or duplicate traffic that must be dropped early. A deployment may use a
|
||||||
|
lower per-game profile, but must not claim a higher one without new versioned
|
||||||
|
evidence.
|
||||||
|
|
||||||
|
The capacity harness fills the complete state ceilings, then measures
|
||||||
|
registration plus presence, renewal, a 100-item compatible browse, join
|
||||||
|
issuance, simultaneous two-peer pairing, principal revocation, and telemetry.
|
||||||
|
It applies 200/100 ms guardrails and minimum 200 control / 2,000 mediation
|
||||||
|
operations per second to the core hot path. Those measurements deliberately
|
||||||
|
exclude Kestrel, LiteNetLib, TLS, JSON, socket scheduling, and the documented
|
||||||
|
mixed traffic shape. The #23 real-network canary must exercise those layers,
|
||||||
|
rate-shape the mix, record errors and shedding, and meet the public objectives
|
||||||
|
before launch; a core result is not a public-network latency or throughput claim.
|
||||||
|
|
||||||
|
## Reproduce the evidence
|
||||||
|
|
||||||
|
Every push runs the quick profile and the selected fault matrix:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/run-capacity-gate.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the production candidate on an otherwise idle Linux host and restrict the
|
||||||
|
runtime to two logical CPUs. The default candidate includes a five-minute,
|
||||||
|
high-intensity expiry soak; use 3,600 seconds for a release-candidate endurance
|
||||||
|
run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export RENDEZVOUS_CAPACITY_PROFILE=candidate
|
||||||
|
export RENDEZVOUS_CAPACITY_CPUSET=0,1
|
||||||
|
export RENDEZVOUS_CAPACITY_OUTPUT="$PWD/artifacts/capacity/candidate.json"
|
||||||
|
./scripts/run-capacity-gate.sh
|
||||||
|
|
||||||
|
# Release-candidate endurance override:
|
||||||
|
dotnet run --project tests/FinalFactory.Rendezvous.Capacity \
|
||||||
|
--configuration Release --no-build -- \
|
||||||
|
--profile candidate --soak-seconds 3600 \
|
||||||
|
--output artifacts/capacity/candidate-endurance.json
|
||||||
|
```
|
||||||
|
|
||||||
|
The machine must have at least 2 GiB available to the process. For formal
|
||||||
|
deployment evidence, run inside the same cgroup/container shape as production.
|
||||||
|
The v2 JSON embeds the commit and tree state, command, image context, CPU model,
|
||||||
|
kernel, affinity, cgroup quota/limit, collector mode, and workload seed. Supply
|
||||||
|
`RENDEZVOUS_EVIDENCE_IMAGE_DIGEST` when running a release image. Do not compare
|
||||||
|
results collected under a debugger,
|
||||||
|
concurrent build, thermal throttling, or oversubscribed CI host.
|
||||||
|
|
||||||
|
The checked-in baseline is
|
||||||
|
[`candidate-2cpu.json`](../evidence/capacity/v2/candidate-2cpu.json). It was
|
||||||
|
produced on .NET 10.0.9/Linux x64 with CPU affinity restricted to two logical
|
||||||
|
CPUs. It filled 25,000 listings and 10,000 attempts, peaked at about 162 MiB,
|
||||||
|
and cleared all active/retained state. The five-minute baseline supersedes any
|
||||||
|
earlier local probe when its timestamp and target duration differ.
|
||||||
|
|
||||||
|
## Soak and bounded-state interpretation
|
||||||
|
|
||||||
|
Each soak cycle creates a listing, repeatedly renews its lease and refreshes
|
||||||
|
presence, creates a join attempt, replay marker, and retained outcome, checks
|
||||||
|
that scheduled expiry entries remain proportional to live keys, then advances
|
||||||
|
the injected monotonic clock beyond all
|
||||||
|
deadlines, and verifies that listings, attempts, replay, idempotency, and outcome
|
||||||
|
state return to zero. The candidate also measures managed-memory and process
|
||||||
|
handle deltas after full collection. Failure is any retained state, more than
|
||||||
|
64 MiB retained managed memory, more than eight retained handles, a working set
|
||||||
|
above 1.5 GiB, an untyped capacity result, or failure to admit work after expiry.
|
||||||
|
|
||||||
|
This accelerated soak intentionally executes far more state lifecycle/cleanup
|
||||||
|
events than wall-clock traffic would permit. It catches stale deadline-queue
|
||||||
|
entries, cache growth, replay/idempotency retention, and cleanup cost. Because
|
||||||
|
it does not open Kestrel/LiteNetLib connections, its process-handle delta is only
|
||||||
|
a harness guard and is not evidence of transport stability by itself. The
|
||||||
|
selected production-process gate adds a ten-second real HTTP/UDP transport soak,
|
||||||
|
samples child-process handles and RSS, asserts bounded growth, then verifies a
|
||||||
|
clean SIGTERM and socket release. #23 must extend that into the full rate-shaped
|
||||||
|
multi-client canary while sampling queues, managed memory, and state
|
||||||
|
cardinalities. A one-hour core override remains required before tagging a
|
||||||
|
production release.
|
||||||
|
|
||||||
|
## Fault and recovery matrix
|
||||||
|
|
||||||
|
`run-capacity-gate.sh` runs these deterministic production paths before the
|
||||||
|
numeric profile:
|
||||||
|
|
||||||
|
| Fault | Required result |
|
||||||
|
| --- | --- |
|
||||||
|
| HTTP/UDP overload and tracker exhaustion | Typed HTTP `429`/`CapacityExceeded`, silent UDP drop, bounded tracker keys, recovery after the window |
|
||||||
|
| Optional traffic saturation | Lease-critical renew/update/delete capacity remains available |
|
||||||
|
| Store/dependency unavailable | Readiness fails; new authorization returns typed `ServiceUnavailable`; liveness remains independent |
|
||||||
|
| Graceful drain/SIGTERM | New work returns `Draining`; existing pairing may finish; process exits 0 and releases TCP/UDP before the deadline |
|
||||||
|
| Hard restart | In-flight state is lost; SDK reports typed `ServiceUnavailable`; a host re-registers, rebinds presence, and becomes the only browser-visible replacement |
|
||||||
|
| UDP listener bind/restart | Readiness stays false without the required listener; rebinding the advertised port restores native LiteNetLib pairing |
|
||||||
|
| Wall-clock jump/skew | Monotonic lease/attempt authority is neither shortened nor extended; credential skew remains capped at 30 seconds |
|
||||||
|
| Signing-secret rotation | New key signs, overlap verifies, retired/revoked key rejects, missing material fails startup |
|
||||||
|
| Principal revocation | Listing, presence, attempts, and outcome paths are removed atomically within the latency budget |
|
||||||
|
|
||||||
|
No external database exists in v1, so “dependency/store failure” means the
|
||||||
|
process-local atomic store is marked unavailable or a required listener/key is
|
||||||
|
unready. The service fails closed rather than pretending a degraded writable
|
||||||
|
mode exists.
|
||||||
|
|
||||||
|
## Bandwidth and amplification
|
||||||
|
|
||||||
|
- Accepted application datagrams are at most 1,200 bytes.
|
||||||
|
- Malformed, oversized, unauthenticated, stale, replayed, wrong-role, and
|
||||||
|
rate-limited traffic receives zero response bytes.
|
||||||
|
- A completing authenticated contribution produces at most one introduction to
|
||||||
|
each observed peer, and the combined response is at most 2.0 times that
|
||||||
|
contribution's bytes.
|
||||||
|
- The frozen-envelope and native LiteNetLib socket tests measure this on the real
|
||||||
|
UDP listener; the hostile corpus and allocation gate exercise 10,000+ inputs
|
||||||
|
without input-sized logs, tasks, or queues.
|
||||||
|
|
||||||
|
Bandwidth planning must therefore reserve ingress for the configured 2,000
|
||||||
|
datagrams/second plus edge overhead and egress for a worst-case verified 2.0
|
||||||
|
amplification. Actual successful pairs normally use two contributions and two
|
||||||
|
introductions; normal gameplay leaves Rendezvous entirely.
|
||||||
|
|
||||||
|
## Availability decision
|
||||||
|
|
||||||
|
Single-active remains the v1 topology. The measured core profile proves bounded
|
||||||
|
state and substantial core-path headroom, while public launch capacity remains
|
||||||
|
conditional on #23. The service has a bounded stop-before-start restart path.
|
||||||
|
Its failure domain is deliberately
|
||||||
|
one process/node/public UDP endpoint: node, kernel, host network, DNS/TLS edge,
|
||||||
|
secret configuration, or operator error can remove all readiness until the cold
|
||||||
|
replacement owns the same source-preserving endpoint.
|
||||||
|
|
||||||
|
The 99.5% objective permits about 216 minutes of unannounced downtime in a
|
||||||
|
30-day month. Operations must target process readiness within 15 seconds and
|
||||||
|
host-visible re-registration within 90 seconds, page when no ready instance
|
||||||
|
exists, and include detection plus recovery in the monthly budget. The current
|
||||||
|
in-process test validates typed downtime, same-port HTTP restart, fresh
|
||||||
|
registration, presence rebinding, and browser visibility in under five seconds;
|
||||||
|
the production-process test separately validates graceful termination, TCP/UDP
|
||||||
|
release, replacement startup on the same endpoints, UDP readiness, and the
|
||||||
|
15-second process-ready RTO. Cold-standby activation policy and the 90-second
|
||||||
|
operator-to-host recovery objective still require a deployment drill before
|
||||||
|
release. Rollout and rollback use the
|
||||||
|
deployment runbook's drain, stop, socket-release, start, smoke sequence; never
|
||||||
|
overlap old and new active processes.
|
||||||
|
|
||||||
|
Bring shared TTL/CAS state and deterministic mediator routing forward before
|
||||||
|
enabling two active instances if any of these occurs:
|
||||||
|
|
||||||
|
- one node cannot sustain 150% of the measured 30-day peak while meeting SLOs;
|
||||||
|
- CPU stays above 70%, memory above 75%, attempt depth above 70%, or limiter
|
||||||
|
drops/latency remain elevated after abusive traffic is excluded;
|
||||||
|
- the availability target rises above 99.5% or planned maintenance must preserve
|
||||||
|
listings; or
|
||||||
|
- one region requires multiple simultaneously active mediator endpoints.
|
||||||
|
|
||||||
|
Rendezvous makes no multi-instance claim today, so a two-node atomic-pairing
|
||||||
|
test is intentionally not applicable. It becomes a hard release gate with the
|
||||||
|
shared-state/routing implementation; until then `SingleActiveInstance=false`
|
||||||
|
fails production startup. Multi-region and relay remain separate evidence-driven
|
||||||
|
decisions.
|
||||||
@@ -115,7 +115,9 @@ permission denial is audited with actor and target fingerprints.
|
|||||||
Key revocation is process-local in the current single-instance store. Apply the
|
Key revocation is process-local in the current single-instance store. Apply the
|
||||||
same revocation to every instance, then replace configuration before restarting;
|
same revocation to every instance, then replace configuration before restarting;
|
||||||
a restart reconstructs the configured key ring. Principal revocation is bounded
|
a restart reconstructs the configured key ring. Principal revocation is bounded
|
||||||
to ten minutes and removes that principal's active listings and attempts. Use
|
to ten minutes and removes that principal's active listings and attempts. A
|
||||||
|
repeat action may extend an active revocation but never shortens it; wait for its
|
||||||
|
original deadline rather than treating a shorter repeat as an un-revoke. Use
|
||||||
listing revocation for one targeted session and drain before planned shutdown.
|
listing revocation for one targeted session and drain before planned shutdown.
|
||||||
|
|
||||||
## Audit retention and incident handling
|
## Audit retention and incident handling
|
||||||
|
|||||||
Executable
+55
@@ -0,0 +1,55 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
PROFILE="${RENDEZVOUS_CAPACITY_PROFILE:-quick}"
|
||||||
|
OUTPUT="${RENDEZVOUS_CAPACITY_OUTPUT:-$ROOT/artifacts/capacity/rendezvous-capacity-v2.json}"
|
||||||
|
CPUSET="${RENDEZVOUS_CAPACITY_CPUSET:-}"
|
||||||
|
PROJECT="$ROOT/tests/FinalFactory.Rendezvous.Capacity/FinalFactory.Rendezvous.Capacity.csproj"
|
||||||
|
TESTS="$ROOT/tests/FinalFactory.Rendezvous.Tests/FinalFactory.Rendezvous.Tests.csproj"
|
||||||
|
|
||||||
|
if [[ "$PROFILE" != quick && "$PROFILE" != candidate ]]; then
|
||||||
|
printf 'RENDEZVOUS_CAPACITY_PROFILE must be quick or candidate.\n' >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
command -v dotnet >/dev/null || {
|
||||||
|
printf 'Missing required command: dotnet\n' >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
if [[ -n "$CPUSET" ]]; then
|
||||||
|
command -v taskset >/dev/null || {
|
||||||
|
printf 'taskset is required when RENDEZVOUS_CAPACITY_CPUSET is set.\n' >&2
|
||||||
|
exit 2
|
||||||
|
}
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd "$ROOT"
|
||||||
|
dotnet restore "$ROOT/Rendezvous.slnx" --locked-mode
|
||||||
|
dotnet build "$ROOT/Rendezvous.slnx" --configuration Release --no-restore
|
||||||
|
|
||||||
|
filter='FullyQualifiedName~TrackerCapacityFailsClosedWithoutGrowingAndAWindowResetRecovers|FullyQualifiedName~OptionalTrafficCannotConsumeTheLeaseOperationReserve|FullyQualifiedName~ConcurrentAbusiveBurstStaysBoundedAndCannotBlockCriticalHttp|FullyQualifiedName~HttpOverloadIsTypedAndOversizedBodiesAreRejectedBeforeDispatch|FullyQualifiedName~WallClockMovementDoesNotExpireOrExtendLease|FullyQualifiedName~RepeatedMutableDeadlineRefreshesKeepOneScheduledEntryPerKey|FullyQualifiedName~RepeatedPrincipalRevocationCanExtendButCannotShortenProtection|FullyQualifiedName~RestartHasNewGenerationAndNoEphemeralState|FullyQualifiedName~RestartReturnsTypedUnavailabilityThenAllowsHostReregistration|FullyQualifiedName~DrainRejectsNewWorkAllowsInflightCompletionThenClearsState|FullyQualifiedName~UnavailableStoreFailsNewAuthorizationClosedAndErasesActiveState|FullyQualifiedName~KeyRotationHonorsOverlapAndRejectsRetiredKeys|FullyQualifiedName~OperatorSurfaceSeparatesAuthenticationConfirmsActionsAndRedactsInspection|FullyQualifiedName~SigtermDrainsThenReleasesHttpAndUdpSockets|FullyQualifiedName~ProductionTransportSoakKeepsHandlesMemoryAndSocketsBounded|FullyQualifiedName~NativeLiteNetLibRequestsIntroduceTheAuthorizedPair'
|
||||||
|
dotnet test "$TESTS" --configuration Release --no-build --filter "$filter" \
|
||||||
|
--logger 'console;verbosity=minimal'
|
||||||
|
|
||||||
|
mkdir -p "$(dirname "$OUTPUT")"
|
||||||
|
arguments=(
|
||||||
|
dotnet run --project "$PROJECT" --configuration Release --no-build --
|
||||||
|
--profile "$PROFILE" --output "$OUTPUT"
|
||||||
|
)
|
||||||
|
export RENDEZVOUS_EVIDENCE_COMMIT="${RENDEZVOUS_EVIDENCE_COMMIT:-$(git rev-parse HEAD)}"
|
||||||
|
if [[ -z "${RENDEZVOUS_EVIDENCE_TREE_STATE:-}" ]]; then
|
||||||
|
if [[ -n "$(git status --porcelain)" ]]; then
|
||||||
|
export RENDEZVOUS_EVIDENCE_TREE_STATE=dirty
|
||||||
|
else
|
||||||
|
export RENDEZVOUS_EVIDENCE_TREE_STATE=clean
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
export RENDEZVOUS_EVIDENCE_CPUSET="${CPUSET:-unrestricted}"
|
||||||
|
export RENDEZVOUS_EVIDENCE_COMMAND="RENDEZVOUS_CAPACITY_PROFILE=$PROFILE RENDEZVOUS_CAPACITY_CPUSET=${CPUSET:-unrestricted} ./scripts/run-capacity-gate.sh"
|
||||||
|
if [[ -n "$CPUSET" ]]; then
|
||||||
|
taskset -c "$CPUSET" "${arguments[@]}"
|
||||||
|
else
|
||||||
|
"${arguments[@]}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf 'Capacity and resilience gate passed; evidence: %s\n' "$OUTPUT"
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
using System.Runtime.CompilerServices;
|
using System.Runtime.CompilerServices;
|
||||||
|
|
||||||
[assembly: InternalsVisibleTo("FinalFactory.Rendezvous.Tests")]
|
[assembly: InternalsVisibleTo("FinalFactory.Rendezvous.Tests")]
|
||||||
|
[assembly: InternalsVisibleTo("FinalFactory.Rendezvous.Capacity")]
|
||||||
|
|||||||
@@ -11,16 +11,29 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
|||||||
private readonly DateTimeOffset _wallOrigin;
|
private readonly DateTimeOffset _wallOrigin;
|
||||||
private readonly TimeSpan _monotonicOrigin;
|
private readonly TimeSpan _monotonicOrigin;
|
||||||
private readonly Dictionary<SessionListingId, ListingEntry> _listings = [];
|
private readonly Dictionary<SessionListingId, ListingEntry> _listings = [];
|
||||||
|
private readonly Dictionary<string, int> _listingCountsByOwner = new(StringComparer.Ordinal);
|
||||||
|
private readonly PriorityQueue<DeadlineEntry<SessionListingId>, long> _listingExpiries = new();
|
||||||
|
private readonly HashSet<SessionListingId> _scheduledListingExpiries = [];
|
||||||
private readonly Dictionary<LeaseId, SessionListingId> _leases = [];
|
private readonly Dictionary<LeaseId, SessionListingId> _leases = [];
|
||||||
private readonly Dictionary<MediationHandle, SessionListingId> _presenceHandles = [];
|
private readonly Dictionary<MediationHandle, SessionListingId> _presenceHandles = [];
|
||||||
private readonly Dictionary<MediationHandle, PresenceEntry> _presence = [];
|
private readonly Dictionary<MediationHandle, PresenceEntry> _presence = [];
|
||||||
|
private readonly PriorityQueue<DeadlineEntry<MediationHandle>, long> _presenceExpiries = new();
|
||||||
|
private readonly HashSet<MediationHandle> _scheduledPresenceExpiries = [];
|
||||||
private readonly Dictionary<JoinAttemptId, AttemptEntry> _attempts = [];
|
private readonly Dictionary<JoinAttemptId, AttemptEntry> _attempts = [];
|
||||||
private readonly PriorityQueue<AttemptExpiry, long> _attemptExpiries = new();
|
private readonly Dictionary<TenantScope, int> _attemptCountsByScope = [];
|
||||||
|
private readonly Dictionary<SessionListingId, HashSet<JoinAttemptId>> _attemptsByListing = [];
|
||||||
|
private readonly PriorityQueue<DeadlineEntry<JoinAttemptId>, long> _attemptExpiries = new();
|
||||||
private readonly Dictionary<JoinAttemptId, OutcomeReportEntry> _outcomeReports = [];
|
private readonly Dictionary<JoinAttemptId, OutcomeReportEntry> _outcomeReports = [];
|
||||||
|
private readonly Dictionary<SessionListingId, HashSet<JoinAttemptId>> _outcomesByListing = [];
|
||||||
|
private readonly PriorityQueue<DeadlineEntry<JoinAttemptId>, long> _outcomeExpiries = new();
|
||||||
private readonly Dictionary<MediationHandle, JoinAttemptId> _attemptHandles = [];
|
private readonly Dictionary<MediationHandle, JoinAttemptId> _attemptHandles = [];
|
||||||
private readonly Dictionary<string, IdempotencyEntry> _idempotency = new(StringComparer.Ordinal);
|
private readonly Dictionary<string, IdempotencyEntry> _idempotency = new(StringComparer.Ordinal);
|
||||||
|
private readonly PriorityQueue<DeadlineEntry<string>, long> _idempotencyExpiries = new();
|
||||||
private readonly Dictionary<string, TimeSpan> _replay = new(StringComparer.Ordinal);
|
private readonly Dictionary<string, TimeSpan> _replay = new(StringComparer.Ordinal);
|
||||||
|
private readonly PriorityQueue<DeadlineEntry<string>, long> _replayExpiries = new();
|
||||||
private readonly Dictionary<string, TimeSpan> _revocations = new(StringComparer.Ordinal);
|
private readonly Dictionary<string, TimeSpan> _revocations = new(StringComparer.Ordinal);
|
||||||
|
private readonly PriorityQueue<DeadlineEntry<string>, long> _revocationExpiries = new();
|
||||||
|
private readonly HashSet<string> _scheduledRevocationExpiries = new(StringComparer.Ordinal);
|
||||||
private TimeSpan? _drainDeadline;
|
private TimeSpan? _drainDeadline;
|
||||||
private TimeSpan _nextUdpMaintenance;
|
private TimeSpan _nextUdpMaintenance;
|
||||||
private long _maintenanceSweepCount;
|
private long _maintenanceSweepCount;
|
||||||
@@ -47,6 +60,22 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
|||||||
|
|
||||||
public Guid InstanceId { get; }
|
public Guid InstanceId { get; }
|
||||||
internal long MaintenanceSweepCount => Interlocked.Read(ref _maintenanceSweepCount);
|
internal long MaintenanceSweepCount => Interlocked.Read(ref _maintenanceSweepCount);
|
||||||
|
internal int ScheduledExpiryEntryCount
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
lock (_gate)
|
||||||
|
{
|
||||||
|
return _listingExpiries.Count
|
||||||
|
+ _presenceExpiries.Count
|
||||||
|
+ _attemptExpiries.Count
|
||||||
|
+ _outcomeExpiries.Count
|
||||||
|
+ _idempotencyExpiries.Count
|
||||||
|
+ _replayExpiries.Count
|
||||||
|
+ _revocationExpiries.Count;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public bool IsAvailable
|
public bool IsAvailable
|
||||||
{
|
{
|
||||||
@@ -161,10 +190,8 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
|||||||
|
|
||||||
if (_listings.Count >= _options.MaxListings
|
if (_listings.Count >= _options.MaxListings
|
||||||
|| _idempotency.Count >= _options.MaxIdempotencyEntries
|
|| _idempotency.Count >= _options.MaxIdempotencyEntries
|
||||||
|| _listings.Values.Count(entry => string.Equals(
|
|| _listingCountsByOwner.GetValueOrDefault(command.Listing.OwnerSubject)
|
||||||
entry.Definition.OwnerSubject,
|
>= command.OwnerListingLimit)
|
||||||
command.Listing.OwnerSubject,
|
|
||||||
StringComparison.Ordinal)) >= command.OwnerListingLimit)
|
|
||||||
{
|
{
|
||||||
return new(StoreResultCode.CapacityExceeded);
|
return new(StoreResultCode.CapacityExceeded);
|
||||||
}
|
}
|
||||||
@@ -183,12 +210,21 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
|||||||
WallDeadline(now, _options.LeaseLifetime),
|
WallDeadline(now, _options.LeaseLifetime),
|
||||||
version: 1);
|
version: 1);
|
||||||
_listings.Add(frozen.ListingId, entry);
|
_listings.Add(frozen.ListingId, entry);
|
||||||
|
ScheduleMutableDeadline(
|
||||||
|
_listingExpiries,
|
||||||
|
_scheduledListingExpiries,
|
||||||
|
frozen.ListingId,
|
||||||
|
entry.LeaseDeadline);
|
||||||
|
_listingCountsByOwner[frozen.OwnerSubject] =
|
||||||
|
_listingCountsByOwner.GetValueOrDefault(frozen.OwnerSubject) + 1;
|
||||||
_leases.Add(frozen.LeaseId, frozen.ListingId);
|
_leases.Add(frozen.LeaseId, frozen.ListingId);
|
||||||
_presenceHandles.Add(frozen.HostPresenceHandle, frozen.ListingId);
|
_presenceHandles.Add(frozen.HostPresenceHandle, frozen.ListingId);
|
||||||
_idempotency.Add(idempotencyKey, new(
|
IdempotencyEntry idempotency = new(
|
||||||
command.RequestFingerprint,
|
command.RequestFingerprint,
|
||||||
frozen.ListingId,
|
frozen.ListingId,
|
||||||
now + _options.IdempotencyLifetime));
|
now + _options.IdempotencyLifetime);
|
||||||
|
_idempotency.Add(idempotencyKey, idempotency);
|
||||||
|
EnqueueDeadline(_idempotencyExpiries, idempotencyKey, idempotency.Deadline);
|
||||||
return new(StoreResultCode.Success, Snapshot(entry));
|
return new(StoreResultCode.Success, Snapshot(entry));
|
||||||
}, cancellationToken);
|
}, cancellationToken);
|
||||||
|
|
||||||
@@ -346,10 +382,20 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
|||||||
return new(StoreResultCode.CapacityExceeded);
|
return new(StoreResultCode.CapacityExceeded);
|
||||||
}
|
}
|
||||||
|
|
||||||
_presence[command.Handle] = new(
|
bool isNewPresence = !_presence.ContainsKey(command.Handle);
|
||||||
|
PresenceEntry presence = new(
|
||||||
command.PublicEndpoint,
|
command.PublicEndpoint,
|
||||||
command.LocalEndpoint,
|
command.LocalEndpoint,
|
||||||
now + _options.PresenceLifetime);
|
now + _options.PresenceLifetime);
|
||||||
|
_presence[command.Handle] = presence;
|
||||||
|
if (isNewPresence)
|
||||||
|
{
|
||||||
|
ScheduleMutableDeadline(
|
||||||
|
_presenceExpiries,
|
||||||
|
_scheduledPresenceExpiries,
|
||||||
|
command.Handle,
|
||||||
|
presence.Deadline);
|
||||||
|
}
|
||||||
return new(StoreResultCode.Success, Snapshot(entry));
|
return new(StoreResultCode.Success, Snapshot(entry));
|
||||||
}, cancellationToken, eagerCleanup: false);
|
}, cancellationToken, eagerCleanup: false);
|
||||||
|
|
||||||
@@ -444,8 +490,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
|||||||
if (_attempts.Count >= _options.MaxJoinAttempts
|
if (_attempts.Count >= _options.MaxJoinAttempts
|
||||||
|| _outcomeReports.Count >= _options.MaxOutcomeReports
|
|| _outcomeReports.Count >= _options.MaxOutcomeReports
|
||||||
|| _idempotency.Count >= _options.MaxIdempotencyEntries
|
|| _idempotency.Count >= _options.MaxIdempotencyEntries
|
||||||
|| _attempts.Values.Count(entry => entry.Command.Scope == command.Scope)
|
|| _attemptCountsByScope.GetValueOrDefault(command.Scope) >= command.ScopeAttemptLimit)
|
||||||
>= command.ScopeAttemptLimit)
|
|
||||||
{
|
{
|
||||||
return new(StoreResultCode.CapacityExceeded);
|
return new(StoreResultCode.CapacityExceeded);
|
||||||
}
|
}
|
||||||
@@ -461,19 +506,25 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
|||||||
now + _options.JoinAttemptLifetime,
|
now + _options.JoinAttemptLifetime,
|
||||||
WallDeadline(now, _options.JoinAttemptLifetime));
|
WallDeadline(now, _options.JoinAttemptLifetime));
|
||||||
_attempts.Add(command.AttemptId, attempt);
|
_attempts.Add(command.AttemptId, attempt);
|
||||||
_attemptExpiries.Enqueue(
|
AddToIndex(_attemptsByListing, command.ListingId, command.AttemptId);
|
||||||
new AttemptExpiry(command.AttemptId, attempt.Deadline),
|
_attemptCountsByScope[command.Scope] =
|
||||||
attempt.Deadline.Ticks);
|
_attemptCountsByScope.GetValueOrDefault(command.Scope) + 1;
|
||||||
_outcomeReports.Add(command.AttemptId, new(
|
EnqueueDeadline(_attemptExpiries, command.AttemptId, attempt.Deadline);
|
||||||
|
OutcomeReportEntry outcome = new(
|
||||||
command.ListingId,
|
command.ListingId,
|
||||||
command.ClientSubject,
|
command.ClientSubject,
|
||||||
command.ClientCapabilityFingerprint,
|
command.ClientCapabilityFingerprint,
|
||||||
now + _options.JoinAttemptLifetime + _options.IdempotencyLifetime));
|
now + _options.JoinAttemptLifetime + _options.IdempotencyLifetime);
|
||||||
|
_outcomeReports.Add(command.AttemptId, outcome);
|
||||||
|
AddToIndex(_outcomesByListing, command.ListingId, command.AttemptId);
|
||||||
|
EnqueueDeadline(_outcomeExpiries, command.AttemptId, outcome.Deadline);
|
||||||
_attemptHandles.Add(command.MediationHandle, command.AttemptId);
|
_attemptHandles.Add(command.MediationHandle, command.AttemptId);
|
||||||
_idempotency.Add(idempotencyKey, new(
|
IdempotencyEntry idempotency = new(
|
||||||
command.RequestFingerprint,
|
command.RequestFingerprint,
|
||||||
command.AttemptId,
|
command.AttemptId,
|
||||||
now + _options.IdempotencyLifetime));
|
now + _options.IdempotencyLifetime);
|
||||||
|
_idempotency.Add(idempotencyKey, idempotency);
|
||||||
|
EnqueueDeadline(_idempotencyExpiries, idempotencyKey, idempotency.Deadline);
|
||||||
return new(StoreResultCode.Success, Snapshot(attempt));
|
return new(StoreResultCode.Success, Snapshot(attempt));
|
||||||
}, cancellationToken);
|
}, cancellationToken);
|
||||||
|
|
||||||
@@ -757,7 +808,9 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
|||||||
throw new ArgumentOutOfRangeException(nameof(consumption), "Replay lifetime exceeds the configured ceiling.");
|
throw new ArgumentOutOfRangeException(nameof(consumption), "Replay lifetime exceeds the configured ceiling.");
|
||||||
}
|
}
|
||||||
|
|
||||||
_replay.Add(key, now + lifetime);
|
TimeSpan deadline = now + lifetime;
|
||||||
|
_replay.Add(key, deadline);
|
||||||
|
EnqueueDeadline(_replayExpiries, key, deadline);
|
||||||
return new(StoreResultCode.Success, true);
|
return new(StoreResultCode.Success, true);
|
||||||
}, cancellationToken);
|
}, cancellationToken);
|
||||||
|
|
||||||
@@ -794,7 +847,24 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
|||||||
+ _presence.Count
|
+ _presence.Count
|
||||||
+ _attempts.Count
|
+ _attempts.Count
|
||||||
+ _outcomeReports.Count;
|
+ _outcomeReports.Count;
|
||||||
_revocations[subject] = now + lifetime;
|
TimeSpan deadline = now + lifetime;
|
||||||
|
bool isNewRevocation = !_revocations.TryGetValue(subject, out TimeSpan existingDeadline);
|
||||||
|
if (!isNewRevocation && existingDeadline > deadline)
|
||||||
|
{
|
||||||
|
// A repeated operator action may extend protection but cannot silently
|
||||||
|
// shorten an already-authoritative security revocation.
|
||||||
|
deadline = existingDeadline;
|
||||||
|
}
|
||||||
|
|
||||||
|
_revocations[subject] = deadline;
|
||||||
|
if (isNewRevocation)
|
||||||
|
{
|
||||||
|
ScheduleMutableDeadline(
|
||||||
|
_revocationExpiries,
|
||||||
|
_scheduledRevocationExpiries,
|
||||||
|
subject,
|
||||||
|
deadline);
|
||||||
|
}
|
||||||
SessionListingId[] listings = _listings
|
SessionListingId[] listings = _listings
|
||||||
.Where(item => string.Equals(item.Value.Definition.OwnerSubject, subject, StringComparison.Ordinal))
|
.Where(item => string.Equals(item.Value.Definition.OwnerSubject, subject, StringComparison.Ordinal))
|
||||||
.Select(static item => item.Key)
|
.Select(static item => item.Key)
|
||||||
@@ -818,7 +888,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
|||||||
}
|
}
|
||||||
foreach (JoinAttemptId attemptId in outcomeReports)
|
foreach (JoinAttemptId attemptId in outcomeReports)
|
||||||
{
|
{
|
||||||
_outcomeReports.Remove(attemptId);
|
RemoveOutcome(attemptId);
|
||||||
}
|
}
|
||||||
|
|
||||||
int activeResourcesAfter = _listings.Count
|
int activeResourcesAfter = _listings.Count
|
||||||
@@ -911,63 +981,42 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
|||||||
ClearActiveState();
|
ClearActiveState();
|
||||||
}
|
}
|
||||||
|
|
||||||
_expiryChurn += RemoveExpired(_revocations, now);
|
_expiryChurn += RemoveExpiredMutableDeadlines(
|
||||||
_expiryChurn += RemoveExpired(_replay, now);
|
_revocations,
|
||||||
string[] expiredIdempotency = _idempotency
|
_revocationExpiries,
|
||||||
.Where(item => item.Value.Deadline <= now)
|
_scheduledRevocationExpiries,
|
||||||
.Select(static item => item.Key)
|
now);
|
||||||
.ToArray();
|
_expiryChurn += RemoveExpiredDeadlines(_replay, _replayExpiries, now);
|
||||||
_expiryChurn += expiredIdempotency.Length;
|
_expiryChurn += RemoveExpiredIdempotency(now);
|
||||||
foreach (string key in expiredIdempotency)
|
_expiryChurn += RemoveExpiredPresence(now);
|
||||||
{
|
|
||||||
_idempotency.Remove(key);
|
|
||||||
}
|
|
||||||
|
|
||||||
MediationHandle[] expiredPresence = _presence
|
|
||||||
.Where(item => item.Value.Deadline <= now)
|
|
||||||
.Select(static item => item.Key)
|
|
||||||
.ToArray();
|
|
||||||
_expiryChurn += expiredPresence.Length;
|
|
||||||
foreach (MediationHandle handle in expiredPresence)
|
|
||||||
{
|
|
||||||
_presence.Remove(handle);
|
|
||||||
}
|
|
||||||
|
|
||||||
_expiryChurn += RemoveExpiredAttempts(now);
|
_expiryChurn += RemoveExpiredAttempts(now);
|
||||||
|
_expiryChurn += RemoveExpiredOutcomes(now);
|
||||||
JoinAttemptId[] expiredOutcomes = _outcomeReports
|
_expiryChurn += RemoveExpiredListings(now);
|
||||||
.Where(item => item.Value.Deadline <= now)
|
|
||||||
.Select(static item => item.Key)
|
|
||||||
.ToArray();
|
|
||||||
_expiryChurn += expiredOutcomes.Length;
|
|
||||||
foreach (JoinAttemptId attemptId in expiredOutcomes)
|
|
||||||
{
|
|
||||||
_outcomeReports.Remove(attemptId);
|
|
||||||
}
|
|
||||||
|
|
||||||
SessionListingId[] expiredListings = _listings
|
|
||||||
.Where(item => item.Value.LeaseDeadline <= now)
|
|
||||||
.Select(static item => item.Key)
|
|
||||||
.ToArray();
|
|
||||||
_expiryChurn += expiredListings.Length;
|
|
||||||
foreach (SessionListingId listingId in expiredListings)
|
|
||||||
{
|
|
||||||
RemoveListing(listingId);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void ClearActiveState()
|
private void ClearActiveState()
|
||||||
{
|
{
|
||||||
_listings.Clear();
|
_listings.Clear();
|
||||||
|
_listingCountsByOwner.Clear();
|
||||||
|
_listingExpiries.Clear();
|
||||||
|
_scheduledListingExpiries.Clear();
|
||||||
_leases.Clear();
|
_leases.Clear();
|
||||||
_presenceHandles.Clear();
|
_presenceHandles.Clear();
|
||||||
_presence.Clear();
|
_presence.Clear();
|
||||||
|
_presenceExpiries.Clear();
|
||||||
|
_scheduledPresenceExpiries.Clear();
|
||||||
_attempts.Clear();
|
_attempts.Clear();
|
||||||
|
_attemptCountsByScope.Clear();
|
||||||
|
_attemptsByListing.Clear();
|
||||||
_attemptExpiries.Clear();
|
_attemptExpiries.Clear();
|
||||||
_outcomeReports.Clear();
|
_outcomeReports.Clear();
|
||||||
|
_outcomesByListing.Clear();
|
||||||
|
_outcomeExpiries.Clear();
|
||||||
_attemptHandles.Clear();
|
_attemptHandles.Clear();
|
||||||
_idempotency.Clear();
|
_idempotency.Clear();
|
||||||
|
_idempotencyExpiries.Clear();
|
||||||
_replay.Clear();
|
_replay.Clear();
|
||||||
|
_replayExpiries.Clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void RemoveListing(SessionListingId listingId)
|
private void RemoveListing(SessionListingId listingId)
|
||||||
@@ -978,23 +1027,23 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
|||||||
}
|
}
|
||||||
|
|
||||||
_leases.Remove(listing.Definition.LeaseId);
|
_leases.Remove(listing.Definition.LeaseId);
|
||||||
|
DecrementCount(_listingCountsByOwner, listing.Definition.OwnerSubject);
|
||||||
_presenceHandles.Remove(listing.Definition.HostPresenceHandle);
|
_presenceHandles.Remove(listing.Definition.HostPresenceHandle);
|
||||||
_presence.Remove(listing.Definition.HostPresenceHandle);
|
_presence.Remove(listing.Definition.HostPresenceHandle);
|
||||||
foreach (JoinAttemptId attemptId in _attempts
|
if (_attemptsByListing.TryGetValue(listingId, out HashSet<JoinAttemptId>? attempts))
|
||||||
.Where(item => item.Value.Command.ListingId == listingId)
|
{
|
||||||
.Select(static item => item.Key)
|
foreach (JoinAttemptId attemptId in attempts.ToArray())
|
||||||
.ToArray())
|
|
||||||
{
|
{
|
||||||
RemoveAttempt(attemptId);
|
RemoveAttempt(attemptId);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (_outcomesByListing.TryGetValue(listingId, out HashSet<JoinAttemptId>? outcomes))
|
||||||
foreach (JoinAttemptId attemptId in _outcomeReports
|
|
||||||
.Where(item => item.Value.ListingId == listingId)
|
|
||||||
.Select(static item => item.Key)
|
|
||||||
.ToArray())
|
|
||||||
{
|
{
|
||||||
_outcomeReports.Remove(attemptId);
|
foreach (JoinAttemptId attemptId in outcomes.ToArray())
|
||||||
|
{
|
||||||
|
RemoveOutcome(attemptId);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1003,20 +1052,75 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
|||||||
if (_attempts.Remove(attemptId, out AttemptEntry? attempt))
|
if (_attempts.Remove(attemptId, out AttemptEntry? attempt))
|
||||||
{
|
{
|
||||||
_attemptHandles.Remove(attempt.Command.MediationHandle);
|
_attemptHandles.Remove(attempt.Command.MediationHandle);
|
||||||
|
DecrementCount(_attemptCountsByScope, attempt.Command.Scope);
|
||||||
|
RemoveFromIndex(_attemptsByListing, attempt.Command.ListingId, attemptId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void RemoveOutcome(JoinAttemptId attemptId)
|
||||||
|
{
|
||||||
|
if (_outcomeReports.Remove(attemptId, out OutcomeReportEntry? outcome))
|
||||||
|
{
|
||||||
|
RemoveFromIndex(_outcomesByListing, outcome.ListingId, attemptId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void AddToIndex<TKey>(
|
||||||
|
Dictionary<TKey, HashSet<JoinAttemptId>> index,
|
||||||
|
TKey key,
|
||||||
|
JoinAttemptId attemptId)
|
||||||
|
where TKey : notnull
|
||||||
|
{
|
||||||
|
if (!index.TryGetValue(key, out HashSet<JoinAttemptId>? values))
|
||||||
|
{
|
||||||
|
values = [];
|
||||||
|
index.Add(key, values);
|
||||||
|
}
|
||||||
|
|
||||||
|
values.Add(attemptId);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void RemoveFromIndex<TKey>(
|
||||||
|
Dictionary<TKey, HashSet<JoinAttemptId>> index,
|
||||||
|
TKey key,
|
||||||
|
JoinAttemptId attemptId)
|
||||||
|
where TKey : notnull
|
||||||
|
{
|
||||||
|
if (index.TryGetValue(key, out HashSet<JoinAttemptId>? values)
|
||||||
|
&& values.Remove(attemptId)
|
||||||
|
&& values.Count == 0)
|
||||||
|
{
|
||||||
|
index.Remove(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void DecrementCount<TKey>(Dictionary<TKey, int> counts, TKey key)
|
||||||
|
where TKey : notnull
|
||||||
|
{
|
||||||
|
int remaining = counts[key] - 1;
|
||||||
|
if (remaining == 0)
|
||||||
|
{
|
||||||
|
counts.Remove(key);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
counts[key] = remaining;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private int RemoveExpiredAttempts(TimeSpan now)
|
private int RemoveExpiredAttempts(TimeSpan now)
|
||||||
{
|
{
|
||||||
int removed = 0;
|
int removed = 0;
|
||||||
while (_attemptExpiries.TryPeek(out AttemptExpiry candidate, out long deadlineTicks)
|
while (_attemptExpiries.TryPeek(
|
||||||
|
out DeadlineEntry<JoinAttemptId> candidate,
|
||||||
|
out long deadlineTicks)
|
||||||
&& deadlineTicks <= now.Ticks)
|
&& deadlineTicks <= now.Ticks)
|
||||||
{
|
{
|
||||||
_attemptExpiries.Dequeue();
|
_attemptExpiries.Dequeue();
|
||||||
if (_attempts.TryGetValue(candidate.AttemptId, out AttemptEntry? current)
|
if (_attempts.TryGetValue(candidate.Key, out AttemptEntry? current)
|
||||||
&& current.Deadline == candidate.Deadline)
|
&& current.Deadline == candidate.Deadline)
|
||||||
{
|
{
|
||||||
RemoveAttempt(candidate.AttemptId);
|
RemoveAttempt(candidate.Key);
|
||||||
removed++;
|
removed++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1024,6 +1128,184 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
|||||||
return removed;
|
return removed;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private int RemoveExpiredListings(TimeSpan now)
|
||||||
|
{
|
||||||
|
int removed = 0;
|
||||||
|
while (_listingExpiries.TryPeek(
|
||||||
|
out DeadlineEntry<SessionListingId> candidate,
|
||||||
|
out long deadlineTicks)
|
||||||
|
&& deadlineTicks <= now.Ticks)
|
||||||
|
{
|
||||||
|
_listingExpiries.Dequeue();
|
||||||
|
_scheduledListingExpiries.Remove(candidate.Key);
|
||||||
|
if (!_listings.TryGetValue(candidate.Key, out ListingEntry? current))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (current.LeaseDeadline > now)
|
||||||
|
{
|
||||||
|
ScheduleMutableDeadline(
|
||||||
|
_listingExpiries,
|
||||||
|
_scheduledListingExpiries,
|
||||||
|
candidate.Key,
|
||||||
|
current.LeaseDeadline);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
RemoveListing(candidate.Key);
|
||||||
|
removed++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return removed;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int RemoveExpiredPresence(TimeSpan now)
|
||||||
|
{
|
||||||
|
int removed = 0;
|
||||||
|
while (_presenceExpiries.TryPeek(
|
||||||
|
out DeadlineEntry<MediationHandle> candidate,
|
||||||
|
out long deadlineTicks)
|
||||||
|
&& deadlineTicks <= now.Ticks)
|
||||||
|
{
|
||||||
|
_presenceExpiries.Dequeue();
|
||||||
|
_scheduledPresenceExpiries.Remove(candidate.Key);
|
||||||
|
if (!_presence.TryGetValue(candidate.Key, out PresenceEntry? current))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (current.Deadline > now)
|
||||||
|
{
|
||||||
|
ScheduleMutableDeadline(
|
||||||
|
_presenceExpiries,
|
||||||
|
_scheduledPresenceExpiries,
|
||||||
|
candidate.Key,
|
||||||
|
current.Deadline);
|
||||||
|
}
|
||||||
|
else if (_presence.Remove(candidate.Key))
|
||||||
|
{
|
||||||
|
removed++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return removed;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int RemoveExpiredOutcomes(TimeSpan now)
|
||||||
|
{
|
||||||
|
int removed = 0;
|
||||||
|
while (_outcomeExpiries.TryPeek(
|
||||||
|
out DeadlineEntry<JoinAttemptId> candidate,
|
||||||
|
out long deadlineTicks)
|
||||||
|
&& deadlineTicks <= now.Ticks)
|
||||||
|
{
|
||||||
|
_outcomeExpiries.Dequeue();
|
||||||
|
if (_outcomeReports.TryGetValue(candidate.Key, out OutcomeReportEntry? current)
|
||||||
|
&& current.Deadline == candidate.Deadline)
|
||||||
|
{
|
||||||
|
RemoveOutcome(candidate.Key);
|
||||||
|
removed++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return removed;
|
||||||
|
}
|
||||||
|
|
||||||
|
private int RemoveExpiredIdempotency(TimeSpan now)
|
||||||
|
{
|
||||||
|
int removed = 0;
|
||||||
|
while (_idempotencyExpiries.TryPeek(
|
||||||
|
out DeadlineEntry<string> candidate,
|
||||||
|
out long deadlineTicks)
|
||||||
|
&& deadlineTicks <= now.Ticks)
|
||||||
|
{
|
||||||
|
_idempotencyExpiries.Dequeue();
|
||||||
|
if (_idempotency.TryGetValue(candidate.Key, out IdempotencyEntry? current)
|
||||||
|
&& current.Deadline == candidate.Deadline
|
||||||
|
&& _idempotency.Remove(candidate.Key))
|
||||||
|
{
|
||||||
|
removed++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return removed;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int RemoveExpiredDeadlines<TKey>(
|
||||||
|
Dictionary<TKey, TimeSpan> entries,
|
||||||
|
PriorityQueue<DeadlineEntry<TKey>, long> expiries,
|
||||||
|
TimeSpan now)
|
||||||
|
where TKey : notnull
|
||||||
|
{
|
||||||
|
int removed = 0;
|
||||||
|
while (expiries.TryPeek(out DeadlineEntry<TKey> candidate, out long deadlineTicks)
|
||||||
|
&& deadlineTicks <= now.Ticks)
|
||||||
|
{
|
||||||
|
expiries.Dequeue();
|
||||||
|
if (entries.TryGetValue(candidate.Key, out TimeSpan current)
|
||||||
|
&& current == candidate.Deadline
|
||||||
|
&& entries.Remove(candidate.Key))
|
||||||
|
{
|
||||||
|
removed++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return removed;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int RemoveExpiredMutableDeadlines<TKey>(
|
||||||
|
Dictionary<TKey, TimeSpan> entries,
|
||||||
|
PriorityQueue<DeadlineEntry<TKey>, long> expiries,
|
||||||
|
HashSet<TKey> scheduled,
|
||||||
|
TimeSpan now)
|
||||||
|
where TKey : notnull
|
||||||
|
{
|
||||||
|
int removed = 0;
|
||||||
|
while (expiries.TryPeek(out DeadlineEntry<TKey> candidate, out long deadlineTicks)
|
||||||
|
&& deadlineTicks <= now.Ticks)
|
||||||
|
{
|
||||||
|
expiries.Dequeue();
|
||||||
|
scheduled.Remove(candidate.Key);
|
||||||
|
if (!entries.TryGetValue(candidate.Key, out TimeSpan current))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (current > now)
|
||||||
|
{
|
||||||
|
ScheduleMutableDeadline(expiries, scheduled, candidate.Key, current);
|
||||||
|
}
|
||||||
|
else if (entries.Remove(candidate.Key))
|
||||||
|
{
|
||||||
|
removed++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return removed;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void ScheduleMutableDeadline<TKey>(
|
||||||
|
PriorityQueue<DeadlineEntry<TKey>, long> expiries,
|
||||||
|
HashSet<TKey> scheduled,
|
||||||
|
TKey key,
|
||||||
|
TimeSpan deadline)
|
||||||
|
where TKey : notnull
|
||||||
|
{
|
||||||
|
if (scheduled.Add(key))
|
||||||
|
{
|
||||||
|
EnqueueDeadline(expiries, key, deadline);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void EnqueueDeadline<TKey>(
|
||||||
|
PriorityQueue<DeadlineEntry<TKey>, long> expiries,
|
||||||
|
TKey key,
|
||||||
|
TimeSpan deadline)
|
||||||
|
where TKey : notnull =>
|
||||||
|
expiries.Enqueue(new(key, deadline), deadline.Ticks);
|
||||||
|
|
||||||
private bool HandleExists(MediationHandle handle) =>
|
private bool HandleExists(MediationHandle handle) =>
|
||||||
_presenceHandles.ContainsKey(handle) || _attemptHandles.ContainsKey(handle);
|
_presenceHandles.ContainsKey(handle) || _attemptHandles.ContainsKey(handle);
|
||||||
|
|
||||||
@@ -1063,20 +1345,6 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
|||||||
IsCancelled = entry.IsCancelled,
|
IsCancelled = entry.IsCancelled,
|
||||||
};
|
};
|
||||||
|
|
||||||
private static int RemoveExpired(Dictionary<string, TimeSpan> entries, TimeSpan now)
|
|
||||||
{
|
|
||||||
string[] expired = entries
|
|
||||||
.Where(item => item.Value <= now)
|
|
||||||
.Select(static item => item.Key)
|
|
||||||
.ToArray();
|
|
||||||
foreach (string key in expired)
|
|
||||||
{
|
|
||||||
entries.Remove(key);
|
|
||||||
}
|
|
||||||
|
|
||||||
return expired.Length;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void ValidateListing(ListingDefinition listing)
|
private static void ValidateListing(ListingDefinition listing)
|
||||||
{
|
{
|
||||||
ArgumentNullException.ThrowIfNull(listing);
|
ArgumentNullException.ThrowIfNull(listing);
|
||||||
@@ -1219,7 +1487,8 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
|
|||||||
public bool IsCancelled { get; set; }
|
public bool IsCancelled { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
private readonly record struct AttemptExpiry(JoinAttemptId AttemptId, TimeSpan Deadline);
|
private readonly record struct DeadlineEntry<TKey>(TKey Key, TimeSpan Deadline)
|
||||||
|
where TKey : notnull;
|
||||||
|
|
||||||
private sealed class OutcomeReportEntry(
|
private sealed class OutcomeReportEntry(
|
||||||
SessionListingId listingId,
|
SessionListingId listingId,
|
||||||
|
|||||||
@@ -0,0 +1,91 @@
|
|||||||
|
namespace FinalFactory.Rendezvous.Capacity;
|
||||||
|
|
||||||
|
internal sealed record CapacityOptions
|
||||||
|
{
|
||||||
|
public required string Profile { get; init; }
|
||||||
|
public required int Listings { get; init; }
|
||||||
|
public required int Attempts { get; init; }
|
||||||
|
public required int Samples { get; init; }
|
||||||
|
public required int SoakCycles { get; init; }
|
||||||
|
public required int SoakSeconds { get; init; }
|
||||||
|
public string? OutputPath { get; init; }
|
||||||
|
|
||||||
|
public static CapacityOptions Parse(string[] args)
|
||||||
|
{
|
||||||
|
Dictionary<string, string> values = ParseArguments(args);
|
||||||
|
string profile = values.GetValueOrDefault("--profile") ?? "quick";
|
||||||
|
(int listings, int attempts, int samples, int soakCycles, int soakSeconds) = profile switch
|
||||||
|
{
|
||||||
|
"quick" => (1_000, 500, 100, 20, 0),
|
||||||
|
"candidate" => (25_000, 10_000, 1_000, 1_000, 300),
|
||||||
|
_ => throw new ArgumentException("--profile must be 'quick' or 'candidate'."),
|
||||||
|
};
|
||||||
|
|
||||||
|
return new()
|
||||||
|
{
|
||||||
|
Profile = profile,
|
||||||
|
Listings = ParsePositive(values.GetValueOrDefault("--listings"), listings, "--listings"),
|
||||||
|
Attempts = ParsePositive(values.GetValueOrDefault("--attempts"), attempts, "--attempts"),
|
||||||
|
Samples = ParsePositive(values.GetValueOrDefault("--samples"), samples, "--samples"),
|
||||||
|
SoakCycles = ParsePositive(
|
||||||
|
values.GetValueOrDefault("--soak-cycles"),
|
||||||
|
soakCycles,
|
||||||
|
"--soak-cycles"),
|
||||||
|
SoakSeconds = ParseNonNegative(
|
||||||
|
values.GetValueOrDefault("--soak-seconds"),
|
||||||
|
soakSeconds,
|
||||||
|
"--soak-seconds"),
|
||||||
|
OutputPath = values.GetValueOrDefault("--output"),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Dictionary<string, string> ParseArguments(string[] args)
|
||||||
|
{
|
||||||
|
HashSet<string> allowed =
|
||||||
|
[
|
||||||
|
"--profile",
|
||||||
|
"--listings",
|
||||||
|
"--attempts",
|
||||||
|
"--samples",
|
||||||
|
"--soak-cycles",
|
||||||
|
"--soak-seconds",
|
||||||
|
"--output",
|
||||||
|
];
|
||||||
|
Dictionary<string, string> values = new(StringComparer.Ordinal);
|
||||||
|
for (int index = 0; index < args.Length; index += 2)
|
||||||
|
{
|
||||||
|
string option = args[index];
|
||||||
|
if (!allowed.Contains(option))
|
||||||
|
{
|
||||||
|
throw new ArgumentException($"Unknown option: {option}.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (index == args.Length - 1
|
||||||
|
|| args[index + 1].StartsWith("--", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
throw new ArgumentException($"{option} requires a value.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!values.TryAdd(option, args[index + 1]))
|
||||||
|
{
|
||||||
|
throw new ArgumentException($"{option} may be supplied only once.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return values;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int ParsePositive(string? value, int fallback, string option) =>
|
||||||
|
value is null
|
||||||
|
? fallback
|
||||||
|
: int.TryParse(value, out int parsed) && parsed > 0
|
||||||
|
? parsed
|
||||||
|
: throw new ArgumentException($"{option} must be a positive integer.");
|
||||||
|
|
||||||
|
private static int ParseNonNegative(string? value, int fallback, string option) =>
|
||||||
|
value is null
|
||||||
|
? fallback
|
||||||
|
: int.TryParse(value, out int parsed) && parsed >= 0
|
||||||
|
? parsed
|
||||||
|
: throw new ArgumentException($"{option} must be a non-negative integer.");
|
||||||
|
}
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
namespace FinalFactory.Rendezvous.Capacity;
|
||||||
|
|
||||||
|
internal sealed record CapacityReport
|
||||||
|
{
|
||||||
|
public required int SchemaVersion { get; init; }
|
||||||
|
public required string EvidenceVersion { get; init; }
|
||||||
|
public required DateTimeOffset GeneratedAt { get; init; }
|
||||||
|
public required string Profile { get; init; }
|
||||||
|
public required RuntimeEvidence Runtime { get; init; }
|
||||||
|
public required CapacityTargets Targets { get; init; }
|
||||||
|
public required IReadOnlyList<CapacityMeasurement> Measurements { get; init; }
|
||||||
|
public required StateEvidence State { get; init; }
|
||||||
|
public required IReadOnlyList<string> Failures { get; init; }
|
||||||
|
public required bool Passed { get; init; }
|
||||||
|
}
|
||||||
|
|
||||||
|
internal sealed record RuntimeEvidence(
|
||||||
|
string Framework,
|
||||||
|
string OperatingSystem,
|
||||||
|
string Kernel,
|
||||||
|
string Architecture,
|
||||||
|
string CpuModel,
|
||||||
|
int ProcessorCount,
|
||||||
|
string CpuAffinity,
|
||||||
|
string CpuQuota,
|
||||||
|
string MemoryLimit,
|
||||||
|
string GarbageCollector,
|
||||||
|
string CommitSha,
|
||||||
|
string TreeState,
|
||||||
|
string Command,
|
||||||
|
string ImageDigest,
|
||||||
|
string WorkloadSeed,
|
||||||
|
double CapacityPhaseAverageCpuPercent,
|
||||||
|
long PeakWorkingSetBytes,
|
||||||
|
long ManagedBytesAfterCleanup);
|
||||||
|
|
||||||
|
internal sealed record CapacityTargets(
|
||||||
|
int VisibleListings,
|
||||||
|
int ActiveJoinAttempts,
|
||||||
|
int CoreControlOperationsPerSecond,
|
||||||
|
int CoreMediationOperationsPerSecond,
|
||||||
|
double CoreControlP95Milliseconds,
|
||||||
|
double CoreMediationP95Milliseconds,
|
||||||
|
double MaximumAverageCpuPercent,
|
||||||
|
long MaximumWorkingSetBytes,
|
||||||
|
int SoakCycles,
|
||||||
|
int SoakDurationSeconds);
|
||||||
|
|
||||||
|
internal sealed record CapacityMeasurement(
|
||||||
|
string Operation,
|
||||||
|
int Samples,
|
||||||
|
double P50Milliseconds,
|
||||||
|
double P95Milliseconds,
|
||||||
|
double P99Milliseconds,
|
||||||
|
double OperationsPerSecond,
|
||||||
|
double MinimumOperationsPerSecond,
|
||||||
|
double BudgetMilliseconds,
|
||||||
|
bool Passed);
|
||||||
|
|
||||||
|
internal sealed record StateEvidence(
|
||||||
|
int PeakListings,
|
||||||
|
int PeakAttempts,
|
||||||
|
int PeakReplayMarkers,
|
||||||
|
int FinalListings,
|
||||||
|
int FinalAttempts,
|
||||||
|
int FinalReplayMarkers,
|
||||||
|
long ExpiryChurn,
|
||||||
|
long MaintenanceSweeps,
|
||||||
|
int SoakCyclesCompleted,
|
||||||
|
double SoakDurationSeconds,
|
||||||
|
int SoakPeakScheduledExpiryEntries,
|
||||||
|
long SoakManagedGrowthBytes,
|
||||||
|
int SoakHandleGrowth,
|
||||||
|
bool RestartStartedEmpty,
|
||||||
|
bool OverloadWasTyped,
|
||||||
|
bool RecoverySucceeded);
|
||||||
@@ -0,0 +1,580 @@
|
|||||||
|
using System.Collections.Concurrent;
|
||||||
|
using System.Diagnostics;
|
||||||
|
using System.Runtime;
|
||||||
|
using System.Runtime.InteropServices;
|
||||||
|
using FinalFactory.Rendezvous.Contracts;
|
||||||
|
using FinalFactory.Rendezvous.Server.Observability;
|
||||||
|
using FinalFactory.Rendezvous.Server.State;
|
||||||
|
|
||||||
|
namespace FinalFactory.Rendezvous.Capacity;
|
||||||
|
|
||||||
|
internal static class CapacityRunner
|
||||||
|
{
|
||||||
|
private static readonly TenantScope Scope = new(new("space-game"), new("production"));
|
||||||
|
private const uint ProtocolVersion = 1;
|
||||||
|
|
||||||
|
public static Task<CapacityReport> RunAsync(CapacityOptions options)
|
||||||
|
{
|
||||||
|
ArgumentNullException.ThrowIfNull(options);
|
||||||
|
Process process = Process.GetCurrentProcess();
|
||||||
|
TimeSpan cpuBefore = process.TotalProcessorTime;
|
||||||
|
Stopwatch capacityPhaseTime = Stopwatch.StartNew();
|
||||||
|
List<string> failures = [];
|
||||||
|
List<CapacityMeasurement> measurements = [];
|
||||||
|
ManualClock clock = new();
|
||||||
|
InMemoryEphemeralRendezvousStore store = CreateStore(options, clock);
|
||||||
|
List<StoredListing> listings = new(options.Listings);
|
||||||
|
List<CreateJoinAttemptCommand> attempts = new(options.Attempts);
|
||||||
|
int registrationSamples = Math.Min(options.Samples, options.Listings);
|
||||||
|
int attemptSamples = Math.Min(options.Samples, options.Attempts);
|
||||||
|
|
||||||
|
for (int index = 0; index < options.Listings - registrationSamples; index++)
|
||||||
|
{
|
||||||
|
listings.Add(CreateVisibleListing(store, index));
|
||||||
|
}
|
||||||
|
|
||||||
|
measurements.Add(Measure(
|
||||||
|
"registration-and-presence",
|
||||||
|
registrationSamples,
|
||||||
|
budgetMilliseconds: 200,
|
||||||
|
minimumOperationsPerSecond: 200,
|
||||||
|
index => listings.Add(CreateVisibleListing(
|
||||||
|
store,
|
||||||
|
options.Listings - registrationSamples + index))));
|
||||||
|
|
||||||
|
measurements.Add(Measure(
|
||||||
|
"lease-renewal",
|
||||||
|
registrationSamples,
|
||||||
|
budgetMilliseconds: 200,
|
||||||
|
minimumOperationsPerSecond: 200,
|
||||||
|
index =>
|
||||||
|
{
|
||||||
|
StoredListing listing = listings[index];
|
||||||
|
StoreResult<StoredListing> renewed = store.RenewLease(new(
|
||||||
|
listing.Definition.ListingId,
|
||||||
|
listing.Definition.LeaseId,
|
||||||
|
listing.Definition.LeaseFingerprint,
|
||||||
|
listing.Definition.OwnerSubject,
|
||||||
|
listing.Version));
|
||||||
|
RequireSuccess(renewed, "renewal");
|
||||||
|
listings[index] = renewed.Value!;
|
||||||
|
}));
|
||||||
|
|
||||||
|
int browseSamples = Math.Min(options.Samples, 250);
|
||||||
|
measurements.Add(Measure(
|
||||||
|
"visible-session-browse",
|
||||||
|
browseSamples,
|
||||||
|
budgetMilliseconds: 200,
|
||||||
|
minimumOperationsPerSecond: 200,
|
||||||
|
_ => RequireSuccess(
|
||||||
|
store.BrowseVisibleListings(new(
|
||||||
|
Scope,
|
||||||
|
ProtocolVersion,
|
||||||
|
new RegionId("eu-central"),
|
||||||
|
ContractLimits.BrowserPageMaxItems,
|
||||||
|
ExcludeFull: true)),
|
||||||
|
"browse")));
|
||||||
|
|
||||||
|
for (int index = 0; index < options.Attempts - attemptSamples; index++)
|
||||||
|
{
|
||||||
|
CreateJoinAttemptCommand command = CreateAttempt(index, listings[index % listings.Count]);
|
||||||
|
RequireSuccess(store.CreateJoinAttempt(command), "join issuance");
|
||||||
|
attempts.Add(command);
|
||||||
|
}
|
||||||
|
|
||||||
|
measurements.Add(Measure(
|
||||||
|
"join-attempt-issuance",
|
||||||
|
attemptSamples,
|
||||||
|
budgetMilliseconds: 200,
|
||||||
|
minimumOperationsPerSecond: 200,
|
||||||
|
index =>
|
||||||
|
{
|
||||||
|
int sequence = options.Attempts - attemptSamples + index;
|
||||||
|
CreateJoinAttemptCommand command = CreateAttempt(
|
||||||
|
sequence,
|
||||||
|
listings[sequence % listings.Count]);
|
||||||
|
RequireSuccess(store.CreateJoinAttempt(command), "join issuance");
|
||||||
|
attempts.Add(command);
|
||||||
|
}));
|
||||||
|
|
||||||
|
int punchSamples = Math.Min(options.Samples, attempts.Count);
|
||||||
|
measurements.Add(MeasureConcurrentPunch(store, attempts, punchSamples));
|
||||||
|
EphemeralStoreSnapshot peak = store.GetSnapshot();
|
||||||
|
|
||||||
|
StoreResult<StoredJoinAttempt> overloaded = store.CreateJoinAttempt(
|
||||||
|
CreateAttempt(options.Attempts + 1, listings[^1]));
|
||||||
|
bool overloadWasTyped = peak.ActiveJoinAttempts == options.Attempts
|
||||||
|
&& overloaded.Code == StoreResultCode.CapacityExceeded;
|
||||||
|
if (!overloadWasTyped)
|
||||||
|
{
|
||||||
|
failures.Add(
|
||||||
|
$"Expected typed CapacityExceeded at {options.Attempts} active attempts, "
|
||||||
|
+ $"observed count={peak.ActiveJoinAttempts}, result={overloaded.Code}.");
|
||||||
|
}
|
||||||
|
|
||||||
|
int revocationSamples = Math.Min(Math.Max(1, options.Samples / 20), listings.Count / 2);
|
||||||
|
measurements.Add(Measure(
|
||||||
|
"principal-revocation",
|
||||||
|
revocationSamples,
|
||||||
|
budgetMilliseconds: 200,
|
||||||
|
minimumOperationsPerSecond: 50,
|
||||||
|
index => RequireSuccess(
|
||||||
|
store.RevokePrincipal(
|
||||||
|
listings[index].Definition.OwnerSubject,
|
||||||
|
TimeSpan.FromMinutes(1)),
|
||||||
|
"principal revocation")));
|
||||||
|
|
||||||
|
using (RendezvousTelemetry telemetry = new(store))
|
||||||
|
{
|
||||||
|
measurements.Add(Measure(
|
||||||
|
"telemetry-recording",
|
||||||
|
Math.Max(100, options.Samples),
|
||||||
|
budgetMilliseconds: 1,
|
||||||
|
minimumOperationsPerSecond: 10_000,
|
||||||
|
_ =>
|
||||||
|
{
|
||||||
|
telemetry.RecordHttp("browse", 200, 1);
|
||||||
|
telemetry.RecordUdp("contribution", "accepted", 1);
|
||||||
|
telemetry.RecordPairingLatency(2);
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
clock.Advance(TimeSpan.FromSeconds(61));
|
||||||
|
EphemeralStoreSnapshot? afterCoincidentExpiry = null;
|
||||||
|
measurements.Add(Measure(
|
||||||
|
"coincident-listing-attempt-expiry",
|
||||||
|
samples: 1,
|
||||||
|
budgetMilliseconds: 200,
|
||||||
|
minimumOperationsPerSecond: 0,
|
||||||
|
_ => afterCoincidentExpiry = store.GetSnapshot()));
|
||||||
|
if (afterCoincidentExpiry!.ActiveJoinAttempts != 0
|
||||||
|
|| afterCoincidentExpiry.ActiveListings != 0)
|
||||||
|
{
|
||||||
|
failures.Add("Coincident 60-second cleanup retained expired listings or join attempts.");
|
||||||
|
}
|
||||||
|
|
||||||
|
clock.Advance(TimeSpan.FromSeconds(90));
|
||||||
|
_ = store.GetSnapshot();
|
||||||
|
StoredListing recoveryListing = CreateVisibleListing(store, options.Listings + 1);
|
||||||
|
StoreResult<StoredJoinAttempt> recovered = store.CreateJoinAttempt(
|
||||||
|
CreateAttempt(options.Attempts + 2, recoveryListing));
|
||||||
|
bool recoverySucceeded = recovered.Succeeded;
|
||||||
|
if (!recoverySucceeded)
|
||||||
|
{
|
||||||
|
failures.Add($"Store did not recover after attempt expiry: {recovered.Code}.");
|
||||||
|
}
|
||||||
|
|
||||||
|
clock.Advance(TimeSpan.FromSeconds(151));
|
||||||
|
EphemeralStoreSnapshot final = store.GetSnapshot();
|
||||||
|
if (final.ActiveListings != 0
|
||||||
|
|| final.ActiveJoinAttempts != 0
|
||||||
|
|| final.ReplayMarkers != 0
|
||||||
|
|| final.IdempotencyEntries != 0
|
||||||
|
|| final.RetainedOutcomeReports != 0)
|
||||||
|
{
|
||||||
|
failures.Add("Expiry cleanup left active or retained state after every configured deadline.");
|
||||||
|
}
|
||||||
|
|
||||||
|
capacityPhaseTime.Stop();
|
||||||
|
process.Refresh();
|
||||||
|
double capacityPhaseCpuPercent = 100
|
||||||
|
* (process.TotalProcessorTime - cpuBefore).TotalSeconds
|
||||||
|
/ Math.Max(capacityPhaseTime.Elapsed.TotalSeconds * Environment.ProcessorCount, 0.000_001);
|
||||||
|
if (options.Profile == "candidate" && capacityPhaseCpuPercent > 70)
|
||||||
|
{
|
||||||
|
failures.Add(
|
||||||
|
$"Capacity-phase CPU {capacityPhaseCpuPercent:F1}% exceeded the 70% candidate budget.");
|
||||||
|
}
|
||||||
|
|
||||||
|
SoakEvidence soak = RunAcceleratedSoak(options, failures);
|
||||||
|
ManualClock restartClock = new();
|
||||||
|
EphemeralStoreSnapshot restarted = CreateStore(options, restartClock).GetSnapshot();
|
||||||
|
bool restartStartedEmpty = restarted.ActiveListings == 0
|
||||||
|
&& restarted.ActiveJoinAttempts == 0
|
||||||
|
&& restarted.ReplayMarkers == 0;
|
||||||
|
if (!restartStartedEmpty)
|
||||||
|
{
|
||||||
|
failures.Add("A restarted store did not begin empty.");
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (CapacityMeasurement measurement in measurements.Where(static item => !item.Passed))
|
||||||
|
{
|
||||||
|
failures.Add(
|
||||||
|
$"{measurement.Operation} missed its budget: p95={measurement.P95Milliseconds:F3} ms, "
|
||||||
|
+ $"rate={measurement.OperationsPerSecond:F1}/s.");
|
||||||
|
}
|
||||||
|
|
||||||
|
GC.Collect();
|
||||||
|
GC.WaitForPendingFinalizers();
|
||||||
|
GC.Collect();
|
||||||
|
process.Refresh();
|
||||||
|
long managedAfterCleanup = GC.GetTotalMemory(forceFullCollection: true);
|
||||||
|
long memoryBudget = 1_610_612_736;
|
||||||
|
if (process.PeakWorkingSet64 > memoryBudget)
|
||||||
|
{
|
||||||
|
failures.Add(
|
||||||
|
$"Peak working set {process.PeakWorkingSet64} exceeded the 1.5 GiB profile budget.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.Profile == "candidate" && Environment.ProcessorCount != 2)
|
||||||
|
{
|
||||||
|
failures.Add(
|
||||||
|
$"Candidate evidence must expose exactly two CPUs; runtime exposed "
|
||||||
|
+ $"{Environment.ProcessorCount}.");
|
||||||
|
}
|
||||||
|
|
||||||
|
CapacityReport report = new()
|
||||||
|
{
|
||||||
|
SchemaVersion = 2,
|
||||||
|
EvidenceVersion = "v2",
|
||||||
|
GeneratedAt = DateTimeOffset.UtcNow,
|
||||||
|
Profile = options.Profile,
|
||||||
|
Runtime = new(
|
||||||
|
RuntimeInformation.FrameworkDescription,
|
||||||
|
RuntimeInformation.OSDescription,
|
||||||
|
Environment.OSVersion.VersionString,
|
||||||
|
RuntimeInformation.ProcessArchitecture.ToString(),
|
||||||
|
ReadCpuModel(),
|
||||||
|
Environment.ProcessorCount,
|
||||||
|
Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_CPUSET") ?? "unrestricted",
|
||||||
|
ReadCgroupValue("/sys/fs/cgroup/cpu.max"),
|
||||||
|
ReadCgroupValue("/sys/fs/cgroup/memory.max"),
|
||||||
|
GCSettings.IsServerGC ? "server" : "workstation",
|
||||||
|
Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_COMMIT") ?? "unrecorded",
|
||||||
|
Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_TREE_STATE") ?? "unrecorded",
|
||||||
|
Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_COMMAND") ?? "unrecorded",
|
||||||
|
Environment.GetEnvironmentVariable("RENDEZVOUS_EVIDENCE_IMAGE_DIGEST") ?? "not-containerized",
|
||||||
|
"fixed-sequences-random-identifiers",
|
||||||
|
capacityPhaseCpuPercent,
|
||||||
|
process.PeakWorkingSet64,
|
||||||
|
managedAfterCleanup),
|
||||||
|
Targets = new(
|
||||||
|
options.Listings,
|
||||||
|
options.Attempts,
|
||||||
|
200,
|
||||||
|
2_000,
|
||||||
|
200,
|
||||||
|
100,
|
||||||
|
70,
|
||||||
|
memoryBudget,
|
||||||
|
options.SoakCycles,
|
||||||
|
options.SoakSeconds),
|
||||||
|
Measurements = measurements,
|
||||||
|
State = new(
|
||||||
|
peak.ActiveListings,
|
||||||
|
peak.ActiveJoinAttempts,
|
||||||
|
peak.ReplayMarkers,
|
||||||
|
final.ActiveListings,
|
||||||
|
final.ActiveJoinAttempts,
|
||||||
|
final.ReplayMarkers,
|
||||||
|
final.ExpiryChurn,
|
||||||
|
final.MaintenanceSweeps,
|
||||||
|
soak.Cycles,
|
||||||
|
soak.Duration.TotalSeconds,
|
||||||
|
soak.PeakScheduledExpiryEntries,
|
||||||
|
soak.ManagedGrowthBytes,
|
||||||
|
soak.HandleGrowth,
|
||||||
|
restartStartedEmpty,
|
||||||
|
overloadWasTyped,
|
||||||
|
recoverySucceeded),
|
||||||
|
Failures = failures,
|
||||||
|
Passed = failures.Count == 0,
|
||||||
|
};
|
||||||
|
return Task.FromResult(report);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static InMemoryEphemeralRendezvousStore CreateStore(
|
||||||
|
CapacityOptions options,
|
||||||
|
ManualClock clock) => new(
|
||||||
|
new EphemeralStoreOptions
|
||||||
|
{
|
||||||
|
MaxListings = options.Listings,
|
||||||
|
MaxPresenceBindings = options.Listings,
|
||||||
|
MaxJoinAttempts = options.Attempts,
|
||||||
|
MaxOutcomeReports = options.Attempts,
|
||||||
|
MaxIdempotencyEntries = options.Listings + options.Attempts + 1,
|
||||||
|
},
|
||||||
|
clock,
|
||||||
|
clock);
|
||||||
|
|
||||||
|
private static StoredListing CreateVisibleListing(
|
||||||
|
InMemoryEphemeralRendezvousStore store,
|
||||||
|
int sequence)
|
||||||
|
{
|
||||||
|
string owner = $"publisher-{sequence}";
|
||||||
|
SecretFingerprint leaseFingerprint = new($"lease-{sequence}");
|
||||||
|
SecretFingerprint presenceFingerprint = new($"presence-{sequence}");
|
||||||
|
ListingDefinition definition = new()
|
||||||
|
{
|
||||||
|
ListingId = new(Guid.NewGuid()),
|
||||||
|
LeaseId = new(Guid.NewGuid()),
|
||||||
|
Scope = Scope,
|
||||||
|
OwnerSubject = owner,
|
||||||
|
RegionId = new("eu-central"),
|
||||||
|
ProtocolVersion = ProtocolVersion,
|
||||||
|
BuildVersion = "1.0.0",
|
||||||
|
DisplayName = $"Capacity host {sequence}",
|
||||||
|
Visibility = ListingVisibility.Public,
|
||||||
|
TrustMode = PublisherTrustMode.ManagedDedicated,
|
||||||
|
CurrentPlayers = 1,
|
||||||
|
MaximumPlayers = 8,
|
||||||
|
Metadata = new Dictionary<string, string>(StringComparer.Ordinal),
|
||||||
|
LeaseFingerprint = leaseFingerprint,
|
||||||
|
HostPresenceHandle = new(Guid.NewGuid()),
|
||||||
|
HostPresenceFingerprint = presenceFingerprint,
|
||||||
|
CapabilityDerivationSalt = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
||||||
|
};
|
||||||
|
StoreResult<StoredListing> created = store.CreateListing(new(
|
||||||
|
$"register-{sequence}",
|
||||||
|
$"register-request-{sequence}",
|
||||||
|
definition));
|
||||||
|
RequireSuccess(created, "registration");
|
||||||
|
StoreResult<StoredListing> bound = store.BindHostPresence(new(
|
||||||
|
definition.HostPresenceHandle,
|
||||||
|
presenceFingerprint,
|
||||||
|
PublicEndpoint(10_000 + sequence % 50_000),
|
||||||
|
null));
|
||||||
|
RequireSuccess(bound, "host presence");
|
||||||
|
return bound.Value!;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static CreateJoinAttemptCommand CreateAttempt(int sequence, StoredListing listing) => new()
|
||||||
|
{
|
||||||
|
IdempotencyOwner = $"client-{sequence}",
|
||||||
|
IdempotencyKey = $"join-{sequence}",
|
||||||
|
RequestFingerprint = $"join-request-{sequence}",
|
||||||
|
ClientSubject = $"client-{sequence}",
|
||||||
|
AttemptId = new(Guid.NewGuid()),
|
||||||
|
MediationHandle = new(Guid.NewGuid()),
|
||||||
|
Scope = Scope,
|
||||||
|
ListingId = listing.Definition.ListingId,
|
||||||
|
ProtocolVersion = ProtocolVersion,
|
||||||
|
HostCapabilityFingerprint = new($"host-capability-{sequence}"),
|
||||||
|
ClientCapabilityFingerprint = new($"client-capability-{sequence}"),
|
||||||
|
ConnectionTicketFingerprint = new($"ticket-{sequence}"),
|
||||||
|
CapabilityDerivationSalt = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
||||||
|
};
|
||||||
|
|
||||||
|
private static CapacityMeasurement MeasureConcurrentPunch(
|
||||||
|
InMemoryEphemeralRendezvousStore store,
|
||||||
|
List<CreateJoinAttemptCommand> attempts,
|
||||||
|
int samples)
|
||||||
|
{
|
||||||
|
ConcurrentBag<double> latencies = [];
|
||||||
|
Stopwatch total = Stopwatch.StartNew();
|
||||||
|
Parallel.ForEach(
|
||||||
|
Enumerable.Range(0, samples),
|
||||||
|
new ParallelOptions { MaxDegreeOfParallelism = Math.Min(64, Environment.ProcessorCount * 4) },
|
||||||
|
index =>
|
||||||
|
{
|
||||||
|
CreateJoinAttemptCommand attempt = attempts[index];
|
||||||
|
Stopwatch elapsed = Stopwatch.StartNew();
|
||||||
|
RequireSuccess(store.BindAttemptEndpoint(new(
|
||||||
|
attempt.MediationHandle,
|
||||||
|
AttemptPeerRole.Host,
|
||||||
|
attempt.HostCapabilityFingerprint,
|
||||||
|
PublicEndpoint(20_000 + index % 20_000),
|
||||||
|
null)), "host punch");
|
||||||
|
RequireSuccess(store.BindAttemptEndpoint(new(
|
||||||
|
attempt.MediationHandle,
|
||||||
|
AttemptPeerRole.Client,
|
||||||
|
attempt.ClientCapabilityFingerprint,
|
||||||
|
PublicEndpoint(40_000 + index % 20_000),
|
||||||
|
null)), "client punch");
|
||||||
|
RequireSuccess(store.ConsumeIntroduction(attempt.MediationHandle), "introduction");
|
||||||
|
latencies.Add(elapsed.Elapsed.TotalMilliseconds);
|
||||||
|
});
|
||||||
|
total.Stop();
|
||||||
|
return BuildMeasurement(
|
||||||
|
"simultaneous-punch-pairing",
|
||||||
|
latencies.ToArray(),
|
||||||
|
total.Elapsed,
|
||||||
|
budgetMilliseconds: 100,
|
||||||
|
minimumOperationsPerSecond: 2_000);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static CapacityMeasurement Measure(
|
||||||
|
string operation,
|
||||||
|
int samples,
|
||||||
|
double budgetMilliseconds,
|
||||||
|
double minimumOperationsPerSecond,
|
||||||
|
Action<int> action)
|
||||||
|
{
|
||||||
|
double[] latencies = new double[samples];
|
||||||
|
Stopwatch total = Stopwatch.StartNew();
|
||||||
|
for (int index = 0; index < samples; index++)
|
||||||
|
{
|
||||||
|
long started = Stopwatch.GetTimestamp();
|
||||||
|
action(index);
|
||||||
|
latencies[index] = Stopwatch.GetElapsedTime(started).TotalMilliseconds;
|
||||||
|
}
|
||||||
|
|
||||||
|
total.Stop();
|
||||||
|
return BuildMeasurement(
|
||||||
|
operation,
|
||||||
|
latencies,
|
||||||
|
total.Elapsed,
|
||||||
|
budgetMilliseconds,
|
||||||
|
minimumOperationsPerSecond);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static CapacityMeasurement BuildMeasurement(
|
||||||
|
string operation,
|
||||||
|
double[] latencies,
|
||||||
|
TimeSpan elapsed,
|
||||||
|
double budgetMilliseconds,
|
||||||
|
double minimumOperationsPerSecond)
|
||||||
|
{
|
||||||
|
Array.Sort(latencies);
|
||||||
|
double operationsPerSecond = latencies.Length / Math.Max(elapsed.TotalSeconds, 0.000_001);
|
||||||
|
double p95 = Percentile(latencies, 0.95);
|
||||||
|
return new(
|
||||||
|
operation,
|
||||||
|
latencies.Length,
|
||||||
|
Percentile(latencies, 0.50),
|
||||||
|
p95,
|
||||||
|
Percentile(latencies, 0.99),
|
||||||
|
operationsPerSecond,
|
||||||
|
minimumOperationsPerSecond,
|
||||||
|
budgetMilliseconds,
|
||||||
|
p95 <= budgetMilliseconds && operationsPerSecond >= minimumOperationsPerSecond);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static double Percentile(double[] sorted, double percentile)
|
||||||
|
{
|
||||||
|
int index = Math.Clamp((int)Math.Ceiling(sorted.Length * percentile) - 1, 0, sorted.Length - 1);
|
||||||
|
return sorted[index];
|
||||||
|
}
|
||||||
|
|
||||||
|
private static SoakEvidence RunAcceleratedSoak(
|
||||||
|
CapacityOptions options,
|
||||||
|
List<string> failures)
|
||||||
|
{
|
||||||
|
GC.Collect();
|
||||||
|
GC.WaitForPendingFinalizers();
|
||||||
|
GC.Collect();
|
||||||
|
long managedBefore = GC.GetTotalMemory(forceFullCollection: true);
|
||||||
|
int handlesBefore = Process.GetCurrentProcess().HandleCount;
|
||||||
|
ManualClock clock = new();
|
||||||
|
CapacityOptions soakOptions = options with { Listings = 100, Attempts = 100 };
|
||||||
|
InMemoryEphemeralRendezvousStore store = CreateStore(soakOptions, clock);
|
||||||
|
Stopwatch elapsed = Stopwatch.StartNew();
|
||||||
|
int cycle = 0;
|
||||||
|
int peakScheduledExpiryEntries = 0;
|
||||||
|
while (cycle < options.SoakCycles
|
||||||
|
|| elapsed.Elapsed < TimeSpan.FromSeconds(options.SoakSeconds))
|
||||||
|
{
|
||||||
|
StoredListing listing = CreateVisibleListing(store, cycle);
|
||||||
|
for (int refresh = 0; refresh < 10; refresh++)
|
||||||
|
{
|
||||||
|
clock.Advance(TimeSpan.FromTicks(1));
|
||||||
|
listing = RequireSuccess(store.RenewLease(new(
|
||||||
|
listing.Definition.ListingId,
|
||||||
|
listing.Definition.LeaseId,
|
||||||
|
listing.Definition.LeaseFingerprint,
|
||||||
|
listing.Definition.OwnerSubject,
|
||||||
|
listing.Version)), "soak lease refresh");
|
||||||
|
listing = RequireSuccess(store.BindHostPresence(new(
|
||||||
|
listing.Definition.HostPresenceHandle,
|
||||||
|
listing.Definition.HostPresenceFingerprint,
|
||||||
|
PublicEndpoint(10_000 + cycle % 50_000),
|
||||||
|
null)), "soak presence refresh");
|
||||||
|
}
|
||||||
|
|
||||||
|
CreateJoinAttemptCommand attempt = CreateAttempt(cycle, listing);
|
||||||
|
RequireSuccess(store.CreateJoinAttempt(attempt), "soak join issuance");
|
||||||
|
RequireSuccess(store.ConsumeReplay(new("capacity-soak", $"replay-{cycle}")), "soak replay");
|
||||||
|
peakScheduledExpiryEntries = Math.Max(
|
||||||
|
peakScheduledExpiryEntries,
|
||||||
|
store.ScheduledExpiryEntryCount);
|
||||||
|
if (store.ScheduledExpiryEntryCount > 7)
|
||||||
|
{
|
||||||
|
failures.Add(
|
||||||
|
$"Mutable deadline refresh grew the expiry queue to "
|
||||||
|
+ $"{store.ScheduledExpiryEntryCount} entries for one lifecycle.");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
clock.Advance(TimeSpan.FromSeconds(151));
|
||||||
|
EphemeralStoreSnapshot snapshot = store.GetSnapshot();
|
||||||
|
if (snapshot.ActiveListings != 0
|
||||||
|
|| snapshot.ActiveJoinAttempts != 0
|
||||||
|
|| snapshot.ReplayMarkers != 0
|
||||||
|
|| snapshot.IdempotencyEntries != 0
|
||||||
|
|| snapshot.RetainedOutcomeReports != 0)
|
||||||
|
{
|
||||||
|
failures.Add($"Accelerated soak retained state after cycle {cycle}.");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
cycle++;
|
||||||
|
}
|
||||||
|
|
||||||
|
elapsed.Stop();
|
||||||
|
GC.Collect();
|
||||||
|
GC.WaitForPendingFinalizers();
|
||||||
|
GC.Collect();
|
||||||
|
long managedGrowth = GC.GetTotalMemory(forceFullCollection: true) - managedBefore;
|
||||||
|
int handleGrowth = Process.GetCurrentProcess().HandleCount - handlesBefore;
|
||||||
|
if (managedGrowth > 67_108_864)
|
||||||
|
{
|
||||||
|
failures.Add($"Soak retained {managedGrowth} managed bytes; budget is 64 MiB.");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (handleGrowth > 8)
|
||||||
|
{
|
||||||
|
failures.Add($"Soak retained {handleGrowth} process handles; budget is 8.");
|
||||||
|
}
|
||||||
|
|
||||||
|
return new(cycle, elapsed.Elapsed, peakScheduledExpiryEntries, managedGrowth, handleGrowth);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ObservedEndpoint PublicEndpoint(int port) =>
|
||||||
|
new(AddressFamilyKind.Ipv4, "203.0.113.10", port);
|
||||||
|
|
||||||
|
private static T RequireSuccess<T>(StoreResult<T> result, string operation)
|
||||||
|
{
|
||||||
|
if (!result.Succeeded)
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"{operation} failed with {result.Code}.");
|
||||||
|
}
|
||||||
|
|
||||||
|
return result.Value!;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string ReadCpuModel()
|
||||||
|
{
|
||||||
|
const string cpuInfoPath = "/proc/cpuinfo";
|
||||||
|
if (!File.Exists(cpuInfoPath))
|
||||||
|
{
|
||||||
|
return "unavailable";
|
||||||
|
}
|
||||||
|
|
||||||
|
string? model = File.ReadLines(cpuInfoPath)
|
||||||
|
.FirstOrDefault(static line => line.StartsWith("model name", StringComparison.Ordinal));
|
||||||
|
int separator = model?.IndexOf(':') ?? -1;
|
||||||
|
return separator >= 0 ? model![(separator + 1)..].Trim() : "unavailable";
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string ReadCgroupValue(string path) =>
|
||||||
|
File.Exists(path) ? File.ReadAllText(path).Trim() : "not-enforced";
|
||||||
|
|
||||||
|
private sealed class ManualClock : IWallClock, IMonotonicClock
|
||||||
|
{
|
||||||
|
public DateTimeOffset UtcNow { get; private set; } = DateTimeOffset.UtcNow;
|
||||||
|
public TimeSpan Elapsed { get; private set; }
|
||||||
|
|
||||||
|
public void Advance(TimeSpan duration)
|
||||||
|
{
|
||||||
|
UtcNow += duration;
|
||||||
|
Elapsed += duration;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private readonly record struct SoakEvidence(
|
||||||
|
int Cycles,
|
||||||
|
TimeSpan Duration,
|
||||||
|
int PeakScheduledExpiryEntries,
|
||||||
|
long ManagedGrowthBytes,
|
||||||
|
int HandleGrowth);
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net10.0</TargetFramework>
|
||||||
|
<AssemblyName>FinalFactory.Rendezvous.Capacity</AssemblyName>
|
||||||
|
<RootNamespace>FinalFactory.Rendezvous.Capacity</RootNamespace>
|
||||||
|
<IsPackable>false</IsPackable>
|
||||||
|
</PropertyGroup>
|
||||||
|
<ItemGroup>
|
||||||
|
<ProjectReference Include="../../src/FinalFactory.Rendezvous.Contracts/FinalFactory.Rendezvous.Contracts.csproj" />
|
||||||
|
<ProjectReference Include="../../src/FinalFactory.Rendezvous.Server/FinalFactory.Rendezvous.Server.csproj" />
|
||||||
|
</ItemGroup>
|
||||||
|
</Project>
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
using System.Text.Json;
|
||||||
|
|
||||||
|
namespace FinalFactory.Rendezvous.Capacity;
|
||||||
|
|
||||||
|
internal static class Program
|
||||||
|
{
|
||||||
|
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
||||||
|
{
|
||||||
|
WriteIndented = true,
|
||||||
|
};
|
||||||
|
|
||||||
|
public static async Task<int> Main(string[] args)
|
||||||
|
{
|
||||||
|
CapacityOptions options;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
options = CapacityOptions.Parse(args);
|
||||||
|
}
|
||||||
|
catch (ArgumentException exception)
|
||||||
|
{
|
||||||
|
Console.Error.WriteLine(exception.Message);
|
||||||
|
return 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
CapacityReport report = await CapacityRunner.RunAsync(options).ConfigureAwait(false);
|
||||||
|
string json = JsonSerializer.Serialize(report, JsonOptions);
|
||||||
|
Console.WriteLine(json);
|
||||||
|
if (options.OutputPath is not null)
|
||||||
|
{
|
||||||
|
string fullPath = Path.GetFullPath(options.OutputPath);
|
||||||
|
Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!);
|
||||||
|
await File.WriteAllTextAsync(fullPath, json + Environment.NewLine).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
return report.Passed ? 0 : 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
{
|
||||||
|
"version": 2,
|
||||||
|
"dependencies": {
|
||||||
|
"net10.0": {
|
||||||
|
"finalfactory.rendezvous.contracts": {
|
||||||
|
"type": "Project"
|
||||||
|
},
|
||||||
|
"finalfactory.rendezvous.server": {
|
||||||
|
"type": "Project",
|
||||||
|
"dependencies": {
|
||||||
|
"FinalFactory.Rendezvous.Contracts": "[1.0.0, )",
|
||||||
|
"LiteNetLib": "[2.1.4, )",
|
||||||
|
"Microsoft.AspNetCore.OpenApi": "[10.0.9, )"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"LiteNetLib": {
|
||||||
|
"type": "CentralTransitive",
|
||||||
|
"requested": "[2.1.4, )",
|
||||||
|
"resolved": "2.1.4",
|
||||||
|
"contentHash": "KWlxvMw3Urpqj9joD96LRiK+LC62pQNs/zkXRJc+rHnxgkGp+vV703xzDrxRmv+V1YhCFfIGzs5nrVWtREIlyA=="
|
||||||
|
},
|
||||||
|
"Microsoft.AspNetCore.OpenApi": {
|
||||||
|
"type": "CentralTransitive",
|
||||||
|
"requested": "[10.0.9, )",
|
||||||
|
"resolved": "10.0.9",
|
||||||
|
"contentHash": "1ihb8FO9cGgEK1/m3CTtT/SfnynwmiZib0W2pcDVj3KSWk/Sca4VOXEtaptKQc582zpFrzTFiwkGRCglt6H+WQ==",
|
||||||
|
"dependencies": {
|
||||||
|
"Microsoft.OpenApi": "2.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"Microsoft.OpenApi": {
|
||||||
|
"type": "CentralTransitive",
|
||||||
|
"requested": "[2.7.5, )",
|
||||||
|
"resolved": "2.7.5",
|
||||||
|
"contentHash": "0FA67RSnRM4tcBKqiqVu/HPdZ9+QOKbmeRjxRUGTCjPU4C0bmUhd97Dso7Yild5P7nOV6GxJ2xrK0Kv/O9xp0w=="
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,3 +1,6 @@
|
|||||||
|
using System.Diagnostics;
|
||||||
|
using System.Net;
|
||||||
|
using System.Net.Sockets;
|
||||||
using FinalFactory.Rendezvous.Client;
|
using FinalFactory.Rendezvous.Client;
|
||||||
using FinalFactory.Rendezvous.Contracts;
|
using FinalFactory.Rendezvous.Contracts;
|
||||||
using FinalFactory.Rendezvous.Server.Abuse;
|
using FinalFactory.Rendezvous.Server.Abuse;
|
||||||
@@ -19,6 +22,49 @@ namespace FinalFactory.Rendezvous.Tests.Client;
|
|||||||
|
|
||||||
public sealed class RendezvousClientIntegrationTests
|
public sealed class RendezvousClientIntegrationTests
|
||||||
{
|
{
|
||||||
|
[Fact]
|
||||||
|
public async Task RestartReturnsTypedUnavailabilityThenAllowsHostReregistration()
|
||||||
|
{
|
||||||
|
int port = ReserveTcpPort();
|
||||||
|
string address = $"http://127.0.0.1:{port}";
|
||||||
|
using HttpClient client = new() { BaseAddress = new Uri(address) };
|
||||||
|
RendezvousClientOptions noRetry = new()
|
||||||
|
{
|
||||||
|
MaximumSafeRetries = 0,
|
||||||
|
RequestTimeout = TimeSpan.FromSeconds(1),
|
||||||
|
};
|
||||||
|
ClientTestHost first = await ClientTestHost.StartAsync(address);
|
||||||
|
RendezvousPublisherClient publisher = new(client, noRetry);
|
||||||
|
RendezvousClientResult<PublishedSession> registered = await publisher.RegisterAsync(
|
||||||
|
CreateRegistration(100),
|
||||||
|
first.PublisherCredential);
|
||||||
|
PublishedSession initialSession = AssertSuccess(registered);
|
||||||
|
BindPresence(first, initialSession, 41_100);
|
||||||
|
RendezvousSessionBrowserClient browser = new(client, noRetry);
|
||||||
|
BrowseSessionsResponse beforeRestart = AssertSuccess(await browser.BrowseAsync(BrowseRequest()));
|
||||||
|
Assert.Equal(initialSession.ListingId, Assert.Single(beforeRestart.Items).ListingId);
|
||||||
|
Stopwatch restart = Stopwatch.StartNew();
|
||||||
|
await first.DisposeAsync();
|
||||||
|
|
||||||
|
RendezvousClientResult<BrowseSessionsResponse> unavailable = await browser.BrowseAsync(BrowseRequest());
|
||||||
|
Assert.False(unavailable.IsSuccess);
|
||||||
|
Assert.Equal(RendezvousErrorCode.ServiceUnavailable, unavailable.Error);
|
||||||
|
|
||||||
|
await using ClientTestHost second = await ClientTestHost.StartAsync(address);
|
||||||
|
RendezvousClientResult<PublishedSession> reregistered = await publisher.RegisterAsync(
|
||||||
|
CreateRegistration(101),
|
||||||
|
second.PublisherCredential);
|
||||||
|
PublishedSession replacementSession = AssertSuccess(reregistered);
|
||||||
|
Assert.NotEqual(initialSession.ListingId, replacementSession.ListingId);
|
||||||
|
BindPresence(second, replacementSession, 41_101);
|
||||||
|
BrowseSessionsResponse afterRestart = AssertSuccess(await browser.BrowseAsync(BrowseRequest()));
|
||||||
|
Assert.Equal(replacementSession.ListingId, Assert.Single(afterRestart.Items).ListingId);
|
||||||
|
Assert.DoesNotContain(afterRestart.Items, item => item.ListingId == initialSession.ListingId);
|
||||||
|
Assert.True(
|
||||||
|
restart.Elapsed < TimeSpan.FromSeconds(5),
|
||||||
|
$"Local restart and host re-registration took {restart.Elapsed}.");
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task PublisherAndBrowserClientsCompleteTheRealSessionLifecycleAndPaging()
|
public async Task PublisherAndBrowserClientsCompleteTheRealSessionLifecycleAndPaging()
|
||||||
{
|
{
|
||||||
@@ -102,6 +148,27 @@ public sealed class RendezvousClientIntegrationTests
|
|||||||
return Assert.IsAssignableFrom<T>(result.Value);
|
return Assert.IsAssignableFrom<T>(result.Value);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static void BindPresence(ClientTestHost host, PublishedSession session, int port)
|
||||||
|
{
|
||||||
|
Assert.True(host.Capabilities.TryFingerprint(
|
||||||
|
session.HostPresenceCapability,
|
||||||
|
out SecretFingerprint fingerprint));
|
||||||
|
Assert.Equal(StoreResultCode.Success, host.Store.BindHostPresence(new(
|
||||||
|
session.HostPresenceHandle,
|
||||||
|
fingerprint,
|
||||||
|
new(AddressFamilyKind.Ipv4, "203.0.113.80", port),
|
||||||
|
null)).Code);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static BrowseSessionsRequest BrowseRequest() => new()
|
||||||
|
{
|
||||||
|
GameId = new("space-game"),
|
||||||
|
EnvironmentId = new("production"),
|
||||||
|
RegionId = new("eu-central"),
|
||||||
|
ProtocolVersion = 7,
|
||||||
|
PageSize = 10,
|
||||||
|
};
|
||||||
|
|
||||||
private static RegisterSessionRequest CreateRegistration(int index) => new()
|
private static RegisterSessionRequest CreateRegistration(int index) => new()
|
||||||
{
|
{
|
||||||
IdempotencyKey = $"sdk-integration-{index}",
|
IdempotencyKey = $"sdk-integration-{index}",
|
||||||
@@ -139,7 +206,7 @@ public sealed class RendezvousClientIntegrationTests
|
|||||||
internal EphemeralCapabilityIssuer Capabilities { get; }
|
internal EphemeralCapabilityIssuer Capabilities { get; }
|
||||||
internal string PublisherCredential { get; }
|
internal string PublisherCredential { get; }
|
||||||
|
|
||||||
internal static async Task<ClientTestHost> StartAsync()
|
internal static async Task<ClientTestHost> StartAsync(string? bindAddress = null)
|
||||||
{
|
{
|
||||||
ManualRendezvousClock clock = new(ProvisioningTestData.Now);
|
ManualRendezvousClock clock = new(ProvisioningTestData.Now);
|
||||||
EphemeralStoreOptions stateOptions = new();
|
EphemeralStoreOptions stateOptions = new();
|
||||||
@@ -153,7 +220,7 @@ public sealed class RendezvousClientIntegrationTests
|
|||||||
string credential = provisioning.Credentials.Issue(principal, clock.UtcNow);
|
string credential = provisioning.Credentials.Issue(principal, clock.UtcNow);
|
||||||
|
|
||||||
WebApplicationBuilder builder = WebApplication.CreateBuilder();
|
WebApplicationBuilder builder = WebApplication.CreateBuilder();
|
||||||
builder.WebHost.UseUrls("http://127.0.0.1:0");
|
builder.WebHost.UseUrls(bindAddress ?? "http://127.0.0.1:0");
|
||||||
builder.Services.ConfigureHttpJsonOptions(static options =>
|
builder.Services.ConfigureHttpJsonOptions(static options =>
|
||||||
ContractJson.Configure(options.SerializerOptions));
|
ContractJson.Configure(options.SerializerOptions));
|
||||||
builder.Services.Configure<RouteHandlerOptions>(static options =>
|
builder.Services.Configure<RouteHandlerOptions>(static options =>
|
||||||
@@ -180,10 +247,10 @@ public sealed class RendezvousClientIntegrationTests
|
|||||||
app.MapRendezvousContractEndpoints();
|
app.MapRendezvousContractEndpoints();
|
||||||
await app.StartAsync();
|
await app.StartAsync();
|
||||||
IServer server = app.Services.GetRequiredService<IServer>();
|
IServer server = app.Services.GetRequiredService<IServer>();
|
||||||
string address = Assert.Single(server.Features.Get<IServerAddressesFeature>()!.Addresses);
|
string serviceAddress = Assert.Single(server.Features.Get<IServerAddressesFeature>()!.Addresses);
|
||||||
return new(
|
return new(
|
||||||
app,
|
app,
|
||||||
new HttpClient { BaseAddress = new Uri(address) },
|
new HttpClient { BaseAddress = new Uri(serviceAddress) },
|
||||||
store,
|
store,
|
||||||
capabilities,
|
capabilities,
|
||||||
credential);
|
credential);
|
||||||
@@ -196,4 +263,13 @@ public sealed class RendezvousClientIntegrationTests
|
|||||||
await _application.DisposeAsync();
|
await _application.DisposeAsync();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static int ReserveTcpPort()
|
||||||
|
{
|
||||||
|
TcpListener listener = new(IPAddress.Loopback, 0);
|
||||||
|
listener.Start();
|
||||||
|
int port = ((IPEndPoint)listener.LocalEndpoint).Port;
|
||||||
|
listener.Stop();
|
||||||
|
return port;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -61,6 +61,32 @@ public sealed class ProductionProcessTests
|
|||||||
Assert.InRange(shutdown.Elapsed, TimeSpan.FromMilliseconds(700), TimeSpan.FromSeconds(5));
|
Assert.InRange(shutdown.Elapsed, TimeSpan.FromMilliseconds(700), TimeSpan.FromSeconds(5));
|
||||||
AssertTcpPortIsReleased(httpPort);
|
AssertTcpPortIsReleased(httpPort);
|
||||||
AssertUdpPortIsReleased(udpPort);
|
AssertUdpPortIsReleased(udpPort);
|
||||||
|
|
||||||
|
process.Dispose();
|
||||||
|
process = null;
|
||||||
|
Stopwatch replacementReady = Stopwatch.StartNew();
|
||||||
|
ProcessStartInfo replacementInfo = CreateStartInfo(httpPort, udpPort, secretPath);
|
||||||
|
process = Process.Start(replacementInfo)
|
||||||
|
?? throw new InvalidOperationException("The replacement production process did not start.");
|
||||||
|
Task<string> replacementOutput = process.StandardOutput.ReadToEndAsync();
|
||||||
|
Task<string> replacementError = process.StandardError.ReadToEndAsync();
|
||||||
|
await WaitForReadyAsync(httpPort, process, TimeSpan.FromSeconds(15));
|
||||||
|
Assert.True(
|
||||||
|
replacementReady.Elapsed < TimeSpan.FromSeconds(15),
|
||||||
|
$"Replacement readiness took {replacementReady.Elapsed}.");
|
||||||
|
AssertUdpPortIsBound(udpPort);
|
||||||
|
|
||||||
|
await SendSigtermAsync(process);
|
||||||
|
using CancellationTokenSource replacementTimeout = new(TimeSpan.FromSeconds(6));
|
||||||
|
await process.WaitForExitAsync(replacementTimeout.Token);
|
||||||
|
string replacementFinalOutput = await replacementOutput;
|
||||||
|
string replacementFinalError = await replacementError;
|
||||||
|
Assert.True(
|
||||||
|
process.ExitCode == 0,
|
||||||
|
$"Replacement exited with {process.ExitCode}. "
|
||||||
|
+ $"stdout: {replacementFinalOutput} stderr: {replacementFinalError}");
|
||||||
|
AssertTcpPortIsReleased(httpPort);
|
||||||
|
AssertUdpPortIsReleased(udpPort);
|
||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
{
|
{
|
||||||
@@ -79,6 +105,97 @@ public sealed class ProductionProcessTests
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ProductionTransportSoakKeepsHandlesMemoryAndSocketsBounded()
|
||||||
|
{
|
||||||
|
if (!OperatingSystem.IsLinux())
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
int httpPort = ReserveTcpPort();
|
||||||
|
int udpPort = ReserveUdpPort();
|
||||||
|
string secretPath = Path.Combine(
|
||||||
|
Path.GetTempPath(),
|
||||||
|
$"rendezvous-transport-soak-secret-{Guid.NewGuid():N}");
|
||||||
|
await File.WriteAllBytesAsync(secretPath, RandomNumberGenerator.GetBytes(32));
|
||||||
|
Process? process = null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
process = Process.Start(CreateStartInfo(httpPort, udpPort, secretPath))
|
||||||
|
?? throw new InvalidOperationException("The production soak process did not start.");
|
||||||
|
Task<string> standardOutput = process.StandardOutput.ReadToEndAsync();
|
||||||
|
Task<string> standardError = process.StandardError.ReadToEndAsync();
|
||||||
|
await WaitForReadyAsync(httpPort, process, TimeSpan.FromSeconds(10));
|
||||||
|
process.Refresh();
|
||||||
|
int baselineHandles = process.HandleCount;
|
||||||
|
long baselineWorkingSet = process.WorkingSet64;
|
||||||
|
int peakHandles = baselineHandles;
|
||||||
|
byte[] invalidDatagram = RandomNumberGenerator.GetBytes(64);
|
||||||
|
IPEndPoint udpEndpoint = new(IPAddress.Loopback, udpPort);
|
||||||
|
using HttpClient client = new() { Timeout = TimeSpan.FromSeconds(1) };
|
||||||
|
using UdpClient udp = new();
|
||||||
|
Stopwatch soak = Stopwatch.StartNew();
|
||||||
|
int cycles = 0;
|
||||||
|
int accepted = 0;
|
||||||
|
int shed = 0;
|
||||||
|
while (soak.Elapsed < TimeSpan.FromSeconds(10))
|
||||||
|
{
|
||||||
|
using HttpResponseMessage response = await client.GetAsync(
|
||||||
|
$"http://127.0.0.1:{httpPort}/health/live");
|
||||||
|
if (response.StatusCode == HttpStatusCode.OK)
|
||||||
|
{
|
||||||
|
accepted++;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode);
|
||||||
|
shed++;
|
||||||
|
}
|
||||||
|
|
||||||
|
await udp.SendAsync(invalidDatagram, udpEndpoint);
|
||||||
|
cycles++;
|
||||||
|
if (cycles % 100 == 0)
|
||||||
|
{
|
||||||
|
process.Refresh();
|
||||||
|
peakHandles = Math.Max(peakHandles, process.HandleCount);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.True(cycles >= 100, $"Transport soak completed only {cycles} cycles.");
|
||||||
|
Assert.True(accepted > 0, "Transport soak never admitted a health request.");
|
||||||
|
Assert.True(shed > 0, "Transport soak never exercised typed HTTP load shedding.");
|
||||||
|
await Task.Delay(TimeSpan.FromSeconds(2));
|
||||||
|
using (HttpResponseMessage recovered = await client.GetAsync(
|
||||||
|
$"http://127.0.0.1:{httpPort}/health/live"))
|
||||||
|
{
|
||||||
|
Assert.Equal(HttpStatusCode.OK, recovered.StatusCode);
|
||||||
|
}
|
||||||
|
|
||||||
|
process.Refresh();
|
||||||
|
Assert.InRange(peakHandles, 0, baselineHandles + 32);
|
||||||
|
Assert.InRange(process.HandleCount, 0, baselineHandles + 16);
|
||||||
|
Assert.InRange(process.WorkingSet64, 0, baselineWorkingSet + 67_108_864);
|
||||||
|
AssertUdpPortIsBound(udpPort);
|
||||||
|
|
||||||
|
await SendSigtermAsync(process);
|
||||||
|
using CancellationTokenSource shutdownTimeout = new(TimeSpan.FromSeconds(6));
|
||||||
|
await process.WaitForExitAsync(shutdownTimeout.Token);
|
||||||
|
string output = await standardOutput;
|
||||||
|
string error = await standardError;
|
||||||
|
Assert.True(
|
||||||
|
process.ExitCode == 0,
|
||||||
|
$"Transport soak process failed. stdout: {output} stderr: {error}");
|
||||||
|
AssertTcpPortIsReleased(httpPort);
|
||||||
|
AssertUdpPortIsReleased(udpPort);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
await StopProcessTreeAsync(process);
|
||||||
|
File.Delete(secretPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task DocumentedSmokeScriptReachesHttpAndAuthenticatedUdpFlow()
|
public async Task DocumentedSmokeScriptReachesHttpAndAuthenticatedUdpFlow()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -411,6 +411,92 @@ public sealed class InMemoryEphemeralRendezvousStoreTests
|
|||||||
Assert.True(fixture.Store.CreateJoinAttempt(attempt).Succeeded);
|
Assert.True(fixture.Store.CreateJoinAttempt(attempt).Succeeded);
|
||||||
Assert.Equal(StoreResultCode.CapacityExceeded, fixture.Store.CreateJoinAttempt(
|
Assert.Equal(StoreResultCode.CapacityExceeded, fixture.Store.CreateJoinAttempt(
|
||||||
fixture.AttemptCommand(listing, "client-quota-2") with { ScopeAttemptLimit = 1 }).Code);
|
fixture.AttemptCommand(listing, "client-quota-2") with { ScopeAttemptLimit = 1 }).Code);
|
||||||
|
|
||||||
|
fixture.Clock.Advance(TimeSpan.FromSeconds(30));
|
||||||
|
Assert.True(fixture.Store.BindHostPresence(new(
|
||||||
|
first.Listing.HostPresenceHandle,
|
||||||
|
first.Listing.HostPresenceFingerprint,
|
||||||
|
EphemeralStateFixture.PublicEndpoint(42_101),
|
||||||
|
null)).Succeeded);
|
||||||
|
Assert.True(fixture.Store.CreateJoinAttempt(
|
||||||
|
fixture.AttemptCommand(listing, "client-quota-3") with { ScopeAttemptLimit = 1 }).Succeeded);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void RemovingAListingReleasesItsConstantTimeOwnerQuota()
|
||||||
|
{
|
||||||
|
EphemeralStateFixture fixture = new();
|
||||||
|
CreateListingCommand first = fixture.ListingCommand(owner: "publisher-quota") with
|
||||||
|
{
|
||||||
|
OwnerListingLimit = 1,
|
||||||
|
};
|
||||||
|
CreateListingCommand second = fixture.ListingCommand(owner: "publisher-quota") with
|
||||||
|
{
|
||||||
|
OwnerListingLimit = 1,
|
||||||
|
};
|
||||||
|
StoreResult<StoredListing> created = fixture.Store.CreateListing(first);
|
||||||
|
Assert.True(created.Succeeded);
|
||||||
|
Assert.Equal(StoreResultCode.CapacityExceeded, fixture.Store.CreateListing(second).Code);
|
||||||
|
|
||||||
|
Assert.True(fixture.Store.DeleteListing(new(
|
||||||
|
first.Listing.ListingId,
|
||||||
|
first.Listing.LeaseId,
|
||||||
|
first.Listing.LeaseFingerprint,
|
||||||
|
first.Listing.OwnerSubject)).Succeeded);
|
||||||
|
Assert.True(fixture.Store.CreateListing(second).Succeeded);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void RepeatedMutableDeadlineRefreshesKeepOneScheduledEntryPerKey()
|
||||||
|
{
|
||||||
|
EphemeralStateFixture fixture = new();
|
||||||
|
StoredListing listing = fixture.CreateVisibleListing(out _);
|
||||||
|
|
||||||
|
for (int iteration = 0; iteration < 10_000; iteration++)
|
||||||
|
{
|
||||||
|
fixture.Clock.Advance(TimeSpan.FromTicks(1));
|
||||||
|
StoreResult<StoredListing> renewed = fixture.Store.RenewLease(new(
|
||||||
|
listing.Definition.ListingId,
|
||||||
|
listing.Definition.LeaseId,
|
||||||
|
listing.Definition.LeaseFingerprint,
|
||||||
|
listing.Definition.OwnerSubject,
|
||||||
|
listing.Version));
|
||||||
|
Assert.True(renewed.Succeeded, $"Renewal {iteration} failed with {renewed.Code}.");
|
||||||
|
listing = renewed.Value!;
|
||||||
|
StoreResult<StoredListing> presence = fixture.Store.BindHostPresence(new(
|
||||||
|
listing.Definition.HostPresenceHandle,
|
||||||
|
listing.Definition.HostPresenceFingerprint,
|
||||||
|
EphemeralStateFixture.PublicEndpoint(42_200),
|
||||||
|
null));
|
||||||
|
Assert.True(presence.Succeeded, $"Presence {iteration} failed with {presence.Code}.");
|
||||||
|
StoreResult<int> revoked = fixture.Store.RevokePrincipal(
|
||||||
|
"repeated-revocation",
|
||||||
|
TimeSpan.FromMinutes(1));
|
||||||
|
Assert.True(revoked.Succeeded, $"Revocation {iteration} failed with {revoked.Code}.");
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.Equal(4, fixture.Store.ScheduledExpiryEntryCount);
|
||||||
|
fixture.Clock.Advance(TimeSpan.FromSeconds(19));
|
||||||
|
Assert.True(fixture.Store.GetListing(listing.Definition.ListingId, true).Succeeded);
|
||||||
|
Assert.Equal(4, fixture.Store.ScheduledExpiryEntryCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void RepeatedPrincipalRevocationCanExtendButCannotShortenProtection()
|
||||||
|
{
|
||||||
|
EphemeralStateFixture fixture = new();
|
||||||
|
const string subject = "protected-publisher";
|
||||||
|
|
||||||
|
Assert.True(fixture.Store.RevokePrincipal(subject, TimeSpan.FromMinutes(10)).Succeeded);
|
||||||
|
fixture.Clock.Advance(TimeSpan.FromSeconds(1));
|
||||||
|
Assert.True(fixture.Store.RevokePrincipal(subject, TimeSpan.FromSeconds(1)).Succeeded);
|
||||||
|
fixture.Clock.Advance(TimeSpan.FromSeconds(2));
|
||||||
|
Assert.Equal(
|
||||||
|
StoreResultCode.Revoked,
|
||||||
|
fixture.Store.CreateListing(fixture.ListingCommand(owner: subject)).Code);
|
||||||
|
|
||||||
|
fixture.Clock.Advance(TimeSpan.FromSeconds(597));
|
||||||
|
Assert.True(fixture.Store.CreateListing(fixture.ListingCommand(owner: subject)).Succeeded);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
|
|||||||
Reference in New Issue
Block a user