From 2ff7cd6d9d02b0364cb0a57c36633438ea45ef3a Mon Sep 17 00:00:00 2001 From: KyuubiYoru Date: Thu, 16 Jul 2026 11:50:53 +0200 Subject: [PATCH] test(integration): add deterministic NAT topology harness (#14) --- .gitea/workflows/ci.yml | 53 + README.md | 3 + docs/integration/test-client.md | 3 + docs/integration/topology-harness.md | 91 ++ .../Server/UdpMediatorServiceTests.cs | 168 ++- .../TestClientProcessIntegrationTests.cs | 1085 ++++++++++++++++- 6 files changed, 1308 insertions(+), 95 deletions(-) create mode 100644 docs/integration/topology-harness.md diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index f75b738..552f80d 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -35,3 +35,56 @@ jobs: - name: Test run: dotnet test Rendezvous.slnx --configuration Release --no-build + + - name: Test privileged Linux namespace topology when available + shell: bash + run: | + set -euo pipefail + probe="rendezvous-probe-$$" + suffix="$(( $$ % 100000 ))" + bridge="rvb${suffix}" + veth_root="rvr${suffix}" + veth_peer="rvp${suffix}" + cleanup_probe() { + if [[ -n "$veth_root" ]]; then + ip link delete "$veth_root" >/dev/null 2>&1 || true + fi + if [[ -n "$bridge" ]]; then + ip link delete "$bridge" >/dev/null 2>&1 || true + fi + if [[ -n "$probe" ]]; then + ip netns delete "$probe" >/dev/null 2>&1 || true + fi + } + trap cleanup_probe EXIT + if command -v ip >/dev/null 2>&1 \ + && command -v iptables >/dev/null 2>&1 \ + && command -v sysctl >/dev/null 2>&1 \ + && ip netns add "$probe" 2>/dev/null \ + && ip link add "$bridge" type bridge \ + && ip link add "$veth_root" type veth peer name "$veth_peer" \ + && ip link set "$veth_root" master "$bridge" \ + && ip link set "$veth_peer" netns "$probe" \ + && ip netns exec "$probe" sysctl -q -w net.ipv4.ip_forward=1 \ + && ip netns exec "$probe" iptables -t nat -A POSTROUTING -o "$veth_peer" -j MASQUERADE \ + && ip netns exec "$probe" iptables -A FORWARD -i "$veth_peer" -o lo \ + -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT; then + ip link delete "$veth_root" + veth_root="" + ip link delete "$bridge" + bridge="" + ip netns delete "$probe" + probe="" + results="${RUNNER_TEMP:-/tmp}/rendezvous-netns-results" + mkdir -p "$results" + RENDEZVOUS_RUN_NETNS_TESTS=1 dotnet test Rendezvous.slnx \ + --configuration Release \ + --no-build \ + --filter FullyQualifiedName~PrivilegedLinuxNatNamespacesCompleteDirectTrafficAcrossSeparateObservedEndpoints \ + --logger "trx;LogFileName=netns.trx" \ + --results-directory "$results" + grep -q 'testName="[^"]*\.PrivilegedLinuxNatNamespacesCompleteDirectTrafficAcrossSeparateObservedEndpoints"' \ + "$results/netns.trx" + else + echo "Network namespaces/NAT tooling unavailable; deterministic loopback topology remains the required gate." + fi diff --git a/README.md b/README.md index 12ea614..ec8eff5 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,9 @@ Tenant policy, publisher/operator principals, and production key custody are defined in [game provisioning and signing-key lifecycle](docs/security/provisioning.md). The scriptable host/browser/join diagnostic and its stable automation contract are documented in the [TestClient integration guide](docs/integration/test-client.md). +The always-on three-party scenarios, optional Linux namespace topology, and +simulation limits are documented in the +[deterministic topology harness](docs/integration/topology-harness.md). ## Development diff --git a/docs/integration/test-client.md b/docs/integration/test-client.md index a60ed04..5ac8d69 100644 --- a/docs/integration/test-client.md +++ b/docs/integration/test-client.md @@ -7,6 +7,9 @@ It exists for integration development, CI smoke checks, deployment verification, and operator diagnosis. It is intentionally not a production game client, game server, matchmaking UI, or relay. +The automated scenario matrix, privileged Linux namespace run, and topology +limitations are documented in the [deterministic topology harness](topology-harness.md). + ## Prerequisites Start a configured Rendezvous service and note both its HTTP base URL and UDP diff --git a/docs/integration/topology-harness.md b/docs/integration/topology-harness.md new file mode 100644 index 0000000..316c69a --- /dev/null +++ b/docs/integration/topology-harness.md @@ -0,0 +1,91 @@ +# Deterministic topology harness + +Issue #14 is verified at three layers. The layers are deliberately separate so +the always-on gate remains deterministic while privileged CI workers can add a +stronger operating-system topology without overstating what local emulation +proves about the public Internet. + +## Always-on public-process gate + +`TestClientProcessIntegrationTests` launches the built server and the same +`FinalFactory.Rendezvous.TestClient` executable shipped to operators. Every +child process uses `--script --json`, dynamic HTTP and UDP ports, bounded +state-driven waits, and enforced process-tree cleanup. + +The suite proves: + +| Scenario | Required observation | +| --- | --- | +| Three-party happy path | register, presence-ready, browse, authorize, punch, authenticated LiteNetLib connection, direct ping/echo/ack/completion traffic, outcome report, disconnect, deregister | +| Same-LAN candidate | the connected peer is reported as `loopback` or `private`, never inferred merely from an introduction callback | +| Empty and missing selection | browse exits `11`; exact missing lookup exits `10` | +| Wrong tenant/protocol | no listing is returned for an incompatible protocol; exact joins with either mismatch fail before `join.punch` | +| Traversal timeout | an unreachable mediator produces typed `PunchTimedOut`, exits `12`, advertises the configured dedicated fallback, and never connects to it | +| Caller cancellation | POSIX `SIGINT` exits `130`, deregisters the listing, and removes it from public lookup | +| Abrupt host loss | the listing disappears after the presence window and before its lease expires; public exact lookup intentionally reports `NotFound` | +| Bounded host without a peer | exits `13` and still deregisters | + +Captured output is parsed as the stable JSON v1 event schema. Publisher +credentials and signing-key material are checked against all captured output. +The direct traffic payload is handled only by the caller-owned host and client +LiteNetLib managers; the HTTP service and mediator do not implement or observe +the echo protocol. + +Run the always-on scenarios with: + +```bash +dotnet test Rendezvous.slnx --configuration Release --no-build \ + --filter FullyQualifiedName~TestClientProcessIntegrationTests +``` + +## Deterministic protocol and adverse-state gate + +The following real service-boundary tests cover conditions that a public CLI +cannot safely manufacture by accepting raw capabilities or tickets: + +| Scenario | Test evidence | +| --- | --- | +| Same-NAT private candidates | `NatMediationProcessorTests.MatchedPeersReceiveOneIntroductionAndSameNatPrivateCandidates` | +| Separate observed endpoints | `NatMediationProcessorTests.DifferentNatsAndInvalidLocalClaimsExposeOnlyObservedPublicEndpoints` | +| One-time introduction and replay | `InMemoryEphemeralRendezvousStoreTests.AttemptCapabilitiesAndIntroductionAreOneTime` | +| Direct ticket replay | `RendezvousCoordinatorIntegrationTests.CallerOwnedManagersCompleteAuthenticatedDirectConnectionAndRejectTicketReplay` | +| Wrong tenant/protocol and stale presence | `InMemoryEphemeralRendezvousStoreTests.JoinRequiresExactScopeProtocolAndFreshHostPresence` | +| Cancellation and late callbacks | `RendezvousCoordinatorBehaviorTests.CancellationCompletesExactlyOnceAndLateCallbacksCannotReopenTheAttempt` | +| Mediator restart | both cases of `UdpMediatorServiceTests.NativeLiteNetLibRequestsIntroduceTheAuthorizedPair`; the restarted case rebinds the same UDP port and completes a native LiteNetLib introduction | + +These tests use fake monotonic clocks or state predicates where expiry and race +ordering matter. They do not use fixed sleeps as proof of state. + +## Privileged Linux namespace gate + +When a Linux CI worker can create network namespaces, the workflow sets +`RENDEZVOUS_RUN_NETNS_TESTS=1` and reruns +`PrivilegedLinuxNatNamespacesCompleteDirectTrafficAcrossSeparateObservedEndpoints`. +The test creates a temporary WAN bridge, an isolated service namespace, two NAT +router namespaces, and isolated host/client LAN namespaces. Each NAT has its own +inside subnet and WAN address. Linux forwarding plus per-router MASQUERADE rules +force the service to observe separate translated endpoints; the public TestClient +processes must then complete authenticated direct traffic through those mappings +using the public candidate. Namespaces, rules, veth pairs, bridge, processes, and +sockets are removed in bounded async-disposal paths. A cleanup failure fails the +test. + +If `ip netns add`/`iptables` is unavailable or the worker lacks `CAP_NET_ADMIN`, +CI records the limitation and keeps the always-on loopback suite as the required gate. +To request the privileged run explicitly: + +```bash +RENDEZVOUS_RUN_NETNS_TESTS=1 dotnet test Rendezvous.slnx \ + --configuration Release --no-build \ + --filter FullyQualifiedName~PrivilegedLinuxNatNamespacesCompleteDirectTrafficAcrossSeparateObservedEndpoints +``` + +## What this does not prove + +Loopback, MASQUERADE, and namespace routing cannot reproduce every consumer router, +carrier-grade NAT, firewall, IPv6 transition mechanism, symmetric NAT mapping, +or real-world packet-loss pattern. The separate-observed-endpoint processor +test proves that untrusted private claims are excluded and public candidates are +selected; it is not presented as universal Internet traversal proof. Real +network canaries and measured production readiness remain the scope of issue +#23. diff --git a/tests/FinalFactory.Rendezvous.Tests/Server/UdpMediatorServiceTests.cs b/tests/FinalFactory.Rendezvous.Tests/Server/UdpMediatorServiceTests.cs index bd9a437..dd0ca0d 100644 --- a/tests/FinalFactory.Rendezvous.Tests/Server/UdpMediatorServiceTests.cs +++ b/tests/FinalFactory.Rendezvous.Tests/Server/UdpMediatorServiceTests.cs @@ -86,8 +86,10 @@ public sealed class UdpMediatorServiceTests await service.StopAsync(timeout.Token); } - [Fact] - public async Task NativeLiteNetLibRequestsIntroduceTheAuthorizedPair() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task NativeLiteNetLibRequestsIntroduceTheAuthorizedPair(bool restartMediator) { using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(5)); using JoinAttemptFixture fixture = new(); @@ -103,18 +105,6 @@ public sealed class UdpMediatorServiceTests fixture.Sessions.Store, fixture.Sessions.Capabilities, fixture.Service); - using UdpMediatorService service = new( - Options.Create(new UdpMediatorOptions - { - ListenAddress = IPAddress.Loopback.ToString(), - Port = 0, - MaxDatagramsPerPoll = 8, - PollIntervalMilliseconds = 1, - }), - NullLogger.Instance, - processor); - await service.StartAsync(timeout.Token); - EventBasedNetListener hostListener = new(); EventBasedNetListener clientListener = new(); NetManager host = new(hostListener) { NatPunchEnabled = true }; @@ -127,53 +117,141 @@ public sealed class UdpMediatorServiceTests clientPunch.NatIntroductionSuccess += (_, _, ticket) => clientTickets.Add(ticket); host.NatPunchModule.Init(hostPunch); client.NatPunchModule.Init(clientPunch); + UdpMediatorService? service = null; + bool serviceStarted = false; try { + service = CreateMediator(processor, port: 0); + await service.StartAsync(timeout.Token); + serviceStarted = true; Assert.True(host.Start(0)); Assert.True(client.Start(0)); - IPEndPoint mediator = Assert.IsType(service.LocalEndpoint); - host.NatPunchModule.SendNatIntroduceRequest( - mediator, - NatPunchRequestTokenCodec.Encode( - NatPunchPeerRole.Host, - created.MediationHandle, - hostAttempt.HostPunchCapability)); - client.NatPunchModule.SendNatIntroduceRequest( - mediator, - NatPunchRequestTokenCodec.Encode( - NatPunchPeerRole.Client, - created.MediationHandle, - created.ClientPunchCapability)); - - while ((hostTickets.Count == 0 || clientTickets.Count == 0) - && !timeout.IsCancellationRequested) + if (restartMediator) { - host.PollEvents(); - host.NatPunchModule.PollEvents(); - client.PollEvents(); - client.NatPunchModule.PollEvents(); - await Task.Delay(5, timeout.Token); + await AssertNativeIntroductionAsync( + service, + host, + client, + hostTickets, + clientTickets, + created, + hostAttempt, + expectedCount: 1, + cancellationToken: timeout.Token); + created = fixture.Create(registration.ListingId, "native-litenet-after-restart"); + hostAttempt = fixture.Service.BrowseForHost( + registration.ListingId, + ContractLimits.ContractVersion, + registration.LeaseToken, + ContractLimits.BrowserPageMaxItems, + null).Value!.Items.Single(item => item.AttemptId == created.AttemptId); + int boundPort = Assert.IsType(service.LocalEndpoint).Port; + await service.StopAsync(timeout.Token); + serviceStarted = false; + service.Dispose(); + service = null; + service = CreateMediator(processor, boundPort); + await service.StartAsync(timeout.Token); + serviceStarted = true; + Assert.Equal(boundPort, Assert.IsType(service.LocalEndpoint).Port); } - string hostTicket = Assert.Single(hostTickets.Distinct(StringComparer.Ordinal)); - string clientTicket = Assert.Single(clientTickets.Distinct(StringComparer.Ordinal)); - Assert.Equal(hostTicket, clientTicket); - Assert.True(NatIntroductionTokenCodec.TryDecode( - hostTicket, - out NatIntroductionToken? introduction)); - Assert.NotNull(introduction); - Assert.Equal(created.AttemptId, introduction.AttemptId); - Assert.Equal(43, introduction.ConnectionTicket.Length); + await AssertNativeIntroductionAsync( + service, + host, + client, + hostTickets, + clientTickets, + created, + hostAttempt, + expectedCount: restartMediator ? 2 : 1, + cancellationToken: timeout.Token); } finally { host.Stop(); client.Stop(); - await service.StopAsync(CancellationToken.None); + if (service is not null) + { + try + { + if (serviceStarted) + { + using CancellationTokenSource cleanup = new(TimeSpan.FromSeconds(5)); + await service.StopAsync(cleanup.Token); + } + } + finally + { + service.Dispose(); + } + } } } + private static async Task AssertNativeIntroductionAsync( + UdpMediatorService service, + NetManager host, + NetManager client, + List hostTickets, + List clientTickets, + CreateJoinAttemptResponse created, + HostJoinAttempt hostAttempt, + int expectedCount, + CancellationToken cancellationToken) + { + IPEndPoint mediator = Assert.IsType(service.LocalEndpoint); + host.NatPunchModule.SendNatIntroduceRequest( + mediator, + NatPunchRequestTokenCodec.Encode( + NatPunchPeerRole.Host, + created.MediationHandle, + hostAttempt.HostPunchCapability)); + client.NatPunchModule.SendNatIntroduceRequest( + mediator, + NatPunchRequestTokenCodec.Encode( + NatPunchPeerRole.Client, + created.MediationHandle, + created.ClientPunchCapability)); + + while ((hostTickets.Distinct(StringComparer.Ordinal).Count() < expectedCount + || clientTickets.Distinct(StringComparer.Ordinal).Count() < expectedCount) + && !cancellationToken.IsCancellationRequested) + { + host.PollEvents(); + host.NatPunchModule.PollEvents(); + client.PollEvents(); + client.NatPunchModule.PollEvents(); + await Task.Delay(5, cancellationToken); + } + + List distinctHostTickets = hostTickets.Distinct(StringComparer.Ordinal).ToList(); + List distinctClientTickets = clientTickets.Distinct(StringComparer.Ordinal).ToList(); + Assert.Equal(expectedCount, distinctHostTickets.Count); + Assert.Equal(expectedCount, distinctClientTickets.Count); + string hostTicket = distinctHostTickets[^1]; + string clientTicket = distinctClientTickets[^1]; + Assert.Equal(hostTicket, clientTicket); + Assert.True(NatIntroductionTokenCodec.TryDecode( + hostTicket, + out NatIntroductionToken? introduction)); + Assert.NotNull(introduction); + Assert.Equal(created.AttemptId, introduction.AttemptId); + Assert.Equal(43, introduction.ConnectionTicket.Length); + } + + private static UdpMediatorService CreateMediator(NatMediationProcessor processor, int port) => new( + Options.Create(new UdpMediatorOptions + { + ListenAddress = IPAddress.Loopback.ToString(), + Port = port, + MaxDatagramsPerPoll = 8, + PollIntervalMilliseconds = 1, + }), + NullLogger.Instance, + processor); + [Fact] public async Task FrozenV1EnvelopeIsConsumedOnTheLiteNetSocketWithinAmplificationBudget() { diff --git a/tests/FinalFactory.Rendezvous.Tests/TestClient/TestClientProcessIntegrationTests.cs b/tests/FinalFactory.Rendezvous.Tests/TestClient/TestClientProcessIntegrationTests.cs index a307335..6e9b0d4 100644 --- a/tests/FinalFactory.Rendezvous.Tests/TestClient/TestClientProcessIntegrationTests.cs +++ b/tests/FinalFactory.Rendezvous.Tests/TestClient/TestClientProcessIntegrationTests.cs @@ -1,8 +1,10 @@ using System.Collections.Concurrent; using System.Diagnostics; using System.Net; +using System.Net.Sockets; using System.Security.Cryptography; using System.Text.Json; +using FinalFactory.Rendezvous.Client; using FinalFactory.Rendezvous.Contracts; using FinalFactory.Rendezvous.Server.Provisioning; using FinalFactory.Rendezvous.TestClient; @@ -21,34 +23,12 @@ public sealed class TestClientProcessIntegrationTests public async Task ServerHostAndJoinProcessesExchangeAuthenticatedDirectTrafficWithoutLeakingSecrets() { using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(30)); - string root = RepositoryRoot(); - string serverAssembly = Path.Combine( - root, - $"src/FinalFactory.Rendezvous.Server/bin/{BuildConfiguration}/net10.0/FinalFactory.Rendezvous.Server.dll"); - string clientAssembly = Path.Combine( - root, - $"src/FinalFactory.Rendezvous.TestClient/bin/{BuildConfiguration}/net8.0/FinalFactory.Rendezvous.TestClient.dll"); - Assert.True(File.Exists(serverAssembly), $"Missing server build output: {serverAssembly}"); - Assert.True(File.Exists(clientAssembly), $"Missing TestClient build output: {clientAssembly}"); - - DateTimeOffset now = DateTimeOffset.UtcNow; - byte[] signingKey = RandomNumberGenerator.GetBytes(32); - string signingKeyText = Convert.ToBase64String(signingKey); - string publisherCredential = IssuePublisherCredential(signingKey, now); - CryptographicOperations.ZeroMemory(signingKey); - Dictionary serverEnvironment = ServerEnvironment( - signingKeyText, - now); - - await using ProcessCapture server = Start( - serverAssembly, - [], - serverEnvironment); - string httpLine = await server.WaitForLineAsync("Now listening on: http://127.0.0.1:", timeout.Token); - string serviceUrl = ParseServiceUrl(httpLine); - string udpLine = await server.WaitForLineAsync("UDP mediator listening on 127.0.0.1:", timeout.Token); - int udpPort = ParseTrailingPort(udpLine); - await WaitForReadyAsync(serviceUrl, server, timeout.Token); + await using RunningService runtime = await RunningService.StartAsync(timeout.Token); + ProcessCapture server = runtime.Server; + string clientAssembly = runtime.ClientAssembly; + string serviceUrl = runtime.ServiceUrl; + int udpPort = runtime.UdpPort; + string publisherCredential = runtime.PublisherCredential; string[] emptyBrowseArguments = [ @@ -195,6 +175,10 @@ public sealed class TestClientProcessIntegrationTests Assert.Equal(0, await join.WaitForExitAsync(timeout.Token)); Assert.Equal(0, await host.WaitForExitAsync(timeout.Token)); Assert.True(join.HasEvent("join.connected", "connected"), join.DiagnosticText()); + Assert.Contains( + join.JsonEvents(), + item => item.GetProperty("event").GetString() == "join.connected" + && item.GetProperty("endpointType").GetString() is "loopback" or "private"); Assert.True(join.HasEvent("join.punch", "started"), join.DiagnosticText()); Assert.True(join.HasEvent("join.direct-connect", "started"), join.DiagnosticText()); Assert.True(join.HasEvent("join.direct-traffic", "verified"), join.DiagnosticText()); @@ -235,10 +219,390 @@ public sealed class TestClientProcessIntegrationTests captured.Contains(publisherCredential, StringComparison.Ordinal), "Captured process output contained the publisher credential."); Assert.False( - captured.Contains(signingKeyText, StringComparison.Ordinal), + captured.Contains(runtime.SigningKeyText, StringComparison.Ordinal), "Captured process output contained signing-key material."); } + [Fact] + public async Task WrongTenantAndProtocolAreRejectedBeforeAnyTraversalTraffic() + { + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(20)); + await using RunningService runtime = await RunningService.StartAsync(timeout.Token); + string[] hostArguments = HostArguments(runtime, runSeconds: 10); + await using ProcessCapture host = Start( + runtime.ClientAssembly, + hostArguments, + PublisherEnvironment(runtime.PublisherCredential)); + JsonElement ready = await host.WaitForEventAsync("host.ready", timeout.Token); + string listingId = Assert.IsType(ready.GetProperty("listingId").GetString()); + + string[] browseArguments = BrowseArguments(runtime, protocolVersion: 2); + await using ProcessCapture browse = Start( + runtime.ClientAssembly, + browseArguments, + new Dictionary(StringComparer.Ordinal)); + Assert.Equal( + (int)TestClientExitCode.NoCompatibleSession, + await browse.WaitForExitAsync(timeout.Token)); + Assert.Contains( + browse.JsonEvents(), + item => item.GetProperty("event").GetString() == "browse.completed" + && item.GetProperty("count").GetInt32() == 0); + + string[] joinArguments = JoinArguments(runtime, listingId, protocolVersion: 2); + await using ProcessCapture join = Start( + runtime.ClientAssembly, + joinArguments, + new Dictionary(StringComparer.Ordinal)); + Assert.Equal((int)TestClientExitCode.ServiceFailure, await join.WaitForExitAsync(timeout.Token)); + Assert.True(join.HasEvent("join.selection", "failed"), join.DiagnosticText()); + Assert.DoesNotContain( + join.JsonEvents(), + item => item.GetProperty("event").GetString() == "join.punch"); + + string[] wrongTenantArguments = JoinArguments(runtime, listingId, gameId: "other-game"); + await using ProcessCapture wrongTenant = Start( + runtime.ClientAssembly, + wrongTenantArguments, + new Dictionary(StringComparer.Ordinal)); + Assert.Equal( + (int)TestClientExitCode.ServiceFailure, + await wrongTenant.WaitForExitAsync(timeout.Token)); + Assert.True(wrongTenant.HasEvent("join.selection", "failed"), wrongTenant.DiagnosticText()); + Assert.DoesNotContain( + wrongTenant.JsonEvents(), + item => item.GetProperty("event").GetString() == "join.punch"); + + string[] controlBrowseArguments = BrowseArguments(runtime); + await using ProcessCapture controlBrowse = Start( + runtime.ClientAssembly, + controlBrowseArguments, + new Dictionary(StringComparer.Ordinal)); + Assert.Equal(0, await controlBrowse.WaitForExitAsync(timeout.Token)); + Assert.Contains( + controlBrowse.JsonEvents(), + item => item.GetProperty("event").GetString() == "browse.session" + && item.GetProperty("listingId").GetString() == listingId); + + Assert.Equal(0, await host.WaitForExitAsync(timeout.Token)); + Assert.True(host.HasEvent("host.deregistered", "complete"), host.DiagnosticText()); + AssertNoSecrets(runtime, host, browse, join, wrongTenant, controlBrowse); + } + + [Fact] + public async Task TraversalTimeoutReturnsTypedFallbackWithoutConnectingToIt() + { + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(20)); + await using RunningService runtime = await RunningService.StartAsync(timeout.Token); + using UdpClient fallbackSentinel = new(new IPEndPoint(IPAddress.Loopback, 0)); + int fallbackPort = Assert.IsType(fallbackSentinel.Client.LocalEndPoint).Port; + using UdpClient mediatorBlackhole = new(new IPEndPoint(IPAddress.Loopback, 0)); + int blackholePort = Assert.IsType(mediatorBlackhole.Client.LocalEndPoint).Port; + string[] hostArguments = HostArguments( + runtime, + runSeconds: 5, + fallback: $"127.0.0.1:{fallbackPort}"); + await using ProcessCapture host = Start( + runtime.ClientAssembly, + hostArguments, + PublisherEnvironment(runtime.PublisherCredential)); + JsonElement ready = await host.WaitForEventAsync("host.ready", timeout.Token); + string listingId = Assert.IsType(ready.GetProperty("listingId").GetString()); + + string[] joinArguments = JoinArguments( + runtime, + listingId, + timeoutSeconds: 2, + mediatorPort: blackholePort); + await using ProcessCapture join = Start( + runtime.ClientAssembly, + joinArguments, + new Dictionary(StringComparer.Ordinal)); + + Assert.Equal((int)TestClientExitCode.TraversalFailed, await join.WaitForExitAsync(timeout.Token)); + Assert.True(join.HasEvent("join.traversal", "failed"), join.DiagnosticText()); + Assert.Contains( + join.JsonEvents(), + item => item.GetProperty("event").GetString() == "join.traversal" + && string.Equals( + item.GetProperty("outcome").GetString(), + ConnectionOutcomeKind.PunchTimedOut.ToString(), + StringComparison.OrdinalIgnoreCase)); + Assert.True(join.HasEvent("join.fallback", "available"), join.DiagnosticText()); + Assert.DoesNotContain( + join.JsonEvents(), + item => item.GetProperty("event").GetString() == "join.connected"); + Assert.True(mediatorBlackhole.Available > 0, "The retained black-hole socket received no punch traffic."); + Assert.Equal(0, fallbackSentinel.Available); + + Assert.Equal(0, await host.WaitForExitAsync(timeout.Token)); + Assert.True(host.HasEvent("host.deregistered", "complete"), host.DiagnosticText()); + AssertNoSecrets(runtime, host, join); + } + + [Fact] + public async Task InterruptCancelsHostAndStillDeregistersItsListing() + { + if (!OperatingSystem.IsLinux() && !OperatingSystem.IsMacOS()) + { + return; + } + + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(20)); + await using RunningService runtime = await RunningService.StartAsync(timeout.Token); + string[] hostArguments = HostArguments(runtime, runSeconds: 30); + await using ProcessCapture host = Start( + runtime.ClientAssembly, + hostArguments, + PublisherEnvironment(runtime.PublisherCredential)); + JsonElement ready = await host.WaitForEventAsync("host.ready", timeout.Token); + string listingId = Assert.IsType(ready.GetProperty("listingId").GetString()); + + await host.InterruptAsync(timeout.Token); + + Assert.Equal((int)TestClientExitCode.Cancelled, await host.WaitForExitAsync(timeout.Token)); + Assert.True(host.HasEvent("host.deregistered", "complete"), host.DiagnosticText()); + using HttpClient service = new() { BaseAddress = new Uri(runtime.ServiceUrl) }; + using HttpResponseMessage removed = await service.GetAsync( + $"v1/sessions/{listingId}?contractVersion=1&gameId=space-game&environmentId=integration&protocolVersion=1", + timeout.Token); + Assert.Equal(HttpStatusCode.NotFound, removed.StatusCode); + AssertNoSecrets(runtime, host); + } + + [Fact] + public async Task PrivilegedLinuxNatNamespacesCompleteDirectTrafficAcrossSeparateObservedEndpoints() + { + if (!OperatingSystem.IsLinux() + || !string.Equals( + Environment.GetEnvironmentVariable("RENDEZVOUS_RUN_NETNS_TESTS"), + "1", + StringComparison.Ordinal)) + { + return; + } + + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(30)); + await using NetworkNamespaceTopology topology = await NetworkNamespaceTopology.CreateAsync( + timeout.Token); + await using RunningService runtime = await RunningService.StartAsync( + timeout.Token, + listenAddress: "0.0.0.0", + processNamespace: topology.ServiceNamespace, + readinessAddress: topology.ServiceAddress); + int httpPort = new Uri(runtime.ServiceUrl).Port; + string serviceUrl = $"http://{topology.ServiceAddress}:{httpPort}/"; + string mediator = $"{topology.ServiceAddress}:{runtime.UdpPort}"; + string[] hostArguments = + [ + "host", + "--service", serviceUrl, + "--mediator", mediator, + "--game", "space-game", + "--environment", "integration", + "--region", "local", + "--protocol", "1", + "--script", + "--json", + "--exit-after-echo", + "--timeout-seconds", "15", + ]; + await using ProcessCapture host = StartInNamespace( + topology.HostNamespace, + runtime.ClientAssembly, + hostArguments, + PublisherEnvironment(runtime.PublisherCredential)); + JsonElement ready = await host.WaitForEventAsync("host.ready", timeout.Token); + string listingId = Assert.IsType(ready.GetProperty("listingId").GetString()); + string[] joinArguments = + [ + "join", + "--service", serviceUrl, + "--mediator", mediator, + "--game", "space-game", + "--environment", "integration", + "--region", "local", + "--protocol", "1", + "--listing", listingId, + "--script", + "--json", + "--timeout-seconds", "15", + ]; + await using ProcessCapture join = StartInNamespace( + topology.ClientNamespace, + runtime.ClientAssembly, + joinArguments, + new Dictionary(StringComparer.Ordinal)); + + Assert.Equal(0, await join.WaitForExitAsync(timeout.Token)); + Assert.Equal(0, await host.WaitForExitAsync(timeout.Token)); + Assert.True(join.HasEvent("join.direct-traffic", "verified"), join.DiagnosticText()); + Assert.True(host.HasEvent("host.direct-traffic", "verified"), host.DiagnosticText()); + Assert.Contains( + join.JsonEvents(), + item => item.GetProperty("event").GetString() == "join.connected" + && item.GetProperty("endpointType").GetString() == "public"); + AssertNoSecrets(runtime, host, join); + } + + [Fact] + public async Task AbruptHostLossBecomesStaleBeforeItsLeaseExpires() + { + using CancellationTokenSource setup = new(TimeSpan.FromSeconds(20)); + await using RunningService runtime = await RunningService.StartAsync(setup.Token); + await using ProcessCapture host = Start( + runtime.ClientAssembly, + HostArguments(runtime, runSeconds: 30), + PublisherEnvironment(runtime.PublisherCredential)); + JsonElement ready = await host.WaitForEventAsync("host.ready", setup.Token); + string listingId = Assert.IsType(ready.GetProperty("listingId").GetString()); + using HttpClient http = new() { BaseAddress = new Uri(runtime.ServiceUrl) }; + RendezvousSessionBrowserClient browser = new( + http, + new RendezvousClientOptions + { + MaximumSafeRetries = 0, + RequestTimeout = TimeSpan.FromSeconds(1), + JitterRatio = 0, + }); + BrowseSessionsRequest browseRequest = new() + { + GameId = new("space-game"), + EnvironmentId = new("integration"), + RegionId = new("local"), + ProtocolVersion = 1, + PageSize = 10, + }; + RendezvousClientResult> visible = await browser.BrowseAllAsync( + browseRequest, + cancellationToken: setup.Token); + Assert.True(visible.IsSuccess, visible.Message); + Assert.Contains(visible.Value!, item => item.ListingId.ToString() == listingId); + + await host.TerminateAsync(setup.Token); + Stopwatch staleWait = Stopwatch.StartNew(); + using CancellationTokenSource presenceExpiry = new(TimeSpan.FromSeconds(30)); + + using PeriodicTimer probe = new(TimeSpan.FromMilliseconds(100)); + RendezvousClientResult> afterLoss; + do + { + afterLoss = await browser.BrowseAllAsync( + browseRequest, + cancellationToken: presenceExpiry.Token); + Assert.True(afterLoss.IsSuccess, afterLoss.Message); + if (afterLoss.Value!.Count == 0) + { + break; + } + } + while (await probe.WaitForNextTickAsync(presenceExpiry.Token)); + + Assert.Empty(afterLoss.Value!); + Assert.True( + staleWait.Elapsed < TimeSpan.FromSeconds(30), + $"Presence did not become stale before the 60-second lease lifetime: {staleWait.Elapsed}."); + RendezvousClientResult exact = await browser.GetAsync( + new SessionListingId(Guid.Parse(listingId)), + new GameId("space-game"), + new EnvironmentId("integration"), + 1, + presenceExpiry.Token); + Assert.False(exact.IsSuccess); + Assert.Equal(RendezvousErrorCode.NotFound, exact.Error); + + string[] browseArguments = BrowseArguments(runtime); + await using ProcessCapture publicBrowse = Start( + runtime.ClientAssembly, + browseArguments, + new Dictionary(StringComparer.Ordinal)); + Assert.Equal( + (int)TestClientExitCode.NoCompatibleSession, + await publicBrowse.WaitForExitAsync(presenceExpiry.Token)); + AssertNoSecrets(runtime, host, publicBrowse); + } + + private static string[] HostArguments( + RunningService runtime, + int runSeconds, + string? fallback = null) + { + List arguments = + [ + "host", + "--service", runtime.ServiceUrl, + "--mediator", $"127.0.0.1:{runtime.UdpPort}", + "--game", "space-game", + "--environment", "integration", + "--region", "local", + "--protocol", "1", + "--script", + "--json", + "--run-seconds", runSeconds.ToString(System.Globalization.CultureInfo.InvariantCulture), + "--timeout-seconds", "10", + ]; + if (fallback is not null) + { + arguments.Add("--fallback"); + arguments.Add(fallback); + } + return [.. arguments]; + } + + private static string[] BrowseArguments(RunningService runtime, int protocolVersion = 1) => + [ + "browse", + "--service", runtime.ServiceUrl, + "--game", "space-game", + "--environment", "integration", + "--region", "local", + "--protocol", protocolVersion.ToString(System.Globalization.CultureInfo.InvariantCulture), + "--script", + "--json", + "--timeout-seconds", "5", + ]; + + private static string[] JoinArguments( + RunningService runtime, + string listingId, + int protocolVersion = 1, + string gameId = "space-game", + int timeoutSeconds = 5, + int? mediatorPort = null) => + [ + "join", + "--service", runtime.ServiceUrl, + "--mediator", $"127.0.0.1:{mediatorPort ?? runtime.UdpPort}", + "--game", gameId, + "--environment", "integration", + "--region", "local", + "--protocol", protocolVersion.ToString(System.Globalization.CultureInfo.InvariantCulture), + "--listing", listingId, + "--script", + "--json", + "--timeout-seconds", timeoutSeconds.ToString(System.Globalization.CultureInfo.InvariantCulture), + ]; + + private static Dictionary PublisherEnvironment(string credential) => + new(StringComparer.Ordinal) + { + ["RENDEZVOUS_PUBLISHER_CREDENTIAL"] = credential, + }; + + private static void AssertNoSecrets(RunningService runtime, params ProcessCapture[] processes) + { + foreach (ProcessCapture process in processes) + { + Assert.Equal(process.AllLines().Count(), process.JsonEvents().Count); + Assert.All(process.JsonEvents(), AssertAllowlistedEventShape); + } + string captured = string.Join( + '\n', + runtime.Server.AllLines().Concat(processes.SelectMany(static process => process.AllLines()))); + Assert.DoesNotContain(runtime.PublisherCredential, captured, StringComparison.Ordinal); + Assert.DoesNotContain(runtime.SigningKeyText, captured, StringComparison.Ordinal); + } + private static string IssuePublisherCredential(byte[] signingKey, DateTimeOffset now) { const string secretReference = "env:RENDEZVOUS_INTEGRATION_SIGNING_KEY"; @@ -301,13 +665,14 @@ public sealed class TestClientProcessIntegrationTests private static Dictionary ServerEnvironment( string signingKey, - DateTimeOffset now) + DateTimeOffset now, + string listenAddress) { Dictionary values = new(StringComparer.Ordinal) { ["ASPNETCORE_ENVIRONMENT"] = "Production", - ["ASPNETCORE_URLS"] = "http://127.0.0.1:0", - ["Rendezvous__Udp__ListenAddress"] = "127.0.0.1", + ["ASPNETCORE_URLS"] = $"http://{listenAddress}:0", + ["Rendezvous__Udp__ListenAddress"] = listenAddress, ["Rendezvous__Udp__Port"] = "0", ["Rendezvous__Udp__PollIntervalMilliseconds"] = "1", ["Rendezvous__Provisioning__Issuer"] = "rendezvous-process-test", @@ -334,6 +699,7 @@ public sealed class TestClientProcessIntegrationTests ["Rendezvous__Provisioning__Games__0__MaxListingsPerPrincipal"] = "4", ["Rendezvous__Provisioning__Games__0__MaxAnonymousListingsPerAddress"] = "1", ["Rendezvous__Provisioning__Games__0__MaxActiveJoinAttempts"] = "16", + ["Rendezvous__Provisioning__Games__0__FallbackPolicy"] = "DedicatedEndpointAllowed", ["RENDEZVOUS_INTEGRATION_SIGNING_KEY"] = signingKey, }; return values; @@ -364,6 +730,36 @@ public sealed class TestClientProcessIntegrationTests return new ProcessCapture(start); } + private static ProcessCapture StartInNamespace( + string namespaceName, + string assembly, + IReadOnlyList arguments, + IReadOnlyDictionary environment) + { + ProcessStartInfo start = new() + { + FileName = "ip", + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + start.ArgumentList.Add("netns"); + start.ArgumentList.Add("exec"); + start.ArgumentList.Add(namespaceName); + start.ArgumentList.Add(Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") ?? "dotnet"); + start.ArgumentList.Add(assembly); + foreach (string argument in arguments) + { + start.ArgumentList.Add(argument); + } + foreach (KeyValuePair item in environment) + { + start.Environment[item.Key] = item.Value; + } + return new ProcessCapture(start); + } + private static async Task WaitForReadyAsync( string serviceUrl, ProcessCapture server, @@ -431,11 +827,568 @@ public sealed class TestClientProcessIntegrationTests } } + private sealed class NetworkNamespaceTopology : IAsyncDisposable + { + private readonly string _bridge; + private readonly string _serviceRootInterface; + private readonly string _serviceWanInterface; + private readonly string _hostWanRootInterface; + private readonly string _hostWanInterface; + private readonly string _clientWanRootInterface; + private readonly string _clientWanInterface; + private readonly string _hostLanInterface; + private readonly string _hostAppInterface; + private readonly string _clientLanInterface; + private readonly string _clientAppInterface; + private readonly HashSet _ownedNamespaces = new(StringComparer.Ordinal); + private readonly HashSet _ownedRootLinks = new(StringComparer.Ordinal); + private bool _bridgeCreated; + private bool _created; + + private NetworkNamespaceTopology( + string suffix, + int wanSecondOctet, + int wanThirdOctet, + int hostThirdOctet, + int clientThirdOctet) + { + _bridge = $"rvb{suffix}"; + _serviceRootInterface = $"rvsr{suffix}"; + _serviceWanInterface = $"rvsw{suffix}"; + _hostWanRootInterface = $"rvhr{suffix}"; + _hostWanInterface = $"rvhw{suffix}"; + _clientWanRootInterface = $"rvcr{suffix}"; + _clientWanInterface = $"rvcw{suffix}"; + _hostLanInterface = $"rvhl{suffix}"; + _hostAppInterface = $"rvha{suffix}"; + _clientLanInterface = $"rvcl{suffix}"; + _clientAppInterface = $"rvca{suffix}"; + ServiceNamespace = $"rv-service-{suffix}"; + HostRouterNamespace = $"rv-host-nat-{suffix}"; + HostNamespace = $"rv-host-{suffix}"; + ClientRouterNamespace = $"rv-client-nat-{suffix}"; + ClientNamespace = $"rv-client-{suffix}"; + NetworkCidr = $"198.{wanSecondOctet}.{wanThirdOctet}.0/24"; + RootWanAddress = $"198.{wanSecondOctet}.{wanThirdOctet}.1"; + ServiceAddress = $"198.{wanSecondOctet}.{wanThirdOctet}.2"; + HostWanAddress = $"198.{wanSecondOctet}.{wanThirdOctet}.3"; + ClientWanAddress = $"198.{wanSecondOctet}.{wanThirdOctet}.4"; + HostRouterAddress = $"10.71.{hostThirdOctet}.1"; + HostAddress = $"10.71.{hostThirdOctet}.2"; + ClientRouterAddress = $"10.72.{clientThirdOctet}.1"; + ClientAddress = $"10.72.{clientThirdOctet}.2"; + } + + internal string ServiceNamespace { get; } + internal string HostNamespace { get; } + internal string ClientNamespace { get; } + internal string ServiceAddress { get; } + private string HostRouterNamespace { get; } + private string ClientRouterNamespace { get; } + private string NetworkCidr { get; } + private string RootWanAddress { get; } + private string HostWanAddress { get; } + private string ClientWanAddress { get; } + private string HostRouterAddress { get; } + private string HostAddress { get; } + private string ClientRouterAddress { get; } + private string ClientAddress { get; } + + internal static async Task CreateAsync( + CancellationToken cancellationToken) + { + for (int attempt = 0; attempt < 20; attempt++) + { + string suffix = Convert.ToHexString(RandomNumberGenerator.GetBytes(3)).ToLowerInvariant(); + NetworkNamespaceTopology topology = new( + suffix, + RandomNumberGenerator.GetInt32(18, 20), + RandomNumberGenerator.GetInt32(0, 256), + RandomNumberGenerator.GetInt32(0, 256), + RandomNumberGenerator.GetInt32(0, 256)); + IpCommandResult bridge = await ExecuteIpAsync( + ["link", "show", "dev", topology._bridge], + cancellationToken); + IpCommandResult route = await ExecuteIpAsync( + ["-4", "route", "show", "exact", topology.NetworkCidr], + cancellationToken); + if (bridge.ExitCode == 0 + || route.ExitCode != 0 + || !string.IsNullOrWhiteSpace(route.Output) + || await topology.HasNameCollisionAsync(cancellationToken)) + { + continue; + } + try + { + await topology.CreateCoreAsync(cancellationToken); + topology._created = true; + return topology; + } + catch + { + await topology.CleanupAsync(requireSuccess: false, CancellationToken.None); + throw; + } + } + throw new Xunit.Sdk.XunitException( + "Could not allocate a collision-free namespace name and private subnet after 20 attempts."); + } + + public async ValueTask DisposeAsync() + { + if (_created) + { + await CleanupAsync(requireSuccess: true, CancellationToken.None); + _created = false; + } + } + + private async Task CreateCoreAsync(CancellationToken cancellationToken) + { + await AddOwnedBridgeAsync(cancellationToken); + await RunIpAsync(["addr", "add", $"{RootWanAddress}/24", "dev", _bridge], true, cancellationToken); + await RunIpAsync(["link", "set", _bridge, "up"], true, cancellationToken); + await AddNamespaceAsync(ServiceNamespace, cancellationToken); + await AddNamespaceAsync(HostRouterNamespace, cancellationToken); + await AddNamespaceAsync(HostNamespace, cancellationToken); + await AddNamespaceAsync(ClientRouterNamespace, cancellationToken); + await AddNamespaceAsync(ClientNamespace, cancellationToken); + await ConfigureWanPeerAsync( + ServiceNamespace, + _serviceRootInterface, + _serviceWanInterface, + ServiceAddress, + cancellationToken); + await ConfigureWanPeerAsync( + HostRouterNamespace, + _hostWanRootInterface, + _hostWanInterface, + HostWanAddress, + cancellationToken); + await ConfigureWanPeerAsync( + ClientRouterNamespace, + _clientWanRootInterface, + _clientWanInterface, + ClientWanAddress, + cancellationToken); + await ConfigureNatPeerAsync( + HostRouterNamespace, + HostNamespace, + _hostWanInterface, + _hostLanInterface, + _hostAppInterface, + HostRouterAddress, + HostAddress, + cancellationToken); + await ConfigureNatPeerAsync( + ClientRouterNamespace, + ClientNamespace, + _clientWanInterface, + _clientLanInterface, + _clientAppInterface, + ClientRouterAddress, + ClientAddress, + cancellationToken); + } + + private async Task AddNamespaceAsync(string namespaceName, CancellationToken cancellationToken) + { + try + { + await RunIpAsync(["netns", "add", namespaceName], true, cancellationToken); + _ownedNamespaces.Add(namespaceName); + cancellationToken.ThrowIfCancellationRequested(); + } + catch (OperationCanceledException) + { + if (await NamespaceExistsAsync(namespaceName)) + { + _ownedNamespaces.Add(namespaceName); + } + throw; + } + } + + private async Task AddOwnedBridgeAsync(CancellationToken cancellationToken) + { + try + { + await RunIpAsync(["link", "add", _bridge, "type", "bridge"], true, cancellationToken); + _bridgeCreated = true; + cancellationToken.ThrowIfCancellationRequested(); + } + catch (OperationCanceledException) + { + _bridgeCreated = await LinkExistsAsync(_bridge); + throw; + } + } + + private async Task AddOwnedRootLinkAsync( + string rootInterface, + string peerInterface, + CancellationToken cancellationToken) + { + try + { + await RunIpAsync( + ["link", "add", rootInterface, "type", "veth", "peer", "name", peerInterface], + true, + cancellationToken); + _ownedRootLinks.Add(rootInterface); + cancellationToken.ThrowIfCancellationRequested(); + } + catch (OperationCanceledException) + { + if (await LinkExistsAsync(rootInterface)) + { + _ownedRootLinks.Add(rootInterface); + } + throw; + } + } + + private async Task ConfigureWanPeerAsync( + string namespaceName, + string rootInterface, + string peerInterface, + string address, + CancellationToken cancellationToken) + { + await AddOwnedRootLinkAsync(rootInterface, peerInterface, cancellationToken); + await RunIpAsync(["link", "set", peerInterface, "netns", namespaceName], true, cancellationToken); + await RunIpAsync(["link", "set", rootInterface, "master", _bridge], true, cancellationToken); + await RunIpAsync(["link", "set", rootInterface, "up"], true, cancellationToken); + await RunIpAsync(["-n", namespaceName, "addr", "add", $"{address}/24", "dev", peerInterface], true, cancellationToken); + await RunIpAsync(["-n", namespaceName, "link", "set", peerInterface, "up"], true, cancellationToken); + await RunIpAsync(["-n", namespaceName, "link", "set", "lo", "up"], true, cancellationToken); + } + + private async Task ConfigureNatPeerAsync( + string routerNamespace, + string appNamespace, + string wanInterface, + string lanInterface, + string appInterface, + string routerAddress, + string appAddress, + CancellationToken cancellationToken) + { + await AddOwnedRootLinkAsync(lanInterface, appInterface, cancellationToken); + await RunIpAsync(["link", "set", lanInterface, "netns", routerNamespace], true, cancellationToken); + await RunIpAsync(["link", "set", appInterface, "netns", appNamespace], true, cancellationToken); + await RunIpAsync(["-n", routerNamespace, "addr", "add", $"{routerAddress}/24", "dev", lanInterface], true, cancellationToken); + await RunIpAsync(["-n", routerNamespace, "link", "set", lanInterface, "up"], true, cancellationToken); + await RunIpAsync(["-n", appNamespace, "addr", "add", $"{appAddress}/24", "dev", appInterface], true, cancellationToken); + await RunIpAsync(["-n", appNamespace, "link", "set", appInterface, "up"], true, cancellationToken); + await RunIpAsync(["-n", appNamespace, "link", "set", "lo", "up"], true, cancellationToken); + await RunIpAsync(["-n", appNamespace, "route", "add", "default", "via", routerAddress], true, cancellationToken); + await RunIpAsync( + ["netns", "exec", routerNamespace, "sysctl", "-q", "-w", "net.ipv4.ip_forward=1"], + true, + cancellationToken); + await RunIpAsync( + ["netns", "exec", routerNamespace, "iptables", "-t", "nat", "-A", "POSTROUTING", "-o", wanInterface, "-j", "MASQUERADE"], + true, + cancellationToken); + await RunIpAsync( + ["netns", "exec", routerNamespace, "iptables", "-A", "FORWARD", "-i", lanInterface, "-o", wanInterface, "-j", "ACCEPT"], + true, + cancellationToken); + await RunIpAsync( + ["netns", "exec", routerNamespace, "iptables", "-A", "FORWARD", "-i", wanInterface, "-o", lanInterface, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"], + true, + cancellationToken); + } + + private async Task HasNameCollisionAsync(CancellationToken cancellationToken) + { + string[] namespaces = + [ + ServiceNamespace, + HostRouterNamespace, + HostNamespace, + ClientRouterNamespace, + ClientNamespace, + ]; + foreach (string namespaceName in namespaces) + { + IpCommandResult existing = await ExecuteIpAsync( + ["netns", "exec", namespaceName, "true"], + cancellationToken); + if (existing.ExitCode == 0) + { + return true; + } + } + string[] links = + [ + _serviceRootInterface, + _serviceWanInterface, + _hostWanRootInterface, + _hostWanInterface, + _clientWanRootInterface, + _clientWanInterface, + _hostLanInterface, + _hostAppInterface, + _clientLanInterface, + _clientAppInterface, + ]; + foreach (string link in links) + { + IpCommandResult existing = await ExecuteIpAsync( + ["link", "show", "dev", link], + cancellationToken); + if (existing.ExitCode == 0) + { + return true; + } + } + return false; + } + + private static async Task LinkExistsAsync(string link) + { + using CancellationTokenSource probe = new(TimeSpan.FromSeconds(2)); + IpCommandResult existing = await ExecuteIpAsync( + ["link", "show", "dev", link], + probe.Token); + return existing.ExitCode == 0; + } + + private static async Task NamespaceExistsAsync(string namespaceName) + { + using CancellationTokenSource probe = new(TimeSpan.FromSeconds(2)); + IpCommandResult existing = await ExecuteIpAsync( + ["netns", "exec", namespaceName, "true"], + probe.Token); + return existing.ExitCode == 0; + } + + private async Task CleanupAsync(bool requireSuccess, CancellationToken cancellationToken) + { + Exception? firstFailure = null; + string[] namespaceOrder = + [ + HostNamespace, + ClientNamespace, + HostRouterNamespace, + ClientRouterNamespace, + ServiceNamespace, + ]; + List> commands = namespaceOrder + .Where(_ownedNamespaces.Contains) + .Select(static name => (IReadOnlyList)["netns", "delete", name]) + .ToList(); + if (_bridgeCreated) + { + commands.Add(["link", "delete", _bridge]); + } + foreach (IReadOnlyList command in commands) + { + try + { + using CancellationTokenSource step = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken); + step.CancelAfter(TimeSpan.FromSeconds(5)); + await RunIpAsync(command, requireSuccess, step.Token); + } + catch (Exception exception) + { + if (requireSuccess) + { + firstFailure ??= exception; + } + } + } + foreach (string link in _ownedRootLinks) + { + try + { + using CancellationTokenSource step = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken); + step.CancelAfter(TimeSpan.FromSeconds(5)); + await RunIpAsync(["link", "delete", link], requireSuccess: false, step.Token); + IpCommandResult remaining = await ExecuteIpAsync( + ["link", "show", "dev", link], + step.Token); + if (remaining.ExitCode == 0) + { + throw new Xunit.Sdk.XunitException($"Owned topology link {link} remained after cleanup."); + } + } + catch (Exception exception) + { + if (requireSuccess) + { + firstFailure ??= exception; + } + } + } + if (firstFailure is not null) + { + throw new Xunit.Sdk.XunitException( + $"Network namespace cleanup failed: {firstFailure.Message}"); + } + } + + private static async Task RunIpAsync( + IReadOnlyList arguments, + bool requireSuccess, + CancellationToken cancellationToken) + { + IpCommandResult result = await ExecuteIpAsync(arguments, cancellationToken); + if (requireSuccess && result.ExitCode != 0) + { + throw new Xunit.Sdk.XunitException( + $"ip topology command failed with exit {result.ExitCode}; stdout={result.Output}; stderr={result.Error}"); + } + } + + private static async Task ExecuteIpAsync( + IReadOnlyList arguments, + CancellationToken cancellationToken) + { + ProcessStartInfo start = new() + { + FileName = "ip", + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + foreach (string argument in arguments) + { + start.ArgumentList.Add(argument); + } + using Process process = Process.Start(start) + ?? throw new Xunit.Sdk.XunitException("Failed to start the ip topology command."); + Task output = process.StandardOutput.ReadToEndAsync(CancellationToken.None); + Task error = process.StandardError.ReadToEndAsync(CancellationToken.None); + try + { + await process.WaitForExitAsync(cancellationToken); + } + catch (OperationCanceledException) + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + } + using CancellationTokenSource reap = new(TimeSpan.FromSeconds(2)); + try + { + await process.WaitForExitAsync(reap.Token); + await Task.WhenAll(output, error); + if (process.ExitCode == 0) + { + return new IpCommandResult(process.ExitCode, output.Result, error.Result); + } + } + catch (OperationCanceledException) + { + throw new Xunit.Sdk.XunitException( + "A timed-out ip topology helper did not exit after it was killed."); + } + throw; + } + await Task.WhenAll(output, error); + return new IpCommandResult(process.ExitCode, output.Result, error.Result); + } + + private sealed record IpCommandResult(int ExitCode, string Output, string Error); + } + + private sealed class RunningService : IAsyncDisposable + { + private RunningService( + ProcessCapture server, + string clientAssembly, + string serviceUrl, + int udpPort, + string publisherCredential, + string signingKeyText) + { + Server = server; + ClientAssembly = clientAssembly; + ServiceUrl = serviceUrl; + UdpPort = udpPort; + PublisherCredential = publisherCredential; + SigningKeyText = signingKeyText; + } + + internal ProcessCapture Server { get; } + internal string ClientAssembly { get; } + internal string ServiceUrl { get; } + internal int UdpPort { get; } + internal string PublisherCredential { get; } + internal string SigningKeyText { get; } + + internal static async Task StartAsync( + CancellationToken cancellationToken, + string listenAddress = "127.0.0.1", + string? processNamespace = null, + string? readinessAddress = null) + { + string root = RepositoryRoot(); + string serverAssembly = Path.Combine( + root, + $"src/FinalFactory.Rendezvous.Server/bin/{BuildConfiguration}/net10.0/FinalFactory.Rendezvous.Server.dll"); + string clientAssembly = Path.Combine( + root, + $"src/FinalFactory.Rendezvous.TestClient/bin/{BuildConfiguration}/net8.0/FinalFactory.Rendezvous.TestClient.dll"); + Assert.True(File.Exists(serverAssembly), $"Missing server build output: {serverAssembly}"); + Assert.True(File.Exists(clientAssembly), $"Missing TestClient build output: {clientAssembly}"); + + DateTimeOffset now = DateTimeOffset.UtcNow; + byte[] signingKey = RandomNumberGenerator.GetBytes(32); + string signingKeyText = Convert.ToBase64String(signingKey); + string publisherCredential = IssuePublisherCredential(signingKey, now); + CryptographicOperations.ZeroMemory(signingKey); + Dictionary environment = ServerEnvironment(signingKeyText, now, listenAddress); + ProcessCapture server = processNamespace is null + ? Start(serverAssembly, [], environment) + : StartInNamespace(processNamespace, serverAssembly, [], environment); + try + { + string httpLine = await server.WaitForLineAsync( + $"Now listening on: http://{listenAddress}:", + cancellationToken); + Uri boundService = new(ParseServiceUrl(httpLine)); + string serviceUrl = new UriBuilder(boundService) + { + Host = readinessAddress ?? IPAddress.Loopback.ToString(), + }.Uri.AbsoluteUri; + string udpLine = await server.WaitForLineAsync( + $"UDP mediator listening on {listenAddress}:", + cancellationToken); + int udpPort = ParseTrailingPort(udpLine); + await WaitForReadyAsync(serviceUrl, server, cancellationToken); + return new RunningService( + server, + clientAssembly, + serviceUrl, + udpPort, + publisherCredential, + signingKeyText); + } + catch + { + await server.DisposeAsync(); + throw; + } + } + + public ValueTask DisposeAsync() => Server.DisposeAsync(); + } + private sealed class ProcessCapture : IAsyncDisposable { private readonly Process _process; private readonly ConcurrentQueue _standardOutput = new(); private readonly ConcurrentQueue _standardError = new(); + private readonly ConcurrentQueue _jsonEvents = new(); internal ProcessCapture(ProcessStartInfo start) { @@ -503,22 +1456,7 @@ public sealed class TestClientProcessIntegrationTests && item.TryGetProperty("status", out JsonElement statusProperty) && statusProperty.GetString() == status); - internal List JsonEvents() - { - List items = []; - foreach (string line in _standardOutput.Concat(_standardError)) - { - try - { - using JsonDocument document = JsonDocument.Parse(line); - items.Add(document.RootElement.Clone()); - } - catch (JsonException) - { - } - } - return items; - } + internal List JsonEvents() => [.. _jsonEvents]; internal IEnumerable AllLines() => _standardOutput.Concat(_standardError); @@ -528,6 +1466,31 @@ public sealed class TestClientProcessIntegrationTests return _process.ExitCode; } + internal async Task InterruptAsync(CancellationToken cancellationToken) + { + ProcessStartInfo signalStart = new() + { + FileName = "/bin/kill", + UseShellExecute = false, + CreateNoWindow = true, + }; + signalStart.ArgumentList.Add("-INT"); + signalStart.ArgumentList.Add(_process.Id.ToString(System.Globalization.CultureInfo.InvariantCulture)); + using Process signal = Process.Start(signalStart) + ?? throw new Xunit.Sdk.XunitException("Failed to start the POSIX signal helper."); + await signal.WaitForExitAsync(cancellationToken); + Assert.Equal(0, signal.ExitCode); + } + + internal async Task TerminateAsync(CancellationToken cancellationToken) + { + if (!_process.HasExited) + { + _process.Kill(entireProcessTree: true); + } + await _process.WaitForExitAsync(cancellationToken); + } + internal string DiagnosticText() { string events = string.Join( @@ -540,6 +1503,7 @@ public sealed class TestClientProcessIntegrationTests public async ValueTask DisposeAsync() { bool cleanupTimedOut = false; + bool reapTimedOut = false; try { if (!_process.HasExited) @@ -564,6 +1528,15 @@ public sealed class TestClientProcessIntegrationTests catch (InvalidOperationException) { } + using CancellationTokenSource reap = new(TimeSpan.FromSeconds(2)); + try + { + await _process.WaitForExitAsync(reap.Token); + } + catch (OperationCanceledException) + { + reapTimedOut = true; + } } } finally @@ -572,15 +1545,27 @@ public sealed class TestClientProcessIntegrationTests } if (cleanupTimedOut) { - throw new Xunit.Sdk.XunitException("Child process did not exit within the cleanup deadline."); + string detail = reapTimedOut + ? " and remained alive after the final kill" + : string.Empty; + throw new Xunit.Sdk.XunitException( + $"Child process did not exit within the cleanup deadline{detail}."); } } - private static void Add(string? line, ConcurrentQueue destination) + private void Add(string? line, ConcurrentQueue destination) { if (!string.IsNullOrEmpty(line)) { destination.Enqueue(line); + try + { + using JsonDocument document = JsonDocument.Parse(line); + _jsonEvents.Enqueue(document.RootElement.Clone()); + } + catch (JsonException) + { + } } } }