KyuubiYoru
1597 lines
67 KiB
C#
1597 lines
67 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Diagnostics;
|
|
using System.Net;
|
|
using System.Net.Sockets;
|
|
using System.Security.Cryptography;
|
|
using System.Text.Json;
|
|
using FinalFactory.Rendezvous.Client;
|
|
using FinalFactory.Rendezvous.Contracts;
|
|
using FinalFactory.Rendezvous.Server.Provisioning;
|
|
using FinalFactory.Rendezvous.TestClient;
|
|
|
|
namespace FinalFactory.Rendezvous.Tests.TestClient;
|
|
|
|
public sealed class TestClientProcessIntegrationTests
|
|
{
|
|
#if DEBUG
|
|
private const string BuildConfiguration = "Debug";
|
|
#else
|
|
private const string BuildConfiguration = "Release";
|
|
#endif
|
|
|
|
[Fact]
|
|
public async Task ServerHostAndJoinProcessesExchangeAuthenticatedDirectTrafficWithoutLeakingSecrets()
|
|
{
|
|
using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(30));
|
|
await using RunningService runtime = await RunningService.StartAsync(timeout.Token);
|
|
ProcessCapture server = runtime.Server;
|
|
string clientAssembly = runtime.ClientAssembly;
|
|
string serviceUrl = runtime.ServiceUrl;
|
|
int udpPort = runtime.UdpPort;
|
|
string publisherCredential = runtime.PublisherCredential;
|
|
|
|
string[] emptyBrowseArguments =
|
|
[
|
|
"browse",
|
|
"--service", serviceUrl,
|
|
"--game", "space-game",
|
|
"--environment", "integration",
|
|
"--region", "local",
|
|
"--protocol", "1",
|
|
"--script",
|
|
"--json",
|
|
"--timeout-seconds", "15",
|
|
];
|
|
await using ProcessCapture emptyBrowse = Start(
|
|
clientAssembly,
|
|
emptyBrowseArguments,
|
|
new Dictionary<string, string>(StringComparer.Ordinal));
|
|
Assert.Equal(
|
|
(int)TestClientExitCode.NoCompatibleSession,
|
|
await emptyBrowse.WaitForExitAsync(timeout.Token));
|
|
Assert.True(emptyBrowse.HasEvent("browse.completed", "complete"), emptyBrowse.DiagnosticText());
|
|
Assert.Contains(
|
|
emptyBrowse.JsonEvents(),
|
|
item => item.GetProperty("event").GetString() == "browse.completed"
|
|
&& item.GetProperty("count").GetInt32() == 0);
|
|
|
|
string[] missingJoinArguments =
|
|
[
|
|
"join",
|
|
"--service", serviceUrl,
|
|
"--mediator", $"127.0.0.1:{udpPort}",
|
|
"--game", "space-game",
|
|
"--environment", "integration",
|
|
"--region", "local",
|
|
"--protocol", "1",
|
|
"--listing", Guid.NewGuid().ToString("D"),
|
|
"--script",
|
|
"--json",
|
|
"--timeout-seconds", "15",
|
|
];
|
|
await using ProcessCapture missingJoin = Start(
|
|
clientAssembly,
|
|
missingJoinArguments,
|
|
new Dictionary<string, string>(StringComparer.Ordinal));
|
|
Assert.Equal((int)TestClientExitCode.ServiceFailure, await missingJoin.WaitForExitAsync(timeout.Token));
|
|
Assert.True(missingJoin.HasEvent("join.selection", "failed"), missingJoin.DiagnosticText());
|
|
|
|
string[] boundedHostArguments =
|
|
[
|
|
"host",
|
|
"--service", serviceUrl,
|
|
"--mediator", $"127.0.0.1:{udpPort}",
|
|
"--game", "space-game",
|
|
"--environment", "integration",
|
|
"--region", "local",
|
|
"--protocol", "1",
|
|
"--script",
|
|
"--json",
|
|
"--exit-after-echo",
|
|
"--run-seconds", "1",
|
|
"--timeout-seconds", "15",
|
|
];
|
|
await using ProcessCapture boundedHost = Start(
|
|
clientAssembly,
|
|
boundedHostArguments,
|
|
new Dictionary<string, string>(StringComparer.Ordinal)
|
|
{
|
|
["RENDEZVOUS_PUBLISHER_CREDENTIAL"] = publisherCredential,
|
|
});
|
|
Assert.Equal((int)TestClientExitCode.DirectTrafficFailed, await boundedHost.WaitForExitAsync(timeout.Token));
|
|
Assert.True(boundedHost.HasEvent("host.direct-traffic", "timed-out"), boundedHost.DiagnosticText());
|
|
Assert.True(boundedHost.HasEvent("host.deregistered", "complete"), boundedHost.DiagnosticText());
|
|
|
|
string[] hostArguments =
|
|
[
|
|
"host",
|
|
"--service", serviceUrl,
|
|
"--mediator", $"127.0.0.1:{udpPort}",
|
|
"--game", "space-game",
|
|
"--environment", "integration",
|
|
"--region", "local",
|
|
"--protocol", "1",
|
|
"--script",
|
|
"--json",
|
|
"--exit-after-echo",
|
|
"--timeout-seconds", "15",
|
|
];
|
|
await using ProcessCapture host = Start(
|
|
clientAssembly,
|
|
hostArguments,
|
|
new Dictionary<string, string>(StringComparer.Ordinal)
|
|
{
|
|
["RENDEZVOUS_PUBLISHER_CREDENTIAL"] = publisherCredential,
|
|
});
|
|
JsonElement hostRegistered = await host.WaitForEventAsync(
|
|
"host.ready",
|
|
timeout.Token);
|
|
string listingId = Assert.IsType<string>(hostRegistered.GetProperty("listingId").GetString());
|
|
|
|
string[] browseArguments =
|
|
[
|
|
"browse",
|
|
"--service", serviceUrl,
|
|
"--game", "space-game",
|
|
"--environment", "integration",
|
|
"--region", "local",
|
|
"--protocol", "1",
|
|
"--script",
|
|
"--json",
|
|
"--timeout-seconds", "15",
|
|
];
|
|
await using ProcessCapture browse = Start(
|
|
clientAssembly,
|
|
browseArguments,
|
|
new Dictionary<string, string>(StringComparer.Ordinal));
|
|
Assert.Equal(0, await browse.WaitForExitAsync(timeout.Token));
|
|
Assert.True(browse.HasEvent("browse.completed", "complete"), browse.DiagnosticText());
|
|
Assert.True(
|
|
browse.JsonEvents().Any(item =>
|
|
item.GetProperty("event").GetString() == "browse.session"
|
|
&& item.GetProperty("status").GetString() == "available"
|
|
&& item.GetProperty("listingId").GetString() == listingId),
|
|
browse.DiagnosticText());
|
|
|
|
string[] joinArguments =
|
|
[
|
|
"join",
|
|
"--service", serviceUrl,
|
|
"--mediator", $"127.0.0.1:{udpPort}",
|
|
"--game", "space-game",
|
|
"--environment", "integration",
|
|
"--region", "local",
|
|
"--protocol", "1",
|
|
"--listing", listingId,
|
|
"--script",
|
|
"--json",
|
|
"--timeout-seconds", "15",
|
|
];
|
|
await using ProcessCapture join = Start(
|
|
clientAssembly,
|
|
joinArguments,
|
|
new Dictionary<string, string>(StringComparer.Ordinal));
|
|
|
|
Assert.Equal(0, await join.WaitForExitAsync(timeout.Token));
|
|
Assert.Equal(0, await host.WaitForExitAsync(timeout.Token));
|
|
Assert.True(join.HasEvent("join.connected", "connected"), join.DiagnosticText());
|
|
Assert.Contains(
|
|
join.JsonEvents(),
|
|
item => item.GetProperty("event").GetString() == "join.connected"
|
|
&& item.GetProperty("endpointType").GetString() is "loopback" or "private"
|
|
&& item.GetProperty("addressFamily").GetString() == "ipv4");
|
|
Assert.True(join.HasEvent("join.punch", "started"), join.DiagnosticText());
|
|
Assert.True(join.HasEvent("join.direct-connect", "started"), join.DiagnosticText());
|
|
Assert.True(join.HasEvent("join.direct-traffic", "verified"), join.DiagnosticText());
|
|
Assert.True(join.HasEvent("join.outcome-report", "accepted"), join.DiagnosticText());
|
|
Assert.True(host.HasEvent("host.direct-traffic", "verified"), host.DiagnosticText());
|
|
Assert.Contains(
|
|
host.JsonEvents(),
|
|
item => item.GetProperty("event").GetString() == "host.direct-traffic"
|
|
&& item.GetProperty("addressFamily").GetString() == "ipv4");
|
|
Assert.True(host.HasEvent("host.punch", "started"), host.DiagnosticText());
|
|
Assert.True(host.HasEvent("host.direct-connect", "connected"), host.DiagnosticText());
|
|
Assert.True(host.HasEvent("host.deregistered", "complete"), host.DiagnosticText());
|
|
Assert.Equal(host.AllLines().Count(), host.JsonEvents().Count);
|
|
Assert.Equal(emptyBrowse.AllLines().Count(), emptyBrowse.JsonEvents().Count);
|
|
Assert.Equal(missingJoin.AllLines().Count(), missingJoin.JsonEvents().Count);
|
|
Assert.Equal(browse.AllLines().Count(), browse.JsonEvents().Count);
|
|
Assert.Equal(join.AllLines().Count(), join.JsonEvents().Count);
|
|
Assert.All(
|
|
host.JsonEvents()
|
|
.Concat(emptyBrowse.JsonEvents())
|
|
.Concat(missingJoin.JsonEvents())
|
|
.Concat(browse.JsonEvents())
|
|
.Concat(join.JsonEvents()),
|
|
AssertAllowlistedEventShape);
|
|
|
|
using HttpClient service = new() { BaseAddress = new Uri(serviceUrl) };
|
|
using HttpResponseMessage removed = await service.GetAsync(
|
|
$"v1/sessions/{listingId}?contractVersion=1&gameId=space-game&environmentId=integration&protocolVersion=1",
|
|
timeout.Token);
|
|
Assert.Equal(HttpStatusCode.NotFound, removed.StatusCode);
|
|
|
|
string captured = string.Join(
|
|
'\n',
|
|
server.AllLines()
|
|
.Concat(boundedHost.AllLines())
|
|
.Concat(emptyBrowse.AllLines())
|
|
.Concat(missingJoin.AllLines())
|
|
.Concat(host.AllLines())
|
|
.Concat(browse.AllLines())
|
|
.Concat(join.AllLines()));
|
|
Assert.False(
|
|
captured.Contains(publisherCredential, StringComparison.Ordinal),
|
|
"Captured process output contained the publisher credential.");
|
|
Assert.False(
|
|
captured.Contains(runtime.SigningKeyText, StringComparison.Ordinal),
|
|
"Captured process output contained signing-key material.");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task WrongTenantAndProtocolAreRejectedBeforeAnyTraversalTraffic()
|
|
{
|
|
using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(20));
|
|
await using RunningService runtime = await RunningService.StartAsync(timeout.Token);
|
|
string[] hostArguments = HostArguments(runtime, runSeconds: 10);
|
|
await using ProcessCapture host = Start(
|
|
runtime.ClientAssembly,
|
|
hostArguments,
|
|
PublisherEnvironment(runtime.PublisherCredential));
|
|
JsonElement ready = await host.WaitForEventAsync("host.ready", timeout.Token);
|
|
string listingId = Assert.IsType<string>(ready.GetProperty("listingId").GetString());
|
|
|
|
string[] browseArguments = BrowseArguments(runtime, protocolVersion: 2);
|
|
await using ProcessCapture browse = Start(
|
|
runtime.ClientAssembly,
|
|
browseArguments,
|
|
new Dictionary<string, string>(StringComparer.Ordinal));
|
|
Assert.Equal(
|
|
(int)TestClientExitCode.NoCompatibleSession,
|
|
await browse.WaitForExitAsync(timeout.Token));
|
|
Assert.Contains(
|
|
browse.JsonEvents(),
|
|
item => item.GetProperty("event").GetString() == "browse.completed"
|
|
&& item.GetProperty("count").GetInt32() == 0);
|
|
|
|
string[] joinArguments = JoinArguments(runtime, listingId, protocolVersion: 2);
|
|
await using ProcessCapture join = Start(
|
|
runtime.ClientAssembly,
|
|
joinArguments,
|
|
new Dictionary<string, string>(StringComparer.Ordinal));
|
|
Assert.Equal((int)TestClientExitCode.ServiceFailure, await join.WaitForExitAsync(timeout.Token));
|
|
Assert.True(join.HasEvent("join.selection", "failed"), join.DiagnosticText());
|
|
Assert.DoesNotContain(
|
|
join.JsonEvents(),
|
|
item => item.GetProperty("event").GetString() == "join.punch");
|
|
|
|
string[] wrongTenantArguments = JoinArguments(runtime, listingId, gameId: "other-game");
|
|
await using ProcessCapture wrongTenant = Start(
|
|
runtime.ClientAssembly,
|
|
wrongTenantArguments,
|
|
new Dictionary<string, string>(StringComparer.Ordinal));
|
|
Assert.Equal(
|
|
(int)TestClientExitCode.ServiceFailure,
|
|
await wrongTenant.WaitForExitAsync(timeout.Token));
|
|
Assert.True(wrongTenant.HasEvent("join.selection", "failed"), wrongTenant.DiagnosticText());
|
|
Assert.DoesNotContain(
|
|
wrongTenant.JsonEvents(),
|
|
item => item.GetProperty("event").GetString() == "join.punch");
|
|
|
|
string[] controlBrowseArguments = BrowseArguments(runtime);
|
|
await using ProcessCapture controlBrowse = Start(
|
|
runtime.ClientAssembly,
|
|
controlBrowseArguments,
|
|
new Dictionary<string, string>(StringComparer.Ordinal));
|
|
Assert.Equal(0, await controlBrowse.WaitForExitAsync(timeout.Token));
|
|
Assert.Contains(
|
|
controlBrowse.JsonEvents(),
|
|
item => item.GetProperty("event").GetString() == "browse.session"
|
|
&& item.GetProperty("listingId").GetString() == listingId);
|
|
|
|
Assert.Equal(0, await host.WaitForExitAsync(timeout.Token));
|
|
Assert.True(host.HasEvent("host.deregistered", "complete"), host.DiagnosticText());
|
|
AssertNoSecrets(runtime, host, browse, join, wrongTenant, controlBrowse);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task TraversalTimeoutReturnsTypedFallbackWithoutConnectingToIt()
|
|
{
|
|
using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(20));
|
|
await using RunningService runtime = await RunningService.StartAsync(timeout.Token);
|
|
using UdpClient fallbackSentinel = new(new IPEndPoint(IPAddress.Loopback, 0));
|
|
int fallbackPort = Assert.IsType<IPEndPoint>(fallbackSentinel.Client.LocalEndPoint).Port;
|
|
using UdpClient mediatorBlackhole = new(new IPEndPoint(IPAddress.Loopback, 0));
|
|
int blackholePort = Assert.IsType<IPEndPoint>(mediatorBlackhole.Client.LocalEndPoint).Port;
|
|
string[] hostArguments = HostArguments(
|
|
runtime,
|
|
runSeconds: 5,
|
|
fallback: $"127.0.0.1:{fallbackPort}");
|
|
await using ProcessCapture host = Start(
|
|
runtime.ClientAssembly,
|
|
hostArguments,
|
|
PublisherEnvironment(runtime.PublisherCredential));
|
|
JsonElement ready = await host.WaitForEventAsync("host.ready", timeout.Token);
|
|
string listingId = Assert.IsType<string>(ready.GetProperty("listingId").GetString());
|
|
|
|
string[] joinArguments = JoinArguments(
|
|
runtime,
|
|
listingId,
|
|
timeoutSeconds: 2,
|
|
mediatorPort: blackholePort);
|
|
await using ProcessCapture join = Start(
|
|
runtime.ClientAssembly,
|
|
joinArguments,
|
|
new Dictionary<string, string>(StringComparer.Ordinal));
|
|
|
|
Assert.Equal((int)TestClientExitCode.TraversalFailed, await join.WaitForExitAsync(timeout.Token));
|
|
Assert.True(join.HasEvent("join.traversal", "failed"), join.DiagnosticText());
|
|
Assert.Contains(
|
|
join.JsonEvents(),
|
|
item => item.GetProperty("event").GetString() == "join.traversal"
|
|
&& string.Equals(
|
|
item.GetProperty("outcome").GetString(),
|
|
ConnectionOutcomeKind.PunchTimedOut.ToString(),
|
|
StringComparison.OrdinalIgnoreCase));
|
|
Assert.True(join.HasEvent("join.fallback", "available"), join.DiagnosticText());
|
|
Assert.DoesNotContain(
|
|
join.JsonEvents(),
|
|
item => item.GetProperty("event").GetString() == "join.connected");
|
|
Assert.True(mediatorBlackhole.Available > 0, "The retained black-hole socket received no punch traffic.");
|
|
Assert.Equal(0, fallbackSentinel.Available);
|
|
|
|
Assert.Equal(0, await host.WaitForExitAsync(timeout.Token));
|
|
Assert.True(host.HasEvent("host.deregistered", "complete"), host.DiagnosticText());
|
|
AssertNoSecrets(runtime, host, join);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task InterruptCancelsHostAndStillDeregistersItsListing()
|
|
{
|
|
if (!OperatingSystem.IsLinux() && !OperatingSystem.IsMacOS())
|
|
{
|
|
return;
|
|
}
|
|
|
|
using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(20));
|
|
await using RunningService runtime = await RunningService.StartAsync(timeout.Token);
|
|
string[] hostArguments = HostArguments(runtime, runSeconds: 30);
|
|
await using ProcessCapture host = Start(
|
|
runtime.ClientAssembly,
|
|
hostArguments,
|
|
PublisherEnvironment(runtime.PublisherCredential));
|
|
JsonElement ready = await host.WaitForEventAsync("host.ready", timeout.Token);
|
|
string listingId = Assert.IsType<string>(ready.GetProperty("listingId").GetString());
|
|
|
|
await host.InterruptAsync(timeout.Token);
|
|
|
|
Assert.Equal((int)TestClientExitCode.Cancelled, await host.WaitForExitAsync(timeout.Token));
|
|
Assert.True(host.HasEvent("host.deregistered", "complete"), host.DiagnosticText());
|
|
using HttpClient service = new() { BaseAddress = new Uri(runtime.ServiceUrl) };
|
|
using HttpResponseMessage removed = await service.GetAsync(
|
|
$"v1/sessions/{listingId}?contractVersion=1&gameId=space-game&environmentId=integration&protocolVersion=1",
|
|
timeout.Token);
|
|
Assert.Equal(HttpStatusCode.NotFound, removed.StatusCode);
|
|
AssertNoSecrets(runtime, host);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task PrivilegedLinuxNatNamespacesCompleteDirectTrafficAcrossSeparateObservedEndpoints()
|
|
{
|
|
if (!OperatingSystem.IsLinux()
|
|
|| !string.Equals(
|
|
Environment.GetEnvironmentVariable("RENDEZVOUS_RUN_NETNS_TESTS"),
|
|
"1",
|
|
StringComparison.Ordinal))
|
|
{
|
|
return;
|
|
}
|
|
|
|
using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(30));
|
|
await using NetworkNamespaceTopology topology = await NetworkNamespaceTopology.CreateAsync(
|
|
timeout.Token);
|
|
await using RunningService runtime = await RunningService.StartAsync(
|
|
timeout.Token,
|
|
listenAddress: "0.0.0.0",
|
|
processNamespace: topology.ServiceNamespace,
|
|
readinessAddress: topology.ServiceAddress);
|
|
int httpPort = new Uri(runtime.ServiceUrl).Port;
|
|
string serviceUrl = $"http://{topology.ServiceAddress}:{httpPort}/";
|
|
string mediator = $"{topology.ServiceAddress}:{runtime.UdpPort}";
|
|
string[] hostArguments =
|
|
[
|
|
"host",
|
|
"--service", serviceUrl,
|
|
"--mediator", mediator,
|
|
"--game", "space-game",
|
|
"--environment", "integration",
|
|
"--region", "local",
|
|
"--protocol", "1",
|
|
"--script",
|
|
"--json",
|
|
"--exit-after-echo",
|
|
"--timeout-seconds", "15",
|
|
];
|
|
await using ProcessCapture host = StartInNamespace(
|
|
topology.HostNamespace,
|
|
runtime.ClientAssembly,
|
|
hostArguments,
|
|
PublisherEnvironment(runtime.PublisherCredential));
|
|
JsonElement ready = await host.WaitForEventAsync("host.ready", timeout.Token);
|
|
string listingId = Assert.IsType<string>(ready.GetProperty("listingId").GetString());
|
|
string[] joinArguments =
|
|
[
|
|
"join",
|
|
"--service", serviceUrl,
|
|
"--mediator", mediator,
|
|
"--game", "space-game",
|
|
"--environment", "integration",
|
|
"--region", "local",
|
|
"--protocol", "1",
|
|
"--listing", listingId,
|
|
"--script",
|
|
"--json",
|
|
"--timeout-seconds", "15",
|
|
];
|
|
await using ProcessCapture join = StartInNamespace(
|
|
topology.ClientNamespace,
|
|
runtime.ClientAssembly,
|
|
joinArguments,
|
|
new Dictionary<string, string>(StringComparer.Ordinal));
|
|
|
|
Assert.Equal(0, await join.WaitForExitAsync(timeout.Token));
|
|
Assert.Equal(0, await host.WaitForExitAsync(timeout.Token));
|
|
Assert.True(join.HasEvent("join.direct-traffic", "verified"), join.DiagnosticText());
|
|
Assert.True(host.HasEvent("host.direct-traffic", "verified"), host.DiagnosticText());
|
|
Assert.Contains(
|
|
join.JsonEvents(),
|
|
item => item.GetProperty("event").GetString() == "join.connected"
|
|
&& item.GetProperty("endpointType").GetString() == "public");
|
|
AssertNoSecrets(runtime, host, join);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task AbruptHostLossBecomesStaleBeforeItsLeaseExpires()
|
|
{
|
|
using CancellationTokenSource setup = new(TimeSpan.FromSeconds(20));
|
|
await using RunningService runtime = await RunningService.StartAsync(setup.Token);
|
|
await using ProcessCapture host = Start(
|
|
runtime.ClientAssembly,
|
|
HostArguments(runtime, runSeconds: 30),
|
|
PublisherEnvironment(runtime.PublisherCredential));
|
|
JsonElement ready = await host.WaitForEventAsync("host.ready", setup.Token);
|
|
string listingId = Assert.IsType<string>(ready.GetProperty("listingId").GetString());
|
|
using HttpClient http = new() { BaseAddress = new Uri(runtime.ServiceUrl) };
|
|
RendezvousSessionBrowserClient browser = new(
|
|
http,
|
|
new RendezvousClientOptions
|
|
{
|
|
MaximumSafeRetries = 0,
|
|
RequestTimeout = TimeSpan.FromSeconds(1),
|
|
JitterRatio = 0,
|
|
});
|
|
BrowseSessionsRequest browseRequest = new()
|
|
{
|
|
GameId = new("space-game"),
|
|
EnvironmentId = new("integration"),
|
|
RegionId = new("local"),
|
|
ProtocolVersion = 1,
|
|
PageSize = 10,
|
|
};
|
|
RendezvousClientResult<IReadOnlyList<SessionListing>> visible = await browser.BrowseAllAsync(
|
|
browseRequest,
|
|
cancellationToken: setup.Token);
|
|
Assert.True(visible.IsSuccess, visible.Message);
|
|
Assert.Contains(visible.Value!, item => item.ListingId.ToString() == listingId);
|
|
|
|
await host.TerminateAsync(setup.Token);
|
|
Stopwatch staleWait = Stopwatch.StartNew();
|
|
using CancellationTokenSource presenceExpiry = new(TimeSpan.FromSeconds(30));
|
|
|
|
using PeriodicTimer probe = new(TimeSpan.FromMilliseconds(100));
|
|
RendezvousClientResult<IReadOnlyList<SessionListing>> afterLoss;
|
|
do
|
|
{
|
|
afterLoss = await browser.BrowseAllAsync(
|
|
browseRequest,
|
|
cancellationToken: presenceExpiry.Token);
|
|
Assert.True(afterLoss.IsSuccess, afterLoss.Message);
|
|
if (afterLoss.Value!.Count == 0)
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
while (await probe.WaitForNextTickAsync(presenceExpiry.Token));
|
|
|
|
Assert.Empty(afterLoss.Value!);
|
|
Assert.True(
|
|
staleWait.Elapsed < TimeSpan.FromSeconds(30),
|
|
$"Presence did not become stale before the 60-second lease lifetime: {staleWait.Elapsed}.");
|
|
RendezvousClientResult<GetSessionResponse> exact = await browser.GetAsync(
|
|
new SessionListingId(Guid.Parse(listingId)),
|
|
new GameId("space-game"),
|
|
new EnvironmentId("integration"),
|
|
1,
|
|
presenceExpiry.Token);
|
|
Assert.False(exact.IsSuccess);
|
|
Assert.Equal(RendezvousErrorCode.NotFound, exact.Error);
|
|
|
|
string[] browseArguments = BrowseArguments(runtime);
|
|
await using ProcessCapture publicBrowse = Start(
|
|
runtime.ClientAssembly,
|
|
browseArguments,
|
|
new Dictionary<string, string>(StringComparer.Ordinal));
|
|
Assert.Equal(
|
|
(int)TestClientExitCode.NoCompatibleSession,
|
|
await publicBrowse.WaitForExitAsync(presenceExpiry.Token));
|
|
AssertNoSecrets(runtime, host, publicBrowse);
|
|
}
|
|
|
|
private static string[] HostArguments(
|
|
RunningService runtime,
|
|
int runSeconds,
|
|
string? fallback = null)
|
|
{
|
|
List<string> arguments =
|
|
[
|
|
"host",
|
|
"--service", runtime.ServiceUrl,
|
|
"--mediator", $"127.0.0.1:{runtime.UdpPort}",
|
|
"--game", "space-game",
|
|
"--environment", "integration",
|
|
"--region", "local",
|
|
"--protocol", "1",
|
|
"--script",
|
|
"--json",
|
|
"--run-seconds", runSeconds.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
|
"--timeout-seconds", "10",
|
|
];
|
|
if (fallback is not null)
|
|
{
|
|
arguments.Add("--fallback");
|
|
arguments.Add(fallback);
|
|
}
|
|
return [.. arguments];
|
|
}
|
|
|
|
private static string[] BrowseArguments(RunningService runtime, int protocolVersion = 1) =>
|
|
[
|
|
"browse",
|
|
"--service", runtime.ServiceUrl,
|
|
"--game", "space-game",
|
|
"--environment", "integration",
|
|
"--region", "local",
|
|
"--protocol", protocolVersion.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
|
"--script",
|
|
"--json",
|
|
"--timeout-seconds", "5",
|
|
];
|
|
|
|
private static string[] JoinArguments(
|
|
RunningService runtime,
|
|
string listingId,
|
|
int protocolVersion = 1,
|
|
string gameId = "space-game",
|
|
int timeoutSeconds = 5,
|
|
int? mediatorPort = null) =>
|
|
[
|
|
"join",
|
|
"--service", runtime.ServiceUrl,
|
|
"--mediator", $"127.0.0.1:{mediatorPort ?? runtime.UdpPort}",
|
|
"--game", gameId,
|
|
"--environment", "integration",
|
|
"--region", "local",
|
|
"--protocol", protocolVersion.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
|
"--listing", listingId,
|
|
"--script",
|
|
"--json",
|
|
"--timeout-seconds", timeoutSeconds.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
|
];
|
|
|
|
private static Dictionary<string, string> PublisherEnvironment(string credential) =>
|
|
new(StringComparer.Ordinal)
|
|
{
|
|
["RENDEZVOUS_PUBLISHER_CREDENTIAL"] = credential,
|
|
};
|
|
|
|
private static void AssertNoSecrets(RunningService runtime, params ProcessCapture[] processes)
|
|
{
|
|
foreach (ProcessCapture process in processes)
|
|
{
|
|
Assert.Equal(process.AllLines().Count(), process.JsonEvents().Count);
|
|
Assert.All(process.JsonEvents(), AssertAllowlistedEventShape);
|
|
}
|
|
string captured = string.Join(
|
|
'\n',
|
|
runtime.Server.AllLines().Concat(processes.SelectMany(static process => process.AllLines())));
|
|
Assert.DoesNotContain(runtime.PublisherCredential, captured, StringComparison.Ordinal);
|
|
Assert.DoesNotContain(runtime.SigningKeyText, captured, StringComparison.Ordinal);
|
|
}
|
|
|
|
private static string IssuePublisherCredential(byte[] signingKey, DateTimeOffset now)
|
|
{
|
|
const string secretReference = "env:RENDEZVOUS_INTEGRATION_SIGNING_KEY";
|
|
ProvisioningOptions options = ProvisioningOptions(now, secretReference);
|
|
using DictionarySecretProvider secrets = new(new Dictionary<string, byte[]>(StringComparer.Ordinal)
|
|
{
|
|
[secretReference] = signingKey,
|
|
});
|
|
using ProvisioningRuntime provisioning = ProvisioningRuntime.Create(options, secrets, now);
|
|
return provisioning.Credentials.Issue(
|
|
new DedicatedPublisherPrincipal(
|
|
"test-client-process-host",
|
|
now.AddMinutes(5),
|
|
new GameId("space-game"),
|
|
new EnvironmentId("integration"),
|
|
[new RegionId("local")]),
|
|
now);
|
|
}
|
|
|
|
private static ProvisioningOptions ProvisioningOptions(
|
|
DateTimeOffset now,
|
|
string secretReference) => new()
|
|
{
|
|
Issuer = "rendezvous-process-test",
|
|
Audience = "rendezvous-process-test-client",
|
|
ClockSkewSeconds = 5,
|
|
SigningKeys =
|
|
[
|
|
new SigningKeyOptions
|
|
{
|
|
KeyId = "process-test-key",
|
|
SecretReference = secretReference,
|
|
CredentialKinds = [PrincipalCredentialKind.DedicatedPublisher],
|
|
GameId = "space-game",
|
|
EnvironmentId = "integration",
|
|
NotBefore = now.AddMinutes(-1),
|
|
SignUntil = now.AddMinutes(10),
|
|
VerifyUntil = now.AddMinutes(20),
|
|
},
|
|
],
|
|
Games =
|
|
[
|
|
new GamePolicyOptions
|
|
{
|
|
GameId = "space-game",
|
|
EnvironmentId = "integration",
|
|
Enabled = true,
|
|
ProtocolVersions = [1],
|
|
Regions = ["local"],
|
|
VisibilityModes = [ListingVisibility.Public],
|
|
PublisherTrustModes = [PublisherTrustMode.ManagedDedicated],
|
|
MetadataMaxBytes = 256,
|
|
MetadataMaxKeys = 2,
|
|
MaxListingsPerPrincipal = 4,
|
|
MaxAnonymousListingsPerAddress = 1,
|
|
MaxActiveJoinAttempts = 16,
|
|
},
|
|
],
|
|
};
|
|
|
|
private static Dictionary<string, string> ServerEnvironment(
|
|
string signingKey,
|
|
DateTimeOffset now,
|
|
string listenAddress,
|
|
string? readinessAddress)
|
|
{
|
|
string advertisedAddress = readinessAddress ?? listenAddress;
|
|
string allowedHosts = string.Join(
|
|
';',
|
|
new[] { advertisedAddress, listenAddress, "127.0.0.1", "localhost" }
|
|
.Distinct(StringComparer.OrdinalIgnoreCase));
|
|
Dictionary<string, string> values = new(StringComparer.Ordinal)
|
|
{
|
|
["ASPNETCORE_ENVIRONMENT"] = "Production",
|
|
["ASPNETCORE_URLS"] = $"http://{listenAddress}:0",
|
|
["AllowedHosts"] = allowedHosts,
|
|
["Rendezvous__Deployment__PublicHttpBaseUrl"] = $"https://{advertisedAddress}/",
|
|
["Rendezvous__Deployment__PublicUdpHost"] = advertisedAddress,
|
|
["Rendezvous__Deployment__PublicUdpPort"] = "9050",
|
|
["Rendezvous__Deployment__DrainDeadlineSeconds"] = "3",
|
|
["Rendezvous__Deployment__MinimumDrainSeconds"] = "1",
|
|
["Rendezvous__Deployment__SingleActiveInstance"] = "true",
|
|
["Rendezvous__Deployment__AllowPrivatePublicEndpoints"] = "true",
|
|
["Rendezvous__Udp__ListenAddress"] = listenAddress,
|
|
["Rendezvous__Udp__Port"] = "0",
|
|
["Rendezvous__Udp__PollIntervalMilliseconds"] = "1",
|
|
["Rendezvous__AbuseProtection__TrustedProxyAddresses__0"] = "127.0.0.1",
|
|
["Rendezvous__Provisioning__Issuer"] = "rendezvous-process-test",
|
|
["Rendezvous__Provisioning__Audience"] = "rendezvous-process-test-client",
|
|
["Rendezvous__Provisioning__ClockSkewSeconds"] = "5",
|
|
["Rendezvous__Provisioning__SigningKeys__0__KeyId"] = "process-test-key",
|
|
["Rendezvous__Provisioning__SigningKeys__0__SecretReference"] =
|
|
"env:RENDEZVOUS_INTEGRATION_SIGNING_KEY",
|
|
["Rendezvous__Provisioning__SigningKeys__0__CredentialKinds__0"] = "DedicatedPublisher",
|
|
["Rendezvous__Provisioning__SigningKeys__0__GameId"] = "space-game",
|
|
["Rendezvous__Provisioning__SigningKeys__0__EnvironmentId"] = "integration",
|
|
["Rendezvous__Provisioning__SigningKeys__0__NotBefore"] = now.AddMinutes(-1).ToString("O"),
|
|
["Rendezvous__Provisioning__SigningKeys__0__SignUntil"] = now.AddMinutes(10).ToString("O"),
|
|
["Rendezvous__Provisioning__SigningKeys__0__VerifyUntil"] = now.AddMinutes(20).ToString("O"),
|
|
["Rendezvous__Provisioning__Games__0__GameId"] = "space-game",
|
|
["Rendezvous__Provisioning__Games__0__EnvironmentId"] = "integration",
|
|
["Rendezvous__Provisioning__Games__0__Enabled"] = "true",
|
|
["Rendezvous__Provisioning__Games__0__ProtocolVersions__0"] = "1",
|
|
["Rendezvous__Provisioning__Games__0__Regions__0"] = "local",
|
|
["Rendezvous__Provisioning__Games__0__VisibilityModes__0"] = "Public",
|
|
["Rendezvous__Provisioning__Games__0__PublisherTrustModes__0"] = "ManagedDedicated",
|
|
["Rendezvous__Provisioning__Games__0__MetadataMaxBytes"] = "256",
|
|
["Rendezvous__Provisioning__Games__0__MetadataMaxKeys"] = "2",
|
|
["Rendezvous__Provisioning__Games__0__MaxListingsPerPrincipal"] = "4",
|
|
["Rendezvous__Provisioning__Games__0__MaxAnonymousListingsPerAddress"] = "1",
|
|
["Rendezvous__Provisioning__Games__0__MaxActiveJoinAttempts"] = "16",
|
|
["Rendezvous__Provisioning__Games__0__FallbackPolicy"] = "DedicatedEndpointAllowed",
|
|
["RENDEZVOUS_INTEGRATION_SIGNING_KEY"] = signingKey,
|
|
};
|
|
return values;
|
|
}
|
|
|
|
private static ProcessCapture Start(
|
|
string assembly,
|
|
IReadOnlyList<string> arguments,
|
|
IReadOnlyDictionary<string, string> environment)
|
|
{
|
|
ProcessStartInfo start = new()
|
|
{
|
|
FileName = Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") ?? "dotnet",
|
|
UseShellExecute = false,
|
|
RedirectStandardOutput = true,
|
|
RedirectStandardError = true,
|
|
CreateNoWindow = true,
|
|
};
|
|
start.ArgumentList.Add(assembly);
|
|
foreach (string argument in arguments)
|
|
{
|
|
start.ArgumentList.Add(argument);
|
|
}
|
|
foreach (KeyValuePair<string, string> item in environment)
|
|
{
|
|
start.Environment[item.Key] = item.Value;
|
|
}
|
|
return new ProcessCapture(start);
|
|
}
|
|
|
|
private static ProcessCapture StartInNamespace(
|
|
string namespaceName,
|
|
string assembly,
|
|
IReadOnlyList<string> arguments,
|
|
IReadOnlyDictionary<string, string> environment)
|
|
{
|
|
ProcessStartInfo start = new()
|
|
{
|
|
FileName = "ip",
|
|
UseShellExecute = false,
|
|
RedirectStandardOutput = true,
|
|
RedirectStandardError = true,
|
|
CreateNoWindow = true,
|
|
};
|
|
start.ArgumentList.Add("netns");
|
|
start.ArgumentList.Add("exec");
|
|
start.ArgumentList.Add(namespaceName);
|
|
start.ArgumentList.Add(Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") ?? "dotnet");
|
|
start.ArgumentList.Add(assembly);
|
|
foreach (string argument in arguments)
|
|
{
|
|
start.ArgumentList.Add(argument);
|
|
}
|
|
foreach (KeyValuePair<string, string> item in environment)
|
|
{
|
|
start.Environment[item.Key] = item.Value;
|
|
}
|
|
return new ProcessCapture(start);
|
|
}
|
|
|
|
private static async Task WaitForReadyAsync(
|
|
string serviceUrl,
|
|
ProcessCapture server,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
using HttpClient client = new() { BaseAddress = new Uri(serviceUrl) };
|
|
while (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
if (server.HasExited)
|
|
{
|
|
throw new Xunit.Sdk.XunitException($"Server exited before readiness. {server.DiagnosticText()}");
|
|
}
|
|
try
|
|
{
|
|
using HttpResponseMessage response = await client.GetAsync(
|
|
"health/ready",
|
|
cancellationToken);
|
|
if (response.StatusCode == HttpStatusCode.OK)
|
|
{
|
|
return;
|
|
}
|
|
}
|
|
catch (HttpRequestException)
|
|
{
|
|
}
|
|
await Task.Delay(25, cancellationToken);
|
|
}
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
}
|
|
|
|
private static string ParseServiceUrl(string line)
|
|
{
|
|
const string marker = "Now listening on: ";
|
|
string value = line[(line.IndexOf(marker, StringComparison.Ordinal) + marker.Length)..].Trim();
|
|
return new Uri(value).AbsoluteUri;
|
|
}
|
|
|
|
private static int ParseTrailingPort(string line)
|
|
{
|
|
string value = line[(line.LastIndexOf(':') + 1)..].Trim();
|
|
return int.Parse(value, System.Globalization.CultureInfo.InvariantCulture);
|
|
}
|
|
|
|
private static string RepositoryRoot()
|
|
{
|
|
DirectoryInfo? directory = new(AppContext.BaseDirectory);
|
|
while (directory is not null)
|
|
{
|
|
if (File.Exists(Path.Combine(directory.FullName, "Rendezvous.slnx")))
|
|
{
|
|
return directory.FullName;
|
|
}
|
|
directory = directory.Parent;
|
|
}
|
|
throw new DirectoryNotFoundException("Could not locate the Rendezvous repository root.");
|
|
}
|
|
|
|
private static void AssertAllowlistedEventShape(JsonElement item)
|
|
{
|
|
string[] forbidden = ["credential", "capability", "ticket", "token", "secret", "metadata"];
|
|
foreach (JsonProperty property in item.EnumerateObject())
|
|
{
|
|
Assert.DoesNotContain(forbidden, forbiddenName =>
|
|
property.Name.Contains(forbiddenName, StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
}
|
|
|
|
private sealed class NetworkNamespaceTopology : IAsyncDisposable
|
|
{
|
|
private readonly string _bridge;
|
|
private readonly string _serviceRootInterface;
|
|
private readonly string _serviceWanInterface;
|
|
private readonly string _hostWanRootInterface;
|
|
private readonly string _hostWanInterface;
|
|
private readonly string _clientWanRootInterface;
|
|
private readonly string _clientWanInterface;
|
|
private readonly string _hostLanInterface;
|
|
private readonly string _hostAppInterface;
|
|
private readonly string _clientLanInterface;
|
|
private readonly string _clientAppInterface;
|
|
private readonly HashSet<string> _ownedNamespaces = new(StringComparer.Ordinal);
|
|
private readonly HashSet<string> _ownedRootLinks = new(StringComparer.Ordinal);
|
|
private bool _bridgeCreated;
|
|
private bool _created;
|
|
|
|
private NetworkNamespaceTopology(
|
|
string suffix,
|
|
int wanSecondOctet,
|
|
int wanThirdOctet,
|
|
int hostThirdOctet,
|
|
int clientThirdOctet)
|
|
{
|
|
_bridge = $"rvb{suffix}";
|
|
_serviceRootInterface = $"rvsr{suffix}";
|
|
_serviceWanInterface = $"rvsw{suffix}";
|
|
_hostWanRootInterface = $"rvhr{suffix}";
|
|
_hostWanInterface = $"rvhw{suffix}";
|
|
_clientWanRootInterface = $"rvcr{suffix}";
|
|
_clientWanInterface = $"rvcw{suffix}";
|
|
_hostLanInterface = $"rvhl{suffix}";
|
|
_hostAppInterface = $"rvha{suffix}";
|
|
_clientLanInterface = $"rvcl{suffix}";
|
|
_clientAppInterface = $"rvca{suffix}";
|
|
ServiceNamespace = $"rv-service-{suffix}";
|
|
HostRouterNamespace = $"rv-host-nat-{suffix}";
|
|
HostNamespace = $"rv-host-{suffix}";
|
|
ClientRouterNamespace = $"rv-client-nat-{suffix}";
|
|
ClientNamespace = $"rv-client-{suffix}";
|
|
NetworkCidr = $"198.{wanSecondOctet}.{wanThirdOctet}.0/24";
|
|
RootWanAddress = $"198.{wanSecondOctet}.{wanThirdOctet}.1";
|
|
ServiceAddress = $"198.{wanSecondOctet}.{wanThirdOctet}.2";
|
|
HostWanAddress = $"198.{wanSecondOctet}.{wanThirdOctet}.3";
|
|
ClientWanAddress = $"198.{wanSecondOctet}.{wanThirdOctet}.4";
|
|
HostRouterAddress = $"10.71.{hostThirdOctet}.1";
|
|
HostAddress = $"10.71.{hostThirdOctet}.2";
|
|
ClientRouterAddress = $"10.72.{clientThirdOctet}.1";
|
|
ClientAddress = $"10.72.{clientThirdOctet}.2";
|
|
}
|
|
|
|
internal string ServiceNamespace { get; }
|
|
internal string HostNamespace { get; }
|
|
internal string ClientNamespace { get; }
|
|
internal string ServiceAddress { get; }
|
|
private string HostRouterNamespace { get; }
|
|
private string ClientRouterNamespace { get; }
|
|
private string NetworkCidr { get; }
|
|
private string RootWanAddress { get; }
|
|
private string HostWanAddress { get; }
|
|
private string ClientWanAddress { get; }
|
|
private string HostRouterAddress { get; }
|
|
private string HostAddress { get; }
|
|
private string ClientRouterAddress { get; }
|
|
private string ClientAddress { get; }
|
|
|
|
internal static async Task<NetworkNamespaceTopology> CreateAsync(
|
|
CancellationToken cancellationToken)
|
|
{
|
|
for (int attempt = 0; attempt < 20; attempt++)
|
|
{
|
|
string suffix = Convert.ToHexString(RandomNumberGenerator.GetBytes(3)).ToLowerInvariant();
|
|
NetworkNamespaceTopology topology = new(
|
|
suffix,
|
|
RandomNumberGenerator.GetInt32(18, 20),
|
|
RandomNumberGenerator.GetInt32(0, 256),
|
|
RandomNumberGenerator.GetInt32(0, 256),
|
|
RandomNumberGenerator.GetInt32(0, 256));
|
|
IpCommandResult bridge = await ExecuteIpAsync(
|
|
["link", "show", "dev", topology._bridge],
|
|
cancellationToken);
|
|
IpCommandResult route = await ExecuteIpAsync(
|
|
["-4", "route", "show", "exact", topology.NetworkCidr],
|
|
cancellationToken);
|
|
if (bridge.ExitCode == 0
|
|
|| route.ExitCode != 0
|
|
|| !string.IsNullOrWhiteSpace(route.Output)
|
|
|| await topology.HasNameCollisionAsync(cancellationToken))
|
|
{
|
|
continue;
|
|
}
|
|
try
|
|
{
|
|
await topology.CreateCoreAsync(cancellationToken);
|
|
topology._created = true;
|
|
return topology;
|
|
}
|
|
catch
|
|
{
|
|
await topology.CleanupAsync(requireSuccess: false, CancellationToken.None);
|
|
throw;
|
|
}
|
|
}
|
|
throw new Xunit.Sdk.XunitException(
|
|
"Could not allocate a collision-free namespace name and private subnet after 20 attempts.");
|
|
}
|
|
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
if (_created)
|
|
{
|
|
await CleanupAsync(requireSuccess: true, CancellationToken.None);
|
|
_created = false;
|
|
}
|
|
}
|
|
|
|
private async Task CreateCoreAsync(CancellationToken cancellationToken)
|
|
{
|
|
await AddOwnedBridgeAsync(cancellationToken);
|
|
await RunIpAsync(["addr", "add", $"{RootWanAddress}/24", "dev", _bridge], true, cancellationToken);
|
|
await RunIpAsync(["link", "set", _bridge, "up"], true, cancellationToken);
|
|
await AddNamespaceAsync(ServiceNamespace, cancellationToken);
|
|
await AddNamespaceAsync(HostRouterNamespace, cancellationToken);
|
|
await AddNamespaceAsync(HostNamespace, cancellationToken);
|
|
await AddNamespaceAsync(ClientRouterNamespace, cancellationToken);
|
|
await AddNamespaceAsync(ClientNamespace, cancellationToken);
|
|
await ConfigureWanPeerAsync(
|
|
ServiceNamespace,
|
|
_serviceRootInterface,
|
|
_serviceWanInterface,
|
|
ServiceAddress,
|
|
cancellationToken);
|
|
await ConfigureWanPeerAsync(
|
|
HostRouterNamespace,
|
|
_hostWanRootInterface,
|
|
_hostWanInterface,
|
|
HostWanAddress,
|
|
cancellationToken);
|
|
await ConfigureWanPeerAsync(
|
|
ClientRouterNamespace,
|
|
_clientWanRootInterface,
|
|
_clientWanInterface,
|
|
ClientWanAddress,
|
|
cancellationToken);
|
|
await ConfigureNatPeerAsync(
|
|
HostRouterNamespace,
|
|
HostNamespace,
|
|
_hostWanInterface,
|
|
_hostLanInterface,
|
|
_hostAppInterface,
|
|
HostRouterAddress,
|
|
HostAddress,
|
|
cancellationToken);
|
|
await ConfigureNatPeerAsync(
|
|
ClientRouterNamespace,
|
|
ClientNamespace,
|
|
_clientWanInterface,
|
|
_clientLanInterface,
|
|
_clientAppInterface,
|
|
ClientRouterAddress,
|
|
ClientAddress,
|
|
cancellationToken);
|
|
}
|
|
|
|
private async Task AddNamespaceAsync(string namespaceName, CancellationToken cancellationToken)
|
|
{
|
|
try
|
|
{
|
|
await RunIpAsync(["netns", "add", namespaceName], true, cancellationToken);
|
|
_ownedNamespaces.Add(namespaceName);
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
if (await NamespaceExistsAsync(namespaceName))
|
|
{
|
|
_ownedNamespaces.Add(namespaceName);
|
|
}
|
|
throw;
|
|
}
|
|
}
|
|
|
|
private async Task AddOwnedBridgeAsync(CancellationToken cancellationToken)
|
|
{
|
|
try
|
|
{
|
|
await RunIpAsync(["link", "add", _bridge, "type", "bridge"], true, cancellationToken);
|
|
_bridgeCreated = true;
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
_bridgeCreated = await LinkExistsAsync(_bridge);
|
|
throw;
|
|
}
|
|
}
|
|
|
|
private async Task AddOwnedRootLinkAsync(
|
|
string rootInterface,
|
|
string peerInterface,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
try
|
|
{
|
|
await RunIpAsync(
|
|
["link", "add", rootInterface, "type", "veth", "peer", "name", peerInterface],
|
|
true,
|
|
cancellationToken);
|
|
_ownedRootLinks.Add(rootInterface);
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
if (await LinkExistsAsync(rootInterface))
|
|
{
|
|
_ownedRootLinks.Add(rootInterface);
|
|
}
|
|
throw;
|
|
}
|
|
}
|
|
|
|
private async Task ConfigureWanPeerAsync(
|
|
string namespaceName,
|
|
string rootInterface,
|
|
string peerInterface,
|
|
string address,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
await AddOwnedRootLinkAsync(rootInterface, peerInterface, cancellationToken);
|
|
await RunIpAsync(["link", "set", peerInterface, "netns", namespaceName], true, cancellationToken);
|
|
await RunIpAsync(["link", "set", rootInterface, "master", _bridge], true, cancellationToken);
|
|
await RunIpAsync(["link", "set", rootInterface, "up"], true, cancellationToken);
|
|
await RunIpAsync(["-n", namespaceName, "addr", "add", $"{address}/24", "dev", peerInterface], true, cancellationToken);
|
|
await RunIpAsync(["-n", namespaceName, "link", "set", peerInterface, "up"], true, cancellationToken);
|
|
await RunIpAsync(["-n", namespaceName, "link", "set", "lo", "up"], true, cancellationToken);
|
|
}
|
|
|
|
private async Task ConfigureNatPeerAsync(
|
|
string routerNamespace,
|
|
string appNamespace,
|
|
string wanInterface,
|
|
string lanInterface,
|
|
string appInterface,
|
|
string routerAddress,
|
|
string appAddress,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
await AddOwnedRootLinkAsync(lanInterface, appInterface, cancellationToken);
|
|
await RunIpAsync(["link", "set", lanInterface, "netns", routerNamespace], true, cancellationToken);
|
|
await RunIpAsync(["link", "set", appInterface, "netns", appNamespace], true, cancellationToken);
|
|
await RunIpAsync(["-n", routerNamespace, "addr", "add", $"{routerAddress}/24", "dev", lanInterface], true, cancellationToken);
|
|
await RunIpAsync(["-n", routerNamespace, "link", "set", lanInterface, "up"], true, cancellationToken);
|
|
await RunIpAsync(["-n", appNamespace, "addr", "add", $"{appAddress}/24", "dev", appInterface], true, cancellationToken);
|
|
await RunIpAsync(["-n", appNamespace, "link", "set", appInterface, "up"], true, cancellationToken);
|
|
await RunIpAsync(["-n", appNamespace, "link", "set", "lo", "up"], true, cancellationToken);
|
|
await RunIpAsync(["-n", appNamespace, "route", "add", "default", "via", routerAddress], true, cancellationToken);
|
|
await RunIpAsync(
|
|
["netns", "exec", routerNamespace, "sysctl", "-q", "-w", "net.ipv4.ip_forward=1"],
|
|
true,
|
|
cancellationToken);
|
|
await RunIpAsync(
|
|
["netns", "exec", routerNamespace, "iptables", "-t", "nat", "-A", "POSTROUTING", "-o", wanInterface, "-j", "MASQUERADE"],
|
|
true,
|
|
cancellationToken);
|
|
await RunIpAsync(
|
|
["netns", "exec", routerNamespace, "iptables", "-A", "FORWARD", "-i", lanInterface, "-o", wanInterface, "-j", "ACCEPT"],
|
|
true,
|
|
cancellationToken);
|
|
await RunIpAsync(
|
|
["netns", "exec", routerNamespace, "iptables", "-A", "FORWARD", "-i", wanInterface, "-o", lanInterface, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"],
|
|
true,
|
|
cancellationToken);
|
|
}
|
|
|
|
private async Task<bool> HasNameCollisionAsync(CancellationToken cancellationToken)
|
|
{
|
|
string[] namespaces =
|
|
[
|
|
ServiceNamespace,
|
|
HostRouterNamespace,
|
|
HostNamespace,
|
|
ClientRouterNamespace,
|
|
ClientNamespace,
|
|
];
|
|
foreach (string namespaceName in namespaces)
|
|
{
|
|
IpCommandResult existing = await ExecuteIpAsync(
|
|
["netns", "exec", namespaceName, "true"],
|
|
cancellationToken);
|
|
if (existing.ExitCode == 0)
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
string[] links =
|
|
[
|
|
_serviceRootInterface,
|
|
_serviceWanInterface,
|
|
_hostWanRootInterface,
|
|
_hostWanInterface,
|
|
_clientWanRootInterface,
|
|
_clientWanInterface,
|
|
_hostLanInterface,
|
|
_hostAppInterface,
|
|
_clientLanInterface,
|
|
_clientAppInterface,
|
|
];
|
|
foreach (string link in links)
|
|
{
|
|
IpCommandResult existing = await ExecuteIpAsync(
|
|
["link", "show", "dev", link],
|
|
cancellationToken);
|
|
if (existing.ExitCode == 0)
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
private static async Task<bool> LinkExistsAsync(string link)
|
|
{
|
|
using CancellationTokenSource probe = new(TimeSpan.FromSeconds(2));
|
|
IpCommandResult existing = await ExecuteIpAsync(
|
|
["link", "show", "dev", link],
|
|
probe.Token);
|
|
return existing.ExitCode == 0;
|
|
}
|
|
|
|
private static async Task<bool> NamespaceExistsAsync(string namespaceName)
|
|
{
|
|
using CancellationTokenSource probe = new(TimeSpan.FromSeconds(2));
|
|
IpCommandResult existing = await ExecuteIpAsync(
|
|
["netns", "exec", namespaceName, "true"],
|
|
probe.Token);
|
|
return existing.ExitCode == 0;
|
|
}
|
|
|
|
private async Task CleanupAsync(bool requireSuccess, CancellationToken cancellationToken)
|
|
{
|
|
Exception? firstFailure = null;
|
|
string[] namespaceOrder =
|
|
[
|
|
HostNamespace,
|
|
ClientNamespace,
|
|
HostRouterNamespace,
|
|
ClientRouterNamespace,
|
|
ServiceNamespace,
|
|
];
|
|
List<IReadOnlyList<string>> commands = namespaceOrder
|
|
.Where(_ownedNamespaces.Contains)
|
|
.Select(static name => (IReadOnlyList<string>)["netns", "delete", name])
|
|
.ToList();
|
|
if (_bridgeCreated)
|
|
{
|
|
commands.Add(["link", "delete", _bridge]);
|
|
}
|
|
foreach (IReadOnlyList<string> command in commands)
|
|
{
|
|
try
|
|
{
|
|
using CancellationTokenSource step = CancellationTokenSource.CreateLinkedTokenSource(
|
|
cancellationToken);
|
|
step.CancelAfter(TimeSpan.FromSeconds(5));
|
|
await RunIpAsync(command, requireSuccess, step.Token);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
if (requireSuccess)
|
|
{
|
|
firstFailure ??= exception;
|
|
}
|
|
}
|
|
}
|
|
foreach (string link in _ownedRootLinks)
|
|
{
|
|
try
|
|
{
|
|
using CancellationTokenSource step = CancellationTokenSource.CreateLinkedTokenSource(
|
|
cancellationToken);
|
|
step.CancelAfter(TimeSpan.FromSeconds(5));
|
|
await RunIpAsync(["link", "delete", link], requireSuccess: false, step.Token);
|
|
IpCommandResult remaining = await ExecuteIpAsync(
|
|
["link", "show", "dev", link],
|
|
step.Token);
|
|
if (remaining.ExitCode == 0)
|
|
{
|
|
throw new Xunit.Sdk.XunitException($"Owned topology link {link} remained after cleanup.");
|
|
}
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
if (requireSuccess)
|
|
{
|
|
firstFailure ??= exception;
|
|
}
|
|
}
|
|
}
|
|
if (firstFailure is not null)
|
|
{
|
|
throw new Xunit.Sdk.XunitException(
|
|
$"Network namespace cleanup failed: {firstFailure.Message}");
|
|
}
|
|
}
|
|
|
|
private static async Task RunIpAsync(
|
|
IReadOnlyList<string> arguments,
|
|
bool requireSuccess,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
IpCommandResult result = await ExecuteIpAsync(arguments, cancellationToken);
|
|
if (requireSuccess && result.ExitCode != 0)
|
|
{
|
|
throw new Xunit.Sdk.XunitException(
|
|
$"ip topology command failed with exit {result.ExitCode}; stdout={result.Output}; stderr={result.Error}");
|
|
}
|
|
}
|
|
|
|
private static async Task<IpCommandResult> ExecuteIpAsync(
|
|
IReadOnlyList<string> arguments,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
ProcessStartInfo start = new()
|
|
{
|
|
FileName = "ip",
|
|
UseShellExecute = false,
|
|
RedirectStandardOutput = true,
|
|
RedirectStandardError = true,
|
|
CreateNoWindow = true,
|
|
};
|
|
foreach (string argument in arguments)
|
|
{
|
|
start.ArgumentList.Add(argument);
|
|
}
|
|
using Process process = Process.Start(start)
|
|
?? throw new Xunit.Sdk.XunitException("Failed to start the ip topology command.");
|
|
Task<string> output = process.StandardOutput.ReadToEndAsync(CancellationToken.None);
|
|
Task<string> error = process.StandardError.ReadToEndAsync(CancellationToken.None);
|
|
try
|
|
{
|
|
await process.WaitForExitAsync(cancellationToken);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
if (!process.HasExited)
|
|
{
|
|
process.Kill(entireProcessTree: true);
|
|
}
|
|
using CancellationTokenSource reap = new(TimeSpan.FromSeconds(2));
|
|
try
|
|
{
|
|
await process.WaitForExitAsync(reap.Token);
|
|
await Task.WhenAll(output, error);
|
|
if (process.ExitCode == 0)
|
|
{
|
|
return new IpCommandResult(process.ExitCode, output.Result, error.Result);
|
|
}
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
throw new Xunit.Sdk.XunitException(
|
|
"A timed-out ip topology helper did not exit after it was killed.");
|
|
}
|
|
throw;
|
|
}
|
|
await Task.WhenAll(output, error);
|
|
return new IpCommandResult(process.ExitCode, output.Result, error.Result);
|
|
}
|
|
|
|
private sealed record IpCommandResult(int ExitCode, string Output, string Error);
|
|
}
|
|
|
|
private sealed class RunningService : IAsyncDisposable
|
|
{
|
|
private RunningService(
|
|
ProcessCapture server,
|
|
string clientAssembly,
|
|
string serviceUrl,
|
|
int udpPort,
|
|
string publisherCredential,
|
|
string signingKeyText)
|
|
{
|
|
Server = server;
|
|
ClientAssembly = clientAssembly;
|
|
ServiceUrl = serviceUrl;
|
|
UdpPort = udpPort;
|
|
PublisherCredential = publisherCredential;
|
|
SigningKeyText = signingKeyText;
|
|
}
|
|
|
|
internal ProcessCapture Server { get; }
|
|
internal string ClientAssembly { get; }
|
|
internal string ServiceUrl { get; }
|
|
internal int UdpPort { get; }
|
|
internal string PublisherCredential { get; }
|
|
internal string SigningKeyText { get; }
|
|
|
|
internal static async Task<RunningService> StartAsync(
|
|
CancellationToken cancellationToken,
|
|
string listenAddress = "127.0.0.1",
|
|
string? processNamespace = null,
|
|
string? readinessAddress = null)
|
|
{
|
|
string root = RepositoryRoot();
|
|
string serverAssembly = Path.Combine(
|
|
root,
|
|
$"src/FinalFactory.Rendezvous.Server/bin/{BuildConfiguration}/net10.0/FinalFactory.Rendezvous.Server.dll");
|
|
string clientAssembly = Path.Combine(
|
|
root,
|
|
$"src/FinalFactory.Rendezvous.TestClient/bin/{BuildConfiguration}/net8.0/FinalFactory.Rendezvous.TestClient.dll");
|
|
Assert.True(File.Exists(serverAssembly), $"Missing server build output: {serverAssembly}");
|
|
Assert.True(File.Exists(clientAssembly), $"Missing TestClient build output: {clientAssembly}");
|
|
|
|
DateTimeOffset now = DateTimeOffset.UtcNow;
|
|
byte[] signingKey = RandomNumberGenerator.GetBytes(32);
|
|
string signingKeyText = Convert.ToBase64String(signingKey);
|
|
string publisherCredential = IssuePublisherCredential(signingKey, now);
|
|
CryptographicOperations.ZeroMemory(signingKey);
|
|
Dictionary<string, string> environment = ServerEnvironment(
|
|
signingKeyText,
|
|
now,
|
|
listenAddress,
|
|
readinessAddress);
|
|
ProcessCapture server = processNamespace is null
|
|
? Start(serverAssembly, [], environment)
|
|
: StartInNamespace(processNamespace, serverAssembly, [], environment);
|
|
try
|
|
{
|
|
string httpLine = await server.WaitForLineAsync(
|
|
$"Now listening on: http://{listenAddress}:",
|
|
cancellationToken);
|
|
Uri boundService = new(ParseServiceUrl(httpLine));
|
|
string serviceUrl = new UriBuilder(boundService)
|
|
{
|
|
Host = readinessAddress ?? IPAddress.Loopback.ToString(),
|
|
}.Uri.AbsoluteUri;
|
|
string udpLine = await server.WaitForLineAsync(
|
|
$"UDP mediator listening on {listenAddress}:",
|
|
cancellationToken);
|
|
int udpPort = ParseTrailingPort(udpLine);
|
|
await WaitForReadyAsync(serviceUrl, server, cancellationToken);
|
|
return new RunningService(
|
|
server,
|
|
clientAssembly,
|
|
serviceUrl,
|
|
udpPort,
|
|
publisherCredential,
|
|
signingKeyText);
|
|
}
|
|
catch
|
|
{
|
|
await server.DisposeAsync();
|
|
throw;
|
|
}
|
|
}
|
|
|
|
public ValueTask DisposeAsync() => Server.DisposeAsync();
|
|
}
|
|
|
|
private sealed class ProcessCapture : IAsyncDisposable
|
|
{
|
|
private readonly Process _process;
|
|
private readonly ConcurrentQueue<string> _standardOutput = new();
|
|
private readonly ConcurrentQueue<string> _standardError = new();
|
|
private readonly ConcurrentQueue<JsonElement> _jsonEvents = new();
|
|
|
|
internal ProcessCapture(ProcessStartInfo start)
|
|
{
|
|
_process = new Process { StartInfo = start };
|
|
_process.OutputDataReceived += (_, eventArgs) => Add(eventArgs.Data, _standardOutput);
|
|
_process.ErrorDataReceived += (_, eventArgs) => Add(eventArgs.Data, _standardError);
|
|
Assert.True(_process.Start(), $"Failed to start {start.FileName}.");
|
|
_process.BeginOutputReadLine();
|
|
_process.BeginErrorReadLine();
|
|
}
|
|
|
|
internal bool HasExited => _process.HasExited;
|
|
|
|
internal async Task<JsonElement> WaitForEventAsync(
|
|
string eventName,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
while (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
foreach (JsonElement item in JsonEvents())
|
|
{
|
|
if (item.TryGetProperty("event", out JsonElement eventProperty)
|
|
&& eventProperty.GetString() == eventName)
|
|
{
|
|
return item;
|
|
}
|
|
}
|
|
if (_process.HasExited)
|
|
{
|
|
throw new Xunit.Sdk.XunitException(
|
|
$"Process exited before event {eventName}. {DiagnosticText()}");
|
|
}
|
|
await Task.Delay(10, cancellationToken);
|
|
}
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
throw new UnreachableException();
|
|
}
|
|
|
|
internal async Task<string> WaitForLineAsync(
|
|
string marker,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
while (!cancellationToken.IsCancellationRequested)
|
|
{
|
|
string? line = AllLines().FirstOrDefault(item =>
|
|
item.Contains(marker, StringComparison.Ordinal));
|
|
if (line is not null)
|
|
{
|
|
return line;
|
|
}
|
|
if (_process.HasExited)
|
|
{
|
|
throw new Xunit.Sdk.XunitException(
|
|
$"Process exited before output marker {marker}. {DiagnosticText()}");
|
|
}
|
|
await Task.Delay(10, cancellationToken);
|
|
}
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
throw new UnreachableException();
|
|
}
|
|
|
|
internal bool HasEvent(string eventName, string status) => JsonEvents().Any(item =>
|
|
item.TryGetProperty("event", out JsonElement eventProperty)
|
|
&& eventProperty.GetString() == eventName
|
|
&& item.TryGetProperty("status", out JsonElement statusProperty)
|
|
&& statusProperty.GetString() == status);
|
|
|
|
internal List<JsonElement> JsonEvents() => [.. _jsonEvents];
|
|
|
|
internal IEnumerable<string> AllLines() => _standardOutput.Concat(_standardError);
|
|
|
|
internal async Task<int> WaitForExitAsync(CancellationToken cancellationToken)
|
|
{
|
|
await _process.WaitForExitAsync(cancellationToken);
|
|
return _process.ExitCode;
|
|
}
|
|
|
|
internal async Task InterruptAsync(CancellationToken cancellationToken)
|
|
{
|
|
ProcessStartInfo signalStart = new()
|
|
{
|
|
FileName = "/bin/kill",
|
|
UseShellExecute = false,
|
|
CreateNoWindow = true,
|
|
};
|
|
signalStart.ArgumentList.Add("-INT");
|
|
signalStart.ArgumentList.Add(_process.Id.ToString(System.Globalization.CultureInfo.InvariantCulture));
|
|
using Process signal = Process.Start(signalStart)
|
|
?? throw new Xunit.Sdk.XunitException("Failed to start the POSIX signal helper.");
|
|
await signal.WaitForExitAsync(cancellationToken);
|
|
Assert.Equal(0, signal.ExitCode);
|
|
}
|
|
|
|
internal async Task TerminateAsync(CancellationToken cancellationToken)
|
|
{
|
|
if (!_process.HasExited)
|
|
{
|
|
_process.Kill(entireProcessTree: true);
|
|
}
|
|
await _process.WaitForExitAsync(cancellationToken);
|
|
}
|
|
|
|
internal string DiagnosticText()
|
|
{
|
|
string events = string.Join(
|
|
',',
|
|
JsonEvents().Select(static item =>
|
|
$"{item.GetProperty("event").GetString()}:{item.GetProperty("status").GetString()}"));
|
|
return $"stdoutLines={_standardOutput.Count}; stderrLines={_standardError.Count}; events=[{events}]";
|
|
}
|
|
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
bool cleanupTimedOut = false;
|
|
bool reapTimedOut = false;
|
|
try
|
|
{
|
|
if (!_process.HasExited)
|
|
{
|
|
_process.Kill(entireProcessTree: true);
|
|
}
|
|
using CancellationTokenSource cleanup = new(TimeSpan.FromSeconds(5));
|
|
await _process.WaitForExitAsync(cleanup.Token);
|
|
}
|
|
catch (InvalidOperationException)
|
|
{
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
cleanupTimedOut = !_process.HasExited;
|
|
if (cleanupTimedOut)
|
|
{
|
|
try
|
|
{
|
|
_process.Kill(entireProcessTree: true);
|
|
}
|
|
catch (InvalidOperationException)
|
|
{
|
|
}
|
|
using CancellationTokenSource reap = new(TimeSpan.FromSeconds(2));
|
|
try
|
|
{
|
|
await _process.WaitForExitAsync(reap.Token);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
reapTimedOut = true;
|
|
}
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
_process.Dispose();
|
|
}
|
|
if (cleanupTimedOut)
|
|
{
|
|
string detail = reapTimedOut
|
|
? " and remained alive after the final kill"
|
|
: string.Empty;
|
|
throw new Xunit.Sdk.XunitException(
|
|
$"Child process did not exit within the cleanup deadline{detail}.");
|
|
}
|
|
}
|
|
|
|
private void Add(string? line, ConcurrentQueue<string> destination)
|
|
{
|
|
if (!string.IsNullOrEmpty(line))
|
|
{
|
|
destination.Enqueue(line);
|
|
try
|
|
{
|
|
using JsonDocument document = JsonDocument.Parse(line);
|
|
_jsonEvents.Enqueue(document.RootElement.Clone());
|
|
}
|
|
catch (JsonException)
|
|
{
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|