using System.Diagnostics; using System.Net; using System.Net.Sockets; using System.Security.Cryptography; namespace FinalFactory.Rendezvous.Tests.Deployment; public sealed class ProductionProcessTests { [Fact] public async Task SigtermDrainsThenReleasesHttpAndUdpSockets() { if (!OperatingSystem.IsLinux()) { return; } int httpPort = ReserveTcpPort(); int udpPort = ReserveUdpPort(); string secretPath = Path.Combine( Path.GetTempPath(), $"rendezvous-process-secret-{Guid.NewGuid():N}"); await File.WriteAllBytesAsync(secretPath, RandomNumberGenerator.GetBytes(32)); Process? process = null; try { ProcessStartInfo startInfo = CreateStartInfo(httpPort, udpPort, secretPath); process = Process.Start(startInfo) ?? throw new InvalidOperationException("The production server process did not start."); Task standardOutput = process.StandardOutput.ReadToEndAsync(); Task standardError = process.StandardError.ReadToEndAsync(); await WaitForReadyAsync(httpPort, process, TimeSpan.FromSeconds(10)); AssertUdpPortIsBound(udpPort); Stopwatch shutdown = Stopwatch.StartNew(); ProcessStartInfo signalInfo = new() { FileName = "/bin/kill", UseShellExecute = false, ArgumentList = { "-TERM", process.Id.ToString(System.Globalization.CultureInfo.InvariantCulture), }, }; using (Process signal = Process.Start(signalInfo) ?? throw new InvalidOperationException("Could not send SIGTERM.")) { await signal.WaitForExitAsync(); Assert.Equal(0, signal.ExitCode); } using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(6)); await process.WaitForExitAsync(timeout.Token); string output = await standardOutput; string error = await standardError; Assert.True( process.ExitCode == 0, $"Server exited with {process.ExitCode}. stdout: {output} stderr: {error}"); Assert.InRange(shutdown.Elapsed, TimeSpan.FromMilliseconds(700), TimeSpan.FromSeconds(5)); AssertTcpPortIsReleased(httpPort); AssertUdpPortIsReleased(udpPort); process.Dispose(); process = null; Stopwatch replacementReady = Stopwatch.StartNew(); ProcessStartInfo replacementInfo = CreateStartInfo(httpPort, udpPort, secretPath); process = Process.Start(replacementInfo) ?? throw new InvalidOperationException("The replacement production process did not start."); Task replacementOutput = process.StandardOutput.ReadToEndAsync(); Task replacementError = process.StandardError.ReadToEndAsync(); await WaitForReadyAsync(httpPort, process, TimeSpan.FromSeconds(15)); Assert.True( replacementReady.Elapsed < TimeSpan.FromSeconds(15), $"Replacement readiness took {replacementReady.Elapsed}."); AssertUdpPortIsBound(udpPort); await SendSigtermAsync(process); using CancellationTokenSource replacementTimeout = new(TimeSpan.FromSeconds(6)); await process.WaitForExitAsync(replacementTimeout.Token); string replacementFinalOutput = await replacementOutput; string replacementFinalError = await replacementError; Assert.True( process.ExitCode == 0, $"Replacement exited with {process.ExitCode}. " + $"stdout: {replacementFinalOutput} stderr: {replacementFinalError}"); AssertTcpPortIsReleased(httpPort); AssertUdpPortIsReleased(udpPort); } finally { if (process is not null) { if (!process.HasExited) { process.Kill(entireProcessTree: true); await process.WaitForExitAsync(); } process.Dispose(); } File.Delete(secretPath); } } [Fact] public async Task ProductionTransportSoakKeepsHandlesMemoryAndSocketsBounded() { if (!OperatingSystem.IsLinux()) { return; } int httpPort = ReserveTcpPort(); int udpPort = ReserveUdpPort(); string secretPath = Path.Combine( Path.GetTempPath(), $"rendezvous-transport-soak-secret-{Guid.NewGuid():N}"); await File.WriteAllBytesAsync(secretPath, RandomNumberGenerator.GetBytes(32)); Process? process = null; try { process = Process.Start(CreateStartInfo(httpPort, udpPort, secretPath)) ?? throw new InvalidOperationException("The production soak process did not start."); Task standardOutput = process.StandardOutput.ReadToEndAsync(); Task standardError = process.StandardError.ReadToEndAsync(); await WaitForReadyAsync(httpPort, process, TimeSpan.FromSeconds(10)); process.Refresh(); int baselineHandles = process.HandleCount; long baselineWorkingSet = process.WorkingSet64; int peakHandles = baselineHandles; byte[] invalidDatagram = RandomNumberGenerator.GetBytes(64); IPEndPoint udpEndpoint = new(IPAddress.Loopback, udpPort); using HttpClient client = new() { Timeout = TimeSpan.FromSeconds(1) }; using UdpClient udp = new(); Stopwatch soak = Stopwatch.StartNew(); int cycles = 0; int accepted = 0; int shed = 0; while (soak.Elapsed < TimeSpan.FromSeconds(10)) { using HttpResponseMessage response = await client.GetAsync( $"http://127.0.0.1:{httpPort}/health/live"); if (response.StatusCode == HttpStatusCode.OK) { accepted++; } else { Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); shed++; } await udp.SendAsync(invalidDatagram, udpEndpoint); cycles++; if (cycles % 100 == 0) { process.Refresh(); peakHandles = Math.Max(peakHandles, process.HandleCount); } } Assert.True(cycles >= 100, $"Transport soak completed only {cycles} cycles."); Assert.True(accepted > 0, "Transport soak never admitted a health request."); Assert.True(shed > 0, "Transport soak never exercised typed HTTP load shedding."); await Task.Delay(TimeSpan.FromSeconds(2)); using (HttpResponseMessage recovered = await client.GetAsync( $"http://127.0.0.1:{httpPort}/health/live")) { Assert.Equal(HttpStatusCode.OK, recovered.StatusCode); } process.Refresh(); Assert.InRange(peakHandles, 0, baselineHandles + 32); Assert.InRange(process.HandleCount, 0, baselineHandles + 16); Assert.InRange(process.WorkingSet64, 0, baselineWorkingSet + 67_108_864); AssertUdpPortIsBound(udpPort); await SendSigtermAsync(process); using CancellationTokenSource shutdownTimeout = new(TimeSpan.FromSeconds(6)); await process.WaitForExitAsync(shutdownTimeout.Token); string output = await standardOutput; string error = await standardError; Assert.True( process.ExitCode == 0, $"Transport soak process failed. stdout: {output} stderr: {error}"); AssertTcpPortIsReleased(httpPort); AssertUdpPortIsReleased(udpPort); } finally { await StopProcessTreeAsync(process); File.Delete(secretPath); } } [Fact] public async Task DocumentedSmokeScriptReachesHttpAndAuthenticatedUdpFlow() { if (!OperatingSystem.IsLinux()) { return; } string root = RepositoryRoot(); int httpPort = ReserveTcpPort(); int udpPort = ReserveUdpPort(); string secretPath = Path.Combine( Path.GetTempPath(), $"rendezvous-smoke-secret-{Guid.NewGuid():N}"); await File.WriteAllBytesAsync(secretPath, RandomNumberGenerator.GetBytes(32)); Process? server = null; Process? smoke = null; try { DateTimeOffset now = DateTimeOffset.UtcNow; string assembly = typeof(Program).Assembly.Location; ProcessStartInfo serverInfo = new() { FileName = "dotnet", WorkingDirectory = root, RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, ArgumentList = { assembly, "--contentRoot", Path.Combine(root, "deploy", "compose"), "--Rendezvous:Provisioning:SigningKeys:0:SecretReference", $"file:{secretPath}", "--Rendezvous:Provisioning:SigningKeys:0:NotBefore", now.AddHours(-1).ToString("O"), "--Rendezvous:Provisioning:SigningKeys:0:SignUntil", now.AddHours(1).ToString("O"), "--Rendezvous:Provisioning:SigningKeys:0:VerifyUntil", now.AddHours(2).ToString("O"), "--Rendezvous:Udp:Port", udpPort.ToString(System.Globalization.CultureInfo.InvariantCulture), "--Rendezvous:Deployment:PublicUdpPort", udpPort.ToString(System.Globalization.CultureInfo.InvariantCulture), }, }; serverInfo.Environment["ASPNETCORE_ENVIRONMENT"] = "Production"; serverInfo.Environment["ASPNETCORE_URLS"] = $"http://127.0.0.1:{httpPort}"; server = Process.Start(serverInfo) ?? throw new InvalidOperationException("The smoke server process did not start."); Task serverOutput = server.StandardOutput.ReadToEndAsync(); Task serverError = server.StandardError.ReadToEndAsync(); await WaitForReadyAsync(httpPort, server, TimeSpan.FromSeconds(10)); ProcessStartInfo smokeInfo = new() { FileName = Path.Combine(root, "scripts", "smoke-deployment.sh"), WorkingDirectory = root, RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, }; smokeInfo.Environment.Remove("RENDEZVOUS_PUBLISHER_CREDENTIAL"); smokeInfo.Environment["RENDEZVOUS_SMOKE_HTTP_URL"] = $"http://127.0.0.1:{httpPort}/"; smokeInfo.Environment["RENDEZVOUS_SMOKE_UDP_ENDPOINT"] = $"127.0.0.1:{udpPort}"; smokeInfo.Environment["RENDEZVOUS_SMOKE_LOCAL_KEY"] = secretPath; smokeInfo.Environment["RENDEZVOUS_SMOKE_TIMEOUT_SECONDS"] = "15"; smokeInfo.Environment["RENDEZVOUS_SMOKE_CONFIGURATION"] = BuildConfiguration(); smoke = Process.Start(smokeInfo) ?? throw new InvalidOperationException("The deployment smoke process did not start."); Task smokeOutput = smoke.StandardOutput.ReadToEndAsync(); Task smokeError = smoke.StandardError.ReadToEndAsync(); using (CancellationTokenSource timeout = new(TimeSpan.FromSeconds(25))) { await smoke.WaitForExitAsync(timeout.Token); } string output = await smokeOutput; string error = await smokeError; Assert.True( smoke.ExitCode == 0, $"Smoke exited with {smoke.ExitCode}. stdout: {output} stderr: {error}"); Assert.Contains("deployment smoke passed", output, StringComparison.OrdinalIgnoreCase); Assert.DoesNotContain("rv1.", output, StringComparison.Ordinal); Assert.DoesNotContain("rv1.", error, StringComparison.Ordinal); await SendSigtermAsync(server); using CancellationTokenSource shutdownTimeout = new(TimeSpan.FromSeconds(6)); await server.WaitForExitAsync(shutdownTimeout.Token); string finalServerOutput = await serverOutput; string finalServerError = await serverError; Assert.True( server.ExitCode == 0, $"Smoke server failed. stdout: {finalServerOutput} stderr: {finalServerError}"); } finally { await StopProcessTreeAsync(smoke); if (server is not null) { await StopProcessTreeAsync(server); } File.Delete(secretPath); } } [Theory] [InlineData("missing-deployment", "PublicHttpBaseUrl")] [InlineData("wildcard-host", "AllowedHosts")] [InlineData("reserved-endpoint", "public DNS name or address")] [InlineData("missing-key", "process-test-key")] public async Task UnsafeProductionConfigurationFailsBeforeBinding( string scenario, string expectedDiagnostic) { if (!OperatingSystem.IsLinux()) { return; } int httpPort = ReserveTcpPort(); int udpPort = ReserveUdpPort(); string secretPath = Path.Combine( Path.GetTempPath(), $"rendezvous-rejected-secret-{Guid.NewGuid():N}"); await File.WriteAllBytesAsync(secretPath, RandomNumberGenerator.GetBytes(32)); Process? process = null; try { ProcessStartInfo startInfo = scenario == "missing-deployment" ? CreateBareProductionStartInfo(httpPort) : CreateStartInfo(httpPort, udpPort, secretPath); if (scenario == "wildcard-host") { startInfo.Environment["AllowedHosts"] = "*"; } else if (scenario == "reserved-endpoint") { startInfo.Environment["Rendezvous__Deployment__AllowPrivatePublicEndpoints"] = "false"; startInfo.Environment["Rendezvous__Deployment__PublicHttpBaseUrl"] = "https://192.0.2.1/"; startInfo.Environment["Rendezvous__Deployment__PublicUdpHost"] = "203.0.113.1"; startInfo.Environment["AllowedHosts"] = "192.0.2.1"; } else if (scenario == "missing-key") { File.Delete(secretPath); } process = Process.Start(startInfo) ?? throw new InvalidOperationException("The rejected production process did not start."); Task standardOutput = process.StandardOutput.ReadToEndAsync(); Task standardError = process.StandardError.ReadToEndAsync(); using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(6)); await process.WaitForExitAsync(timeout.Token); string diagnostic = $"{await standardOutput}\n{await standardError}"; Assert.NotEqual(0, process.ExitCode); Assert.Contains(expectedDiagnostic, diagnostic, StringComparison.OrdinalIgnoreCase); AssertTcpPortIsReleased(httpPort); AssertUdpPortIsReleased(udpPort); } finally { await StopProcessTreeAsync(process); File.Delete(secretPath); } } private static ProcessStartInfo CreateStartInfo(int httpPort, int udpPort, string secretPath) { string assembly = typeof(Program).Assembly.Location; ProcessStartInfo info = new() { FileName = "dotnet", WorkingDirectory = Path.GetDirectoryName(assembly)!, RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, }; info.ArgumentList.Add(assembly); Dictionary settings = new(StringComparer.Ordinal) { ["ASPNETCORE_ENVIRONMENT"] = "Production", ["ASPNETCORE_URLS"] = $"http://127.0.0.1:{httpPort}", ["AllowedHosts"] = "127.0.0.1", ["Rendezvous__Deployment__PublicHttpBaseUrl"] = "https://127.0.0.1/", ["Rendezvous__Deployment__PublicUdpHost"] = "127.0.0.1", ["Rendezvous__Deployment__PublicUdpPort"] = udpPort.ToString(System.Globalization.CultureInfo.InvariantCulture), ["Rendezvous__Deployment__DrainDeadlineSeconds"] = "3", ["Rendezvous__Deployment__MinimumDrainSeconds"] = "1", ["Rendezvous__Deployment__SingleActiveInstance"] = "true", ["Rendezvous__Deployment__AllowPrivatePublicEndpoints"] = "true", ["Rendezvous__Udp__ListenAddress"] = "127.0.0.1", ["Rendezvous__Udp__Port"] = udpPort.ToString(System.Globalization.CultureInfo.InvariantCulture), ["Rendezvous__AbuseProtection__TrustedProxyAddresses__0"] = "127.0.0.1", ["Rendezvous__Provisioning__Issuer"] = "rendezvous-process-test", ["Rendezvous__Provisioning__Audience"] = "rendezvous-service", ["Rendezvous__Provisioning__ClockSkewSeconds"] = "30", ["Rendezvous__Provisioning__SigningKeys__0__KeyId"] = "process-test-key", ["Rendezvous__Provisioning__SigningKeys__0__SecretReference"] = $"file:{secretPath}", ["Rendezvous__Provisioning__SigningKeys__0__CredentialKinds__0"] = "DedicatedPublisher", ["Rendezvous__Provisioning__SigningKeys__0__GameId"] = "space-game", ["Rendezvous__Provisioning__SigningKeys__0__EnvironmentId"] = "process-test", ["Rendezvous__Provisioning__SigningKeys__0__NotBefore"] = DateTimeOffset.UtcNow.AddHours(-1).ToString("O"), ["Rendezvous__Provisioning__SigningKeys__0__SignUntil"] = DateTimeOffset.UtcNow.AddDays(1).ToString("O"), ["Rendezvous__Provisioning__SigningKeys__0__VerifyUntil"] = DateTimeOffset.UtcNow.AddDays(2).ToString("O"), ["Rendezvous__Provisioning__Games__0__GameId"] = "space-game", ["Rendezvous__Provisioning__Games__0__EnvironmentId"] = "process-test", ["Rendezvous__Provisioning__Games__0__Enabled"] = "true", ["Rendezvous__Provisioning__Games__0__ProtocolVersions__0"] = "1", ["Rendezvous__Provisioning__Games__0__Regions__0"] = "local", ["Rendezvous__Provisioning__Games__0__VisibilityModes__0"] = "Public", ["Rendezvous__Provisioning__Games__0__PublisherTrustModes__0"] = "ManagedDedicated", ["Rendezvous__Provisioning__Games__0__MetadataMaxBytes"] = "512", ["Rendezvous__Provisioning__Games__0__MetadataMaxKeys"] = "0", ["Rendezvous__Provisioning__Games__0__MaxListingsPerPrincipal"] = "10", ["Rendezvous__Provisioning__Games__0__MaxAnonymousListingsPerAddress"] = "0", ["Rendezvous__Provisioning__Games__0__MaxActiveJoinAttempts"] = "100", ["Rendezvous__Provisioning__Games__0__FallbackPolicy"] = "Disabled", }; foreach ((string key, string value) in settings) { info.Environment[key] = value; } return info; } private static ProcessStartInfo CreateBareProductionStartInfo(int httpPort) { string assembly = typeof(Program).Assembly.Location; ProcessStartInfo info = new() { FileName = "dotnet", WorkingDirectory = Path.GetDirectoryName(assembly)!, RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, }; info.ArgumentList.Add(assembly); foreach (string key in info.Environment.Keys .Where(static key => key.StartsWith("Rendezvous__", StringComparison.OrdinalIgnoreCase)) .ToArray()) { info.Environment.Remove(key); } info.Environment["ASPNETCORE_ENVIRONMENT"] = "Production"; info.Environment["ASPNETCORE_URLS"] = $"http://127.0.0.1:{httpPort}"; info.Environment["AllowedHosts"] = "*"; return info; } private static async Task StopProcessTreeAsync(Process? process) { if (process is null) { return; } try { if (!process.HasExited) { process.Kill(entireProcessTree: true); using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(3)); await process.WaitForExitAsync(timeout.Token); } } finally { process.Dispose(); } } private static async Task SendSigtermAsync(Process process) { ProcessStartInfo signalInfo = new() { FileName = "/bin/kill", UseShellExecute = false, ArgumentList = { "-TERM", process.Id.ToString(System.Globalization.CultureInfo.InvariantCulture), }, }; using Process signal = Process.Start(signalInfo) ?? throw new InvalidOperationException("Could not send SIGTERM."); await signal.WaitForExitAsync(); Assert.Equal(0, signal.ExitCode); } private static string BuildConfiguration() { string path = typeof(ProductionProcessTests).Assembly.Location; return path.Contains( $"{Path.DirectorySeparatorChar}Release{Path.DirectorySeparatorChar}", StringComparison.Ordinal) ? "Release" : "Debug"; } private static string RepositoryRoot() { DirectoryInfo? directory = new(AppContext.BaseDirectory); while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "Rendezvous.slnx"))) { directory = directory.Parent; } return directory?.FullName ?? throw new InvalidOperationException("Could not locate the repository root."); } private static async Task WaitForReadyAsync(int port, Process process, TimeSpan timeout) { using HttpClient client = new() { Timeout = TimeSpan.FromMilliseconds(500) }; Stopwatch elapsed = Stopwatch.StartNew(); while (elapsed.Elapsed < timeout) { if (process.HasExited) { throw new InvalidOperationException("The production server exited before readiness."); } try { using HttpResponseMessage response = await client.GetAsync( $"http://127.0.0.1:{port}/health/ready"); if (response.StatusCode == HttpStatusCode.OK) { return; } } catch (HttpRequestException) { } catch (TaskCanceledException) { } await Task.Delay(50); } throw new TimeoutException("The production server did not become ready."); } private static int ReserveTcpPort() { TcpListener listener = new(IPAddress.Loopback, 0); listener.Start(); int port = ((IPEndPoint)listener.LocalEndpoint).Port; listener.Stop(); return port; } private static int ReserveUdpPort() { using UdpClient client = new(new IPEndPoint(IPAddress.Loopback, 0)); return ((IPEndPoint)client.Client.LocalEndPoint!).Port; } private static void AssertUdpPortIsBound(int port) { using Socket socket = new(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); Assert.Throws(() => socket.Bind(new IPEndPoint(IPAddress.Loopback, port))); } private static void AssertTcpPortIsReleased(int port) { TcpListener listener = new(IPAddress.Loopback, port); listener.Start(); listener.Stop(); } private static void AssertUdpPortIsReleased(int port) { using Socket socket = new(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); socket.Bind(new IPEndPoint(IPAddress.Loopback, port)); } }