feat(deployment): add secure Linux runtime (#17)

tests/FinalFactory.Rendezvous.Tests/Deployment/DeploymentOptionsTests.cs

@@ -0,0 +1,137 @@
+using FinalFactory.Rendezvous.Server.Abuse;
+using FinalFactory.Rendezvous.Server.Deployment;
+
+namespace FinalFactory.Rendezvous.Tests.Deployment;
+
+public sealed class DeploymentOptionsTests
+{
+    [Fact]
+    public void ProductionConfigurationAcceptsExplicitPublicEndpointsAndTrustBoundary()
+    {
+        DeploymentOptions options = ValidOptions();
+        AbuseProtectionOptions abuse = new()
+        {
+            TrustedProxyAddresses = ["192.0.2.10"],
+        };
+
+        IReadOnlyList<string> errors = options.ValidateProduction(
+            abuse,
+            "rendezvous.finalfactory.at");
+
+        Assert.Empty(errors);
+    }
+
+    [Fact]
+    public void ProductionConfigurationRejectsUnsafeAndAmbiguousDefaults()
+    {
+        DeploymentOptions options = new()
+        {
+            SingleActiveInstance = false,
+        };
+
+        IReadOnlyList<string> errors = options.ValidateProduction(
+            new AbuseProtectionOptions(),
+            "*");
+
+        Assert.Contains(errors, error => error.Contains("SingleActiveInstance", StringComparison.Ordinal));
+        Assert.Contains(errors, error => error.Contains("PublicHttpBaseUrl", StringComparison.Ordinal));
+        Assert.Contains(errors, error => error.Contains("PublicUdpHost", StringComparison.Ordinal));
+        Assert.Contains(errors, error => error.Contains("TrustedProxyAddresses", StringComparison.Ordinal));
+        Assert.Contains(errors, error => error.Contains("AllowedHosts", StringComparison.Ordinal));
+    }
+
+    [Theory]
+    [InlineData("http://rendezvous.example.com/")]
+    [InlineData("https://user@example.com/")]
+    [InlineData("https://rendezvous.example.com/path")]
+    [InlineData("https://localhost/")]
+    [InlineData("https://10.0.0.1/")]
+    [InlineData("https://192.0.2.10/")]
+    [InlineData("https://[::ffff:10.0.0.1]/")]
+    [InlineData("https://[ff02::1]/")]
+    [InlineData("https://rendezvous.invalid/")]
+    public void ProductionConfigurationRejectsUnsafeHttpEndpoint(string endpoint)
+    {
+        DeploymentOptions options = ValidOptions() with { PublicHttpBaseUrl = endpoint };
+
+        IReadOnlyList<string> errors = options.ValidateProduction(
+            new AbuseProtectionOptions { TrustedProxyAddresses = ["192.0.2.10"] },
+            "rendezvous.finalfactory.at");
+
+        Assert.Contains(errors, error => error.Contains("PublicHttpBaseUrl", StringComparison.Ordinal));
+    }
+
+    [Theory]
+    [InlineData("0.1.2.3")]
+    [InlineData("100.64.0.1")]
+    [InlineData("192.0.2.1")]
+    [InlineData("198.18.0.1")]
+    [InlineData("198.51.100.1")]
+    [InlineData("203.0.113.1")]
+    [InlineData("224.0.0.1")]
+    [InlineData("255.255.255.255")]
+    [InlineData("::ffff:192.168.1.1")]
+    [InlineData("2001:db8::1")]
+    [InlineData("ff02::1")]
+    [InlineData("rendezvous.example.com")]
+    [InlineData("rendezvous.home.arpa")]
+    [InlineData("rendezvous.alt")]
+    [InlineData("service.test")]
+    public void ProductionConfigurationRejectsNonPublicUdpEndpoint(string endpoint)
+    {
+        DeploymentOptions options = ValidOptions() with { PublicUdpHost = endpoint };
+
+        IReadOnlyList<string> errors = options.ValidateProduction(
+            new AbuseProtectionOptions { TrustedProxyAddresses = ["192.0.2.10"] },
+            "rendezvous.finalfactory.at");
+
+        Assert.Contains(errors, error => error.Contains("PublicUdpHost", StringComparison.Ordinal));
+    }
+
+    [Fact]
+    public void IsolatedSmokeTestRequiresExplicitPrivateEndpointOverride()
+    {
+        DeploymentOptions options = ValidOptions() with
+        {
+            PublicHttpBaseUrl = "https://localhost/",
+            PublicUdpHost = "127.0.0.1",
+            AllowPrivatePublicEndpoints = true,
+        };
+
+        IReadOnlyList<string> errors = options.ValidateProduction(
+            new AbuseProtectionOptions { TrustedProxyAddresses = ["127.0.0.1"] },
+            "localhost");
+
+        Assert.Empty(errors);
+    }
+
+    [Fact]
+    public void ProductionConfigurationRejectsInvalidPortsDeadlinesAndMismatchedHostFilter()
+    {
+        DeploymentOptions options = ValidOptions() with
+        {
+            PublicUdpPort = 0,
+            DrainDeadlineSeconds = 31,
+            MinimumDrainSeconds = 6,
+        };
+
+        IReadOnlyList<string> errors = options.ValidateProduction(
+            new AbuseProtectionOptions { TrustedProxyAddresses = ["192.0.2.10"] },
+            "different.finalfactory.at");
+
+        Assert.Contains(errors, error => error.Contains("PublicUdpPort", StringComparison.Ordinal));
+        Assert.Contains(errors, error => error.Contains("DrainDeadlineSeconds", StringComparison.Ordinal));
+        Assert.Contains(errors, error => error.Contains("MinimumDrainSeconds", StringComparison.Ordinal));
+        Assert.Contains(errors, error => error.Contains("exact host", StringComparison.Ordinal));
+    }
+
+    private static DeploymentOptions ValidOptions() => new()
+    {
+        PublicHttpBaseUrl = "https://rendezvous.finalfactory.at/",
+        PublicUdpHost = "rendezvous-udp.finalfactory.at",
+        PublicUdpPort = 9050,
+        DrainDeadlineSeconds = 10,
+        MinimumDrainSeconds = 1,
+        SingleActiveInstance = true,
+    };
+}

tests/FinalFactory.Rendezvous.Tests/Deployment/GracefulDrainServiceTests.cs

@@ -0,0 +1,149 @@
+using System.Diagnostics;
+using FinalFactory.Rendezvous.Server.Deployment;
+using FinalFactory.Rendezvous.Server.State;
+using FinalFactory.Rendezvous.Tests.State;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+
+namespace FinalFactory.Rendezvous.Tests.Deployment;
+
+public sealed class GracefulDrainServiceTests
+{
+    [Fact]
+    public async Task ShutdownRejectsNewWorkAndClearsStateAfterBoundedAttemptDeadline()
+    {
+        EphemeralStoreOptions stateOptions = new()
+        {
+            GracefulDrainLifetime = TimeSpan.FromSeconds(1),
+        };
+        EphemeralStateFixture fixture = new(stateOptions);
+        StoredListing listing = fixture.CreateVisibleListing(out _);
+        StoreResult<StoredJoinAttempt> attempt = fixture.Store.CreateJoinAttempt(
+            fixture.AttemptCommand(listing));
+        Assert.True(attempt.Succeeded);
+        using FakeApplicationLifetime lifetime = new();
+        using GracefulDrainService service = CreateService(fixture.Store, lifetime);
+        await service.StartAsync(CancellationToken.None);
+
+        Task stopping = Task.Run(lifetime.StopApplication);
+        await WaitUntilAsync(() => fixture.Store.IsDraining, TimeSpan.FromSeconds(1));
+        StoreResult<StoredListing> rejected = fixture.Store.CreateListing(fixture.ListingCommand());
+        long sweepsAfterAdmissionCheck = fixture.Store.MaintenanceSweepCount;
+        Stopwatch elapsed = Stopwatch.StartNew();
+        await stopping;
+        await service.StopAsync(CancellationToken.None);
+
+        Assert.Equal(StoreResultCode.Draining, rejected.Code);
+        Assert.Equal(sweepsAfterAdmissionCheck, fixture.Store.MaintenanceSweepCount);
+        Assert.InRange(elapsed.Elapsed, TimeSpan.FromMilliseconds(850), TimeSpan.FromSeconds(2));
+        Assert.False(fixture.Store.IsAvailable);
+        Assert.Equal(0, fixture.Store.GetSnapshot().ActiveJoinAttempts);
+        Assert.Equal(0, fixture.Store.GetSnapshot().ActiveListings);
+    }
+
+    [Fact]
+    public async Task ShutdownWithoutAttemptsStopsAfterTheConfiguredMinimumOnly()
+    {
+        EphemeralStateFixture fixture = new(new EphemeralStoreOptions
+        {
+            GracefulDrainLifetime = TimeSpan.FromSeconds(2),
+        });
+        fixture.CreateVisibleListing(out _);
+        using FakeApplicationLifetime lifetime = new();
+        DeploymentOptions options = new()
+        {
+            DrainDeadlineSeconds = 2,
+            MinimumDrainSeconds = 0,
+        };
+        using GracefulDrainService service = new(
+            fixture.Store,
+            lifetime,
+            Options.Create(options),
+            NullLogger<GracefulDrainService>.Instance);
+        await service.StartAsync(CancellationToken.None);
+
+        Stopwatch elapsed = Stopwatch.StartNew();
+        await service.StopAsync(CancellationToken.None);
+
+        Assert.True(elapsed.Elapsed < TimeSpan.FromMilliseconds(500));
+        Assert.False(fixture.Store.IsAvailable);
+        Assert.Equal(0, fixture.Store.GetSnapshot().ActiveListings);
+    }
+
+    [Fact]
+    public async Task ShutdownStopsWhenTheLastAttemptExpiresWithoutAFullStateSweep()
+    {
+        EphemeralStateFixture fixture = new(new EphemeralStoreOptions
+        {
+            JoinAttemptLifetime = TimeSpan.FromMilliseconds(100),
+            ConnectionTicketLifetime = TimeSpan.FromMilliseconds(50),
+            GracefulDrainLifetime = TimeSpan.FromSeconds(2),
+        });
+        StoredListing listing = fixture.CreateVisibleListing(out _);
+        Assert.True(fixture.Store.CreateJoinAttempt(fixture.AttemptCommand(listing)).Succeeded);
+        using FakeApplicationLifetime lifetime = new();
+        using GracefulDrainService service = new(
+            fixture.Store,
+            lifetime,
+            Options.Create(new DeploymentOptions
+            {
+                DrainDeadlineSeconds = 2,
+                MinimumDrainSeconds = 0,
+            }),
+            NullLogger<GracefulDrainService>.Instance);
+        await service.StartAsync(CancellationToken.None);
+
+        Task stopping = Task.Run(lifetime.StopApplication);
+        await WaitUntilAsync(() => fixture.Store.IsDraining, TimeSpan.FromSeconds(1));
+        long sweepsBeforeExpiry = fixture.Store.MaintenanceSweepCount;
+        fixture.Clock.Advance(TimeSpan.FromMilliseconds(150));
+        using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(1));
+        await stopping.WaitAsync(timeout.Token);
+
+        Assert.Equal(sweepsBeforeExpiry, fixture.Store.MaintenanceSweepCount);
+        Assert.False(fixture.Store.IsAvailable);
+    }
+
+    private static GracefulDrainService CreateService(
+        InMemoryEphemeralRendezvousStore store,
+        IHostApplicationLifetime lifetime) => new(
+            store,
+            lifetime,
+            Options.Create(new DeploymentOptions
+            {
+                DrainDeadlineSeconds = 1,
+                MinimumDrainSeconds = 0,
+            }),
+            NullLogger<GracefulDrainService>.Instance);
+
+    private static async Task WaitUntilAsync(Func<bool> predicate, TimeSpan timeout)
+    {
+        Stopwatch elapsed = Stopwatch.StartNew();
+        while (!predicate())
+        {
+            Assert.True(elapsed.Elapsed < timeout, "The service did not enter drain in time.");
+            await Task.Delay(10);
+        }
+    }
+
+    private sealed class FakeApplicationLifetime : IHostApplicationLifetime, IDisposable
+    {
+        private readonly CancellationTokenSource _started = new();
+        private readonly CancellationTokenSource _stopping = new();
+        private readonly CancellationTokenSource _stopped = new();
+
+        public CancellationToken ApplicationStarted => _started.Token;
+        public CancellationToken ApplicationStopping => _stopping.Token;
+        public CancellationToken ApplicationStopped => _stopped.Token;
+
+        public void StopApplication() => _stopping.Cancel();
+
+        public void Dispose()
+        {
+            _started.Dispose();
+            _stopping.Dispose();
+            _stopped.Dispose();
+        }
+    }
+}

tests/FinalFactory.Rendezvous.Tests/Deployment/ProductionProcessTests.cs

@@ -0,0 +1,458 @@
+using System.Diagnostics;
+using System.Net;
+using System.Net.Sockets;
+using System.Security.Cryptography;
+
+namespace FinalFactory.Rendezvous.Tests.Deployment;
+
+public sealed class ProductionProcessTests
+{
+    [Fact]
+    public async Task SigtermDrainsThenReleasesHttpAndUdpSockets()
+    {
+        if (!OperatingSystem.IsLinux())
+        {
+            return;
+        }
+
+        int httpPort = ReserveTcpPort();
+        int udpPort = ReserveUdpPort();
+        string secretPath = Path.Combine(
+            Path.GetTempPath(),
+            $"rendezvous-process-secret-{Guid.NewGuid():N}");
+        await File.WriteAllBytesAsync(secretPath, RandomNumberGenerator.GetBytes(32));
+        Process? process = null;
+        try
+        {
+            ProcessStartInfo startInfo = CreateStartInfo(httpPort, udpPort, secretPath);
+            process = Process.Start(startInfo)
+                ?? throw new InvalidOperationException("The production server process did not start.");
+            Task<string> standardOutput = process.StandardOutput.ReadToEndAsync();
+            Task<string> standardError = process.StandardError.ReadToEndAsync();
+
+            await WaitForReadyAsync(httpPort, process, TimeSpan.FromSeconds(10));
+            AssertUdpPortIsBound(udpPort);
+
+            Stopwatch shutdown = Stopwatch.StartNew();
+            ProcessStartInfo signalInfo = new()
+            {
+                FileName = "/bin/kill",
+                UseShellExecute = false,
+                ArgumentList =
+                {
+                    "-TERM",
+                    process.Id.ToString(System.Globalization.CultureInfo.InvariantCulture),
+                },
+            };
+            using (Process signal = Process.Start(signalInfo)
+                   ?? throw new InvalidOperationException("Could not send SIGTERM."))
+            {
+                await signal.WaitForExitAsync();
+                Assert.Equal(0, signal.ExitCode);
+            }
+
+            using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(6));
+            await process.WaitForExitAsync(timeout.Token);
+            string output = await standardOutput;
+            string error = await standardError;
+            Assert.True(
+                process.ExitCode == 0,
+                $"Server exited with {process.ExitCode}. stdout: {output} stderr: {error}");
+            Assert.InRange(shutdown.Elapsed, TimeSpan.FromMilliseconds(700), TimeSpan.FromSeconds(5));
+            AssertTcpPortIsReleased(httpPort);
+            AssertUdpPortIsReleased(udpPort);
+        }
+        finally
+        {
+            if (process is not null)
+            {
+                if (!process.HasExited)
+                {
+                    process.Kill(entireProcessTree: true);
+                    await process.WaitForExitAsync();
+                }
+
+                process.Dispose();
+            }
+
+            File.Delete(secretPath);
+        }
+    }
+
+    [Fact]
+    public async Task DocumentedSmokeScriptReachesHttpAndAuthenticatedUdpFlow()
+    {
+        if (!OperatingSystem.IsLinux())
+        {
+            return;
+        }
+
+        string root = RepositoryRoot();
+        int httpPort = ReserveTcpPort();
+        int udpPort = ReserveUdpPort();
+        string secretPath = Path.Combine(
+            Path.GetTempPath(),
+            $"rendezvous-smoke-secret-{Guid.NewGuid():N}");
+        await File.WriteAllBytesAsync(secretPath, RandomNumberGenerator.GetBytes(32));
+        Process? server = null;
+        Process? smoke = null;
+        try
+        {
+            DateTimeOffset now = DateTimeOffset.UtcNow;
+            string assembly = typeof(Program).Assembly.Location;
+            ProcessStartInfo serverInfo = new()
+            {
+                FileName = "dotnet",
+                WorkingDirectory = root,
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+                ArgumentList =
+                {
+                    assembly,
+                    "--contentRoot", Path.Combine(root, "deploy", "compose"),
+                    "--Rendezvous:Provisioning:SigningKeys:0:SecretReference", $"file:{secretPath}",
+                    "--Rendezvous:Provisioning:SigningKeys:0:NotBefore", now.AddHours(-1).ToString("O"),
+                    "--Rendezvous:Provisioning:SigningKeys:0:SignUntil", now.AddHours(1).ToString("O"),
+                    "--Rendezvous:Provisioning:SigningKeys:0:VerifyUntil", now.AddHours(2).ToString("O"),
+                    "--Rendezvous:Udp:Port", udpPort.ToString(System.Globalization.CultureInfo.InvariantCulture),
+                    "--Rendezvous:Deployment:PublicUdpPort", udpPort.ToString(System.Globalization.CultureInfo.InvariantCulture),
+                },
+            };
+            serverInfo.Environment["ASPNETCORE_ENVIRONMENT"] = "Production";
+            serverInfo.Environment["ASPNETCORE_URLS"] = $"http://127.0.0.1:{httpPort}";
+            server = Process.Start(serverInfo)
+                ?? throw new InvalidOperationException("The smoke server process did not start.");
+            Task<string> serverOutput = server.StandardOutput.ReadToEndAsync();
+            Task<string> serverError = server.StandardError.ReadToEndAsync();
+            await WaitForReadyAsync(httpPort, server, TimeSpan.FromSeconds(10));
+
+            ProcessStartInfo smokeInfo = new()
+            {
+                FileName = Path.Combine(root, "scripts", "smoke-deployment.sh"),
+                WorkingDirectory = root,
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+            };
+            smokeInfo.Environment.Remove("RENDEZVOUS_PUBLISHER_CREDENTIAL");
+            smokeInfo.Environment["RENDEZVOUS_SMOKE_HTTP_URL"] = $"http://127.0.0.1:{httpPort}/";
+            smokeInfo.Environment["RENDEZVOUS_SMOKE_UDP_ENDPOINT"] = $"127.0.0.1:{udpPort}";
+            smokeInfo.Environment["RENDEZVOUS_SMOKE_LOCAL_KEY"] = secretPath;
+            smokeInfo.Environment["RENDEZVOUS_SMOKE_TIMEOUT_SECONDS"] = "15";
+            smokeInfo.Environment["RENDEZVOUS_SMOKE_CONFIGURATION"] = BuildConfiguration();
+            smoke = Process.Start(smokeInfo)
+                ?? throw new InvalidOperationException("The deployment smoke process did not start.");
+            Task<string> smokeOutput = smoke.StandardOutput.ReadToEndAsync();
+            Task<string> smokeError = smoke.StandardError.ReadToEndAsync();
+            using (CancellationTokenSource timeout = new(TimeSpan.FromSeconds(25)))
+            {
+                await smoke.WaitForExitAsync(timeout.Token);
+            }
+
+            string output = await smokeOutput;
+            string error = await smokeError;
+            Assert.True(
+                smoke.ExitCode == 0,
+                $"Smoke exited with {smoke.ExitCode}. stdout: {output} stderr: {error}");
+            Assert.Contains("deployment smoke passed", output, StringComparison.OrdinalIgnoreCase);
+            Assert.DoesNotContain("rv1.", output, StringComparison.Ordinal);
+            Assert.DoesNotContain("rv1.", error, StringComparison.Ordinal);
+
+            await SendSigtermAsync(server);
+            using CancellationTokenSource shutdownTimeout = new(TimeSpan.FromSeconds(6));
+            await server.WaitForExitAsync(shutdownTimeout.Token);
+            string finalServerOutput = await serverOutput;
+            string finalServerError = await serverError;
+            Assert.True(
+                server.ExitCode == 0,
+                $"Smoke server failed. stdout: {finalServerOutput} stderr: {finalServerError}");
+        }
+        finally
+        {
+            await StopProcessTreeAsync(smoke);
+            if (server is not null)
+            {
+                await StopProcessTreeAsync(server);
+            }
+
+            File.Delete(secretPath);
+        }
+    }
+
+    [Theory]
+    [InlineData("missing-deployment", "PublicHttpBaseUrl")]
+    [InlineData("wildcard-host", "AllowedHosts")]
+    [InlineData("reserved-endpoint", "public DNS name or address")]
+    [InlineData("missing-key", "process-test-key")]
+    public async Task UnsafeProductionConfigurationFailsBeforeBinding(
+        string scenario,
+        string expectedDiagnostic)
+    {
+        if (!OperatingSystem.IsLinux())
+        {
+            return;
+        }
+
+        int httpPort = ReserveTcpPort();
+        int udpPort = ReserveUdpPort();
+        string secretPath = Path.Combine(
+            Path.GetTempPath(),
+            $"rendezvous-rejected-secret-{Guid.NewGuid():N}");
+        await File.WriteAllBytesAsync(secretPath, RandomNumberGenerator.GetBytes(32));
+        Process? process = null;
+        try
+        {
+            ProcessStartInfo startInfo = scenario == "missing-deployment"
+                ? CreateBareProductionStartInfo(httpPort)
+                : CreateStartInfo(httpPort, udpPort, secretPath);
+            if (scenario == "wildcard-host")
+            {
+                startInfo.Environment["AllowedHosts"] = "*";
+            }
+            else if (scenario == "reserved-endpoint")
+            {
+                startInfo.Environment["Rendezvous__Deployment__AllowPrivatePublicEndpoints"] = "false";
+                startInfo.Environment["Rendezvous__Deployment__PublicHttpBaseUrl"] = "https://192.0.2.1/";
+                startInfo.Environment["Rendezvous__Deployment__PublicUdpHost"] = "203.0.113.1";
+                startInfo.Environment["AllowedHosts"] = "192.0.2.1";
+            }
+            else if (scenario == "missing-key")
+            {
+                File.Delete(secretPath);
+            }
+
+            process = Process.Start(startInfo)
+                ?? throw new InvalidOperationException("The rejected production process did not start.");
+            Task<string> standardOutput = process.StandardOutput.ReadToEndAsync();
+            Task<string> standardError = process.StandardError.ReadToEndAsync();
+            using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(6));
+            await process.WaitForExitAsync(timeout.Token);
+            string diagnostic = $"{await standardOutput}\n{await standardError}";
+
+            Assert.NotEqual(0, process.ExitCode);
+            Assert.Contains(expectedDiagnostic, diagnostic, StringComparison.OrdinalIgnoreCase);
+            AssertTcpPortIsReleased(httpPort);
+            AssertUdpPortIsReleased(udpPort);
+        }
+        finally
+        {
+            await StopProcessTreeAsync(process);
+            File.Delete(secretPath);
+        }
+    }
+
+    private static ProcessStartInfo CreateStartInfo(int httpPort, int udpPort, string secretPath)
+    {
+        string assembly = typeof(Program).Assembly.Location;
+        ProcessStartInfo info = new()
+        {
+            FileName = "dotnet",
+            WorkingDirectory = Path.GetDirectoryName(assembly)!,
+            RedirectStandardOutput = true,
+            RedirectStandardError = true,
+            UseShellExecute = false,
+        };
+        info.ArgumentList.Add(assembly);
+        Dictionary<string, string> settings = new(StringComparer.Ordinal)
+        {
+            ["ASPNETCORE_ENVIRONMENT"] = "Production",
+            ["ASPNETCORE_URLS"] = $"http://127.0.0.1:{httpPort}",
+            ["AllowedHosts"] = "127.0.0.1",
+            ["Rendezvous__Deployment__PublicHttpBaseUrl"] = "https://127.0.0.1/",
+            ["Rendezvous__Deployment__PublicUdpHost"] = "127.0.0.1",
+            ["Rendezvous__Deployment__PublicUdpPort"] = udpPort.ToString(System.Globalization.CultureInfo.InvariantCulture),
+            ["Rendezvous__Deployment__DrainDeadlineSeconds"] = "3",
+            ["Rendezvous__Deployment__MinimumDrainSeconds"] = "1",
+            ["Rendezvous__Deployment__SingleActiveInstance"] = "true",
+            ["Rendezvous__Deployment__AllowPrivatePublicEndpoints"] = "true",
+            ["Rendezvous__Udp__ListenAddress"] = "127.0.0.1",
+            ["Rendezvous__Udp__Port"] = udpPort.ToString(System.Globalization.CultureInfo.InvariantCulture),
+            ["Rendezvous__AbuseProtection__TrustedProxyAddresses__0"] = "127.0.0.1",
+            ["Rendezvous__Provisioning__Issuer"] = "rendezvous-process-test",
+            ["Rendezvous__Provisioning__Audience"] = "rendezvous-service",
+            ["Rendezvous__Provisioning__ClockSkewSeconds"] = "30",
+            ["Rendezvous__Provisioning__SigningKeys__0__KeyId"] = "process-test-key",
+            ["Rendezvous__Provisioning__SigningKeys__0__SecretReference"] = $"file:{secretPath}",
+            ["Rendezvous__Provisioning__SigningKeys__0__CredentialKinds__0"] = "DedicatedPublisher",
+            ["Rendezvous__Provisioning__SigningKeys__0__GameId"] = "space-game",
+            ["Rendezvous__Provisioning__SigningKeys__0__EnvironmentId"] = "process-test",
+            ["Rendezvous__Provisioning__SigningKeys__0__NotBefore"] = DateTimeOffset.UtcNow.AddHours(-1).ToString("O"),
+            ["Rendezvous__Provisioning__SigningKeys__0__SignUntil"] = DateTimeOffset.UtcNow.AddDays(1).ToString("O"),
+            ["Rendezvous__Provisioning__SigningKeys__0__VerifyUntil"] = DateTimeOffset.UtcNow.AddDays(2).ToString("O"),
+            ["Rendezvous__Provisioning__Games__0__GameId"] = "space-game",
+            ["Rendezvous__Provisioning__Games__0__EnvironmentId"] = "process-test",
+            ["Rendezvous__Provisioning__Games__0__Enabled"] = "true",
+            ["Rendezvous__Provisioning__Games__0__ProtocolVersions__0"] = "1",
+            ["Rendezvous__Provisioning__Games__0__Regions__0"] = "local",
+            ["Rendezvous__Provisioning__Games__0__VisibilityModes__0"] = "Public",
+            ["Rendezvous__Provisioning__Games__0__PublisherTrustModes__0"] = "ManagedDedicated",
+            ["Rendezvous__Provisioning__Games__0__MetadataMaxBytes"] = "512",
+            ["Rendezvous__Provisioning__Games__0__MetadataMaxKeys"] = "0",
+            ["Rendezvous__Provisioning__Games__0__MaxListingsPerPrincipal"] = "10",
+            ["Rendezvous__Provisioning__Games__0__MaxAnonymousListingsPerAddress"] = "0",
+            ["Rendezvous__Provisioning__Games__0__MaxActiveJoinAttempts"] = "100",
+            ["Rendezvous__Provisioning__Games__0__FallbackPolicy"] = "Disabled",
+        };
+        foreach ((string key, string value) in settings)
+        {
+            info.Environment[key] = value;
+        }
+
+        return info;
+    }
+
+    private static ProcessStartInfo CreateBareProductionStartInfo(int httpPort)
+    {
+        string assembly = typeof(Program).Assembly.Location;
+        ProcessStartInfo info = new()
+        {
+            FileName = "dotnet",
+            WorkingDirectory = Path.GetDirectoryName(assembly)!,
+            RedirectStandardOutput = true,
+            RedirectStandardError = true,
+            UseShellExecute = false,
+        };
+        info.ArgumentList.Add(assembly);
+        foreach (string key in info.Environment.Keys
+                     .Where(static key => key.StartsWith("Rendezvous__", StringComparison.OrdinalIgnoreCase))
+                     .ToArray())
+        {
+            info.Environment.Remove(key);
+        }
+
+        info.Environment["ASPNETCORE_ENVIRONMENT"] = "Production";
+        info.Environment["ASPNETCORE_URLS"] = $"http://127.0.0.1:{httpPort}";
+        info.Environment["AllowedHosts"] = "*";
+        return info;
+    }
+
+    private static async Task StopProcessTreeAsync(Process? process)
+    {
+        if (process is null)
+        {
+            return;
+        }
+
+        try
+        {
+            if (!process.HasExited)
+            {
+                process.Kill(entireProcessTree: true);
+                using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(3));
+                await process.WaitForExitAsync(timeout.Token);
+            }
+        }
+        finally
+        {
+            process.Dispose();
+        }
+    }
+
+    private static async Task SendSigtermAsync(Process process)
+    {
+        ProcessStartInfo signalInfo = new()
+        {
+            FileName = "/bin/kill",
+            UseShellExecute = false,
+            ArgumentList =
+            {
+                "-TERM",
+                process.Id.ToString(System.Globalization.CultureInfo.InvariantCulture),
+            },
+        };
+        using Process signal = Process.Start(signalInfo)
+            ?? throw new InvalidOperationException("Could not send SIGTERM.");
+        await signal.WaitForExitAsync();
+        Assert.Equal(0, signal.ExitCode);
+    }
+
+    private static string BuildConfiguration()
+    {
+        string path = typeof(ProductionProcessTests).Assembly.Location;
+        return path.Contains(
+            $"{Path.DirectorySeparatorChar}Release{Path.DirectorySeparatorChar}",
+            StringComparison.Ordinal)
+            ? "Release"
+            : "Debug";
+    }
+
+    private static string RepositoryRoot()
+    {
+        DirectoryInfo? directory = new(AppContext.BaseDirectory);
+        while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "Rendezvous.slnx")))
+        {
+            directory = directory.Parent;
+        }
+
+        return directory?.FullName
+            ?? throw new InvalidOperationException("Could not locate the repository root.");
+    }
+
+    private static async Task WaitForReadyAsync(int port, Process process, TimeSpan timeout)
+    {
+        using HttpClient client = new() { Timeout = TimeSpan.FromMilliseconds(500) };
+        Stopwatch elapsed = Stopwatch.StartNew();
+        while (elapsed.Elapsed < timeout)
+        {
+            if (process.HasExited)
+            {
+                throw new InvalidOperationException("The production server exited before readiness.");
+            }
+
+            try
+            {
+                using HttpResponseMessage response = await client.GetAsync(
+                    $"http://127.0.0.1:{port}/health/ready");
+                if (response.StatusCode == HttpStatusCode.OK)
+                {
+                    return;
+                }
+            }
+            catch (HttpRequestException)
+            {
+            }
+            catch (TaskCanceledException)
+            {
+            }
+
+            await Task.Delay(50);
+        }
+
+        throw new TimeoutException("The production server did not become ready.");
+    }
+
+    private static int ReserveTcpPort()
+    {
+        TcpListener listener = new(IPAddress.Loopback, 0);
+        listener.Start();
+        int port = ((IPEndPoint)listener.LocalEndpoint).Port;
+        listener.Stop();
+        return port;
+    }
+
+    private static int ReserveUdpPort()
+    {
+        using UdpClient client = new(new IPEndPoint(IPAddress.Loopback, 0));
+        return ((IPEndPoint)client.Client.LocalEndPoint!).Port;
+    }
+
+    private static void AssertUdpPortIsBound(int port)
+    {
+        using Socket socket = new(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
+        Assert.Throws<SocketException>(() => socket.Bind(new IPEndPoint(IPAddress.Loopback, port)));
+    }
+
+    private static void AssertTcpPortIsReleased(int port)
+    {
+        TcpListener listener = new(IPAddress.Loopback, port);
+        listener.Start();
+        listener.Stop();
+    }
+
+    private static void AssertUdpPortIsReleased(int port)
+    {
+        using Socket socket = new(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
+        socket.Bind(new IPEndPoint(IPAddress.Loopback, port));
+    }
+}