feat(operations): add capacity and resilience gates (#18)

tests/FinalFactory.Rendezvous.Tests/Deployment/ProductionProcessTests.cs

@@ -61,6 +61,32 @@ public sealed class ProductionProcessTests
             Assert.InRange(shutdown.Elapsed, TimeSpan.FromMilliseconds(700), TimeSpan.FromSeconds(5));
             AssertTcpPortIsReleased(httpPort);
             AssertUdpPortIsReleased(udpPort);
+
+            process.Dispose();
+            process = null;
+            Stopwatch replacementReady = Stopwatch.StartNew();
+            ProcessStartInfo replacementInfo = CreateStartInfo(httpPort, udpPort, secretPath);
+            process = Process.Start(replacementInfo)
+                ?? throw new InvalidOperationException("The replacement production process did not start.");
+            Task<string> replacementOutput = process.StandardOutput.ReadToEndAsync();
+            Task<string> replacementError = process.StandardError.ReadToEndAsync();
+            await WaitForReadyAsync(httpPort, process, TimeSpan.FromSeconds(15));
+            Assert.True(
+                replacementReady.Elapsed < TimeSpan.FromSeconds(15),
+                $"Replacement readiness took {replacementReady.Elapsed}.");
+            AssertUdpPortIsBound(udpPort);
+
+            await SendSigtermAsync(process);
+            using CancellationTokenSource replacementTimeout = new(TimeSpan.FromSeconds(6));
+            await process.WaitForExitAsync(replacementTimeout.Token);
+            string replacementFinalOutput = await replacementOutput;
+            string replacementFinalError = await replacementError;
+            Assert.True(
+                process.ExitCode == 0,
+                $"Replacement exited with {process.ExitCode}. "
+                + $"stdout: {replacementFinalOutput} stderr: {replacementFinalError}");
+            AssertTcpPortIsReleased(httpPort);
+            AssertUdpPortIsReleased(udpPort);
         }
         finally
         {
@@ -79,6 +105,97 @@ public sealed class ProductionProcessTests
         }
     }
 
+    [Fact]
+    public async Task ProductionTransportSoakKeepsHandlesMemoryAndSocketsBounded()
+    {
+        if (!OperatingSystem.IsLinux())
+        {
+            return;
+        }
+
+        int httpPort = ReserveTcpPort();
+        int udpPort = ReserveUdpPort();
+        string secretPath = Path.Combine(
+            Path.GetTempPath(),
+            $"rendezvous-transport-soak-secret-{Guid.NewGuid():N}");
+        await File.WriteAllBytesAsync(secretPath, RandomNumberGenerator.GetBytes(32));
+        Process? process = null;
+        try
+        {
+            process = Process.Start(CreateStartInfo(httpPort, udpPort, secretPath))
+                ?? throw new InvalidOperationException("The production soak process did not start.");
+            Task<string> standardOutput = process.StandardOutput.ReadToEndAsync();
+            Task<string> standardError = process.StandardError.ReadToEndAsync();
+            await WaitForReadyAsync(httpPort, process, TimeSpan.FromSeconds(10));
+            process.Refresh();
+            int baselineHandles = process.HandleCount;
+            long baselineWorkingSet = process.WorkingSet64;
+            int peakHandles = baselineHandles;
+            byte[] invalidDatagram = RandomNumberGenerator.GetBytes(64);
+            IPEndPoint udpEndpoint = new(IPAddress.Loopback, udpPort);
+            using HttpClient client = new() { Timeout = TimeSpan.FromSeconds(1) };
+            using UdpClient udp = new();
+            Stopwatch soak = Stopwatch.StartNew();
+            int cycles = 0;
+            int accepted = 0;
+            int shed = 0;
+            while (soak.Elapsed < TimeSpan.FromSeconds(10))
+            {
+                using HttpResponseMessage response = await client.GetAsync(
+                    $"http://127.0.0.1:{httpPort}/health/live");
+                if (response.StatusCode == HttpStatusCode.OK)
+                {
+                    accepted++;
+                }
+                else
+                {
+                    Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode);
+                    shed++;
+                }
+
+                await udp.SendAsync(invalidDatagram, udpEndpoint);
+                cycles++;
+                if (cycles % 100 == 0)
+                {
+                    process.Refresh();
+                    peakHandles = Math.Max(peakHandles, process.HandleCount);
+                }
+            }
+
+            Assert.True(cycles >= 100, $"Transport soak completed only {cycles} cycles.");
+            Assert.True(accepted > 0, "Transport soak never admitted a health request.");
+            Assert.True(shed > 0, "Transport soak never exercised typed HTTP load shedding.");
+            await Task.Delay(TimeSpan.FromSeconds(2));
+            using (HttpResponseMessage recovered = await client.GetAsync(
+                       $"http://127.0.0.1:{httpPort}/health/live"))
+            {
+                Assert.Equal(HttpStatusCode.OK, recovered.StatusCode);
+            }
+
+            process.Refresh();
+            Assert.InRange(peakHandles, 0, baselineHandles + 32);
+            Assert.InRange(process.HandleCount, 0, baselineHandles + 16);
+            Assert.InRange(process.WorkingSet64, 0, baselineWorkingSet + 67_108_864);
+            AssertUdpPortIsBound(udpPort);
+
+            await SendSigtermAsync(process);
+            using CancellationTokenSource shutdownTimeout = new(TimeSpan.FromSeconds(6));
+            await process.WaitForExitAsync(shutdownTimeout.Token);
+            string output = await standardOutput;
+            string error = await standardError;
+            Assert.True(
+                process.ExitCode == 0,
+                $"Transport soak process failed. stdout: {output} stderr: {error}");
+            AssertTcpPortIsReleased(httpPort);
+            AssertUdpPortIsReleased(udpPort);
+        }
+        finally
+        {
+            await StopProcessTreeAsync(process);
+            File.Delete(secretPath);
+        }
+    }
+
     [Fact]
     public async Task DocumentedSmokeScriptReachesHttpAndAuthenticatedUdpFlow()
     {