feat(operations): add capacity and resilience gates (#18)

2026-07-16 15:57:01 +02:00
parent 08729ae25c
commit 609dad7cf1
21 changed files with 1759 additions and 107 deletions
@@ -1,3 +1,6 @@
+using System.Diagnostics;
+using System.Net;
+using System.Net.Sockets;
 using FinalFactory.Rendezvous.Client;
 using FinalFactory.Rendezvous.Contracts;
 using FinalFactory.Rendezvous.Server.Abuse;
@@ -19,6 +22,49 @@ namespace FinalFactory.Rendezvous.Tests.Client;

 public sealed class RendezvousClientIntegrationTests
 {
+    [Fact]
+    public async Task RestartReturnsTypedUnavailabilityThenAllowsHostReregistration()
+    {
+        int port = ReserveTcpPort();
+        string address = $"http://127.0.0.1:{port}";
+        using HttpClient client = new() { BaseAddress = new Uri(address) };
+        RendezvousClientOptions noRetry = new()
+        {
+            MaximumSafeRetries = 0,
+            RequestTimeout = TimeSpan.FromSeconds(1),
+        };
+        ClientTestHost first = await ClientTestHost.StartAsync(address);
+        RendezvousPublisherClient publisher = new(client, noRetry);
+        RendezvousClientResult<PublishedSession> registered = await publisher.RegisterAsync(
+            CreateRegistration(100),
+            first.PublisherCredential);
+        PublishedSession initialSession = AssertSuccess(registered);
+        BindPresence(first, initialSession, 41_100);
+        RendezvousSessionBrowserClient browser = new(client, noRetry);
+        BrowseSessionsResponse beforeRestart = AssertSuccess(await browser.BrowseAsync(BrowseRequest()));
+        Assert.Equal(initialSession.ListingId, Assert.Single(beforeRestart.Items).ListingId);
+        Stopwatch restart = Stopwatch.StartNew();
+        await first.DisposeAsync();
+
+        RendezvousClientResult<BrowseSessionsResponse> unavailable = await browser.BrowseAsync(BrowseRequest());
+        Assert.False(unavailable.IsSuccess);
+        Assert.Equal(RendezvousErrorCode.ServiceUnavailable, unavailable.Error);
+
+        await using ClientTestHost second = await ClientTestHost.StartAsync(address);
+        RendezvousClientResult<PublishedSession> reregistered = await publisher.RegisterAsync(
+            CreateRegistration(101),
+            second.PublisherCredential);
+        PublishedSession replacementSession = AssertSuccess(reregistered);
+        Assert.NotEqual(initialSession.ListingId, replacementSession.ListingId);
+        BindPresence(second, replacementSession, 41_101);
+        BrowseSessionsResponse afterRestart = AssertSuccess(await browser.BrowseAsync(BrowseRequest()));
+        Assert.Equal(replacementSession.ListingId, Assert.Single(afterRestart.Items).ListingId);
+        Assert.DoesNotContain(afterRestart.Items, item => item.ListingId == initialSession.ListingId);
+        Assert.True(
+            restart.Elapsed < TimeSpan.FromSeconds(5),
+            $"Local restart and host re-registration took {restart.Elapsed}.");
+    }
+
    [Fact]
    public async Task PublisherAndBrowserClientsCompleteTheRealSessionLifecycleAndPaging()
    {
@@ -102,6 +148,27 @@ public sealed class RendezvousClientIntegrationTests
        return Assert.IsAssignableFrom<T>(result.Value);
    }

+    private static void BindPresence(ClientTestHost host, PublishedSession session, int port)
+    {
+        Assert.True(host.Capabilities.TryFingerprint(
+            session.HostPresenceCapability,
+            out SecretFingerprint fingerprint));
+        Assert.Equal(StoreResultCode.Success, host.Store.BindHostPresence(new(
+            session.HostPresenceHandle,
+            fingerprint,
+            new(AddressFamilyKind.Ipv4, "203.0.113.80", port),
+            null)).Code);
+    }
+
+    private static BrowseSessionsRequest BrowseRequest() => new()
+    {
+        GameId = new("space-game"),
+        EnvironmentId = new("production"),
+        RegionId = new("eu-central"),
+        ProtocolVersion = 7,
+        PageSize = 10,
+    };
+
    private static RegisterSessionRequest CreateRegistration(int index) => new()
    {
        IdempotencyKey = $"sdk-integration-{index}",
@@ -139,7 +206,7 @@ public sealed class RendezvousClientIntegrationTests
        internal EphemeralCapabilityIssuer Capabilities { get; }
        internal string PublisherCredential { get; }

-        internal static async Task<ClientTestHost> StartAsync()
+        internal static async Task<ClientTestHost> StartAsync(string? bindAddress = null)
        {
            ManualRendezvousClock clock = new(ProvisioningTestData.Now);
            EphemeralStoreOptions stateOptions = new();
@@ -153,7 +220,7 @@ public sealed class RendezvousClientIntegrationTests
            string credential = provisioning.Credentials.Issue(principal, clock.UtcNow);

            WebApplicationBuilder builder = WebApplication.CreateBuilder();
-            builder.WebHost.UseUrls("http://127.0.0.1:0");
+            builder.WebHost.UseUrls(bindAddress ?? "http://127.0.0.1:0");
            builder.Services.ConfigureHttpJsonOptions(static options =>
                ContractJson.Configure(options.SerializerOptions));
            builder.Services.Configure<RouteHandlerOptions>(static options =>
@@ -180,10 +247,10 @@ public sealed class RendezvousClientIntegrationTests
            app.MapRendezvousContractEndpoints();
            await app.StartAsync();
            IServer server = app.Services.GetRequiredService<IServer>();
-            string address = Assert.Single(server.Features.Get<IServerAddressesFeature>()!.Addresses);
+            string serviceAddress = Assert.Single(server.Features.Get<IServerAddressesFeature>()!.Addresses);
            return new(
                app,
-                new HttpClient { BaseAddress = new Uri(address) },
+                new HttpClient { BaseAddress = new Uri(serviceAddress) },
                store,
                capabilities,
                credential);
@@ -196,4 +263,13 @@ public sealed class RendezvousClientIntegrationTests
            await _application.DisposeAsync();
        }
    }
+
+    private static int ReserveTcpPort()
+    {
+        TcpListener listener = new(IPAddress.Loopback, 0);
+        listener.Start();
+        int port = ((IPEndPoint)listener.LocalEndpoint).Port;
+        listener.Stop();
+        return port;
+    }
 }
@@ -61,6 +61,32 @@ public sealed class ProductionProcessTests
            Assert.InRange(shutdown.Elapsed, TimeSpan.FromMilliseconds(700), TimeSpan.FromSeconds(5));
            AssertTcpPortIsReleased(httpPort);
            AssertUdpPortIsReleased(udpPort);
+
+            process.Dispose();
+            process = null;
+            Stopwatch replacementReady = Stopwatch.StartNew();
+            ProcessStartInfo replacementInfo = CreateStartInfo(httpPort, udpPort, secretPath);
+            process = Process.Start(replacementInfo)
+                ?? throw new InvalidOperationException("The replacement production process did not start.");
+            Task<string> replacementOutput = process.StandardOutput.ReadToEndAsync();
+            Task<string> replacementError = process.StandardError.ReadToEndAsync();
+            await WaitForReadyAsync(httpPort, process, TimeSpan.FromSeconds(15));
+            Assert.True(
+                replacementReady.Elapsed < TimeSpan.FromSeconds(15),
+                $"Replacement readiness took {replacementReady.Elapsed}.");
+            AssertUdpPortIsBound(udpPort);
+
+            await SendSigtermAsync(process);
+            using CancellationTokenSource replacementTimeout = new(TimeSpan.FromSeconds(6));
+            await process.WaitForExitAsync(replacementTimeout.Token);
+            string replacementFinalOutput = await replacementOutput;
+            string replacementFinalError = await replacementError;
+            Assert.True(
+                process.ExitCode == 0,
+                $"Replacement exited with {process.ExitCode}. "
+                + $"stdout: {replacementFinalOutput} stderr: {replacementFinalError}");
+            AssertTcpPortIsReleased(httpPort);
+            AssertUdpPortIsReleased(udpPort);
        }
        finally
        {
@@ -79,6 +105,97 @@ public sealed class ProductionProcessTests
        }
    }

+    [Fact]
+    public async Task ProductionTransportSoakKeepsHandlesMemoryAndSocketsBounded()
+    {
+        if (!OperatingSystem.IsLinux())
+        {
+            return;
+        }
+
+        int httpPort = ReserveTcpPort();
+        int udpPort = ReserveUdpPort();
+        string secretPath = Path.Combine(
+            Path.GetTempPath(),
+            $"rendezvous-transport-soak-secret-{Guid.NewGuid():N}");
+        await File.WriteAllBytesAsync(secretPath, RandomNumberGenerator.GetBytes(32));
+        Process? process = null;
+        try
+        {
+            process = Process.Start(CreateStartInfo(httpPort, udpPort, secretPath))
+                ?? throw new InvalidOperationException("The production soak process did not start.");
+            Task<string> standardOutput = process.StandardOutput.ReadToEndAsync();
+            Task<string> standardError = process.StandardError.ReadToEndAsync();
+            await WaitForReadyAsync(httpPort, process, TimeSpan.FromSeconds(10));
+            process.Refresh();
+            int baselineHandles = process.HandleCount;
+            long baselineWorkingSet = process.WorkingSet64;
+            int peakHandles = baselineHandles;
+            byte[] invalidDatagram = RandomNumberGenerator.GetBytes(64);
+            IPEndPoint udpEndpoint = new(IPAddress.Loopback, udpPort);
+            using HttpClient client = new() { Timeout = TimeSpan.FromSeconds(1) };
+            using UdpClient udp = new();
+            Stopwatch soak = Stopwatch.StartNew();
+            int cycles = 0;
+            int accepted = 0;
+            int shed = 0;
+            while (soak.Elapsed < TimeSpan.FromSeconds(10))
+            {
+                using HttpResponseMessage response = await client.GetAsync(
+                    $"http://127.0.0.1:{httpPort}/health/live");
+                if (response.StatusCode == HttpStatusCode.OK)
+                {
+                    accepted++;
+                }
+                else
+                {
+                    Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode);
+                    shed++;
+                }
+
+                await udp.SendAsync(invalidDatagram, udpEndpoint);
+                cycles++;
+                if (cycles % 100 == 0)
+                {
+                    process.Refresh();
+                    peakHandles = Math.Max(peakHandles, process.HandleCount);
+                }
+            }
+
+            Assert.True(cycles >= 100, $"Transport soak completed only {cycles} cycles.");
+            Assert.True(accepted > 0, "Transport soak never admitted a health request.");
+            Assert.True(shed > 0, "Transport soak never exercised typed HTTP load shedding.");
+            await Task.Delay(TimeSpan.FromSeconds(2));
+            using (HttpResponseMessage recovered = await client.GetAsync(
+                       $"http://127.0.0.1:{httpPort}/health/live"))
+            {
+                Assert.Equal(HttpStatusCode.OK, recovered.StatusCode);
+            }
+
+            process.Refresh();
+            Assert.InRange(peakHandles, 0, baselineHandles + 32);
+            Assert.InRange(process.HandleCount, 0, baselineHandles + 16);
+            Assert.InRange(process.WorkingSet64, 0, baselineWorkingSet + 67_108_864);
+            AssertUdpPortIsBound(udpPort);
+
+            await SendSigtermAsync(process);
+            using CancellationTokenSource shutdownTimeout = new(TimeSpan.FromSeconds(6));
+            await process.WaitForExitAsync(shutdownTimeout.Token);
+            string output = await standardOutput;
+            string error = await standardError;
+            Assert.True(
+                process.ExitCode == 0,
+                $"Transport soak process failed. stdout: {output} stderr: {error}");
+            AssertTcpPortIsReleased(httpPort);
+            AssertUdpPortIsReleased(udpPort);
+        }
+        finally
+        {
+            await StopProcessTreeAsync(process);
+            File.Delete(secretPath);
+        }
+    }
+
    [Fact]
    public async Task DocumentedSmokeScriptReachesHttpAndAuthenticatedUdpFlow()
    {
@@ -411,6 +411,92 @@ public sealed class InMemoryEphemeralRendezvousStoreTests
        Assert.True(fixture.Store.CreateJoinAttempt(attempt).Succeeded);
        Assert.Equal(StoreResultCode.CapacityExceeded, fixture.Store.CreateJoinAttempt(
            fixture.AttemptCommand(listing, "client-quota-2") with { ScopeAttemptLimit = 1 }).Code);
+
+        fixture.Clock.Advance(TimeSpan.FromSeconds(30));
+        Assert.True(fixture.Store.BindHostPresence(new(
+            first.Listing.HostPresenceHandle,
+            first.Listing.HostPresenceFingerprint,
+            EphemeralStateFixture.PublicEndpoint(42_101),
+            null)).Succeeded);
+        Assert.True(fixture.Store.CreateJoinAttempt(
+            fixture.AttemptCommand(listing, "client-quota-3") with { ScopeAttemptLimit = 1 }).Succeeded);
+    }
+
+    [Fact]
+    public void RemovingAListingReleasesItsConstantTimeOwnerQuota()
+    {
+        EphemeralStateFixture fixture = new();
+        CreateListingCommand first = fixture.ListingCommand(owner: "publisher-quota") with
+        {
+            OwnerListingLimit = 1,
+        };
+        CreateListingCommand second = fixture.ListingCommand(owner: "publisher-quota") with
+        {
+            OwnerListingLimit = 1,
+        };
+        StoreResult<StoredListing> created = fixture.Store.CreateListing(first);
+        Assert.True(created.Succeeded);
+        Assert.Equal(StoreResultCode.CapacityExceeded, fixture.Store.CreateListing(second).Code);
+
+        Assert.True(fixture.Store.DeleteListing(new(
+            first.Listing.ListingId,
+            first.Listing.LeaseId,
+            first.Listing.LeaseFingerprint,
+            first.Listing.OwnerSubject)).Succeeded);
+        Assert.True(fixture.Store.CreateListing(second).Succeeded);
+    }
+
+    [Fact]
+    public void RepeatedMutableDeadlineRefreshesKeepOneScheduledEntryPerKey()
+    {
+        EphemeralStateFixture fixture = new();
+        StoredListing listing = fixture.CreateVisibleListing(out _);
+
+        for (int iteration = 0; iteration < 10_000; iteration++)
+        {
+            fixture.Clock.Advance(TimeSpan.FromTicks(1));
+            StoreResult<StoredListing> renewed = fixture.Store.RenewLease(new(
+                listing.Definition.ListingId,
+                listing.Definition.LeaseId,
+                listing.Definition.LeaseFingerprint,
+                listing.Definition.OwnerSubject,
+                listing.Version));
+            Assert.True(renewed.Succeeded, $"Renewal {iteration} failed with {renewed.Code}.");
+            listing = renewed.Value!;
+            StoreResult<StoredListing> presence = fixture.Store.BindHostPresence(new(
+                listing.Definition.HostPresenceHandle,
+                listing.Definition.HostPresenceFingerprint,
+                EphemeralStateFixture.PublicEndpoint(42_200),
+                null));
+            Assert.True(presence.Succeeded, $"Presence {iteration} failed with {presence.Code}.");
+            StoreResult<int> revoked = fixture.Store.RevokePrincipal(
+                "repeated-revocation",
+                TimeSpan.FromMinutes(1));
+            Assert.True(revoked.Succeeded, $"Revocation {iteration} failed with {revoked.Code}.");
+        }
+
+        Assert.Equal(4, fixture.Store.ScheduledExpiryEntryCount);
+        fixture.Clock.Advance(TimeSpan.FromSeconds(19));
+        Assert.True(fixture.Store.GetListing(listing.Definition.ListingId, true).Succeeded);
+        Assert.Equal(4, fixture.Store.ScheduledExpiryEntryCount);
+    }
+
+    [Fact]
+    public void RepeatedPrincipalRevocationCanExtendButCannotShortenProtection()
+    {
+        EphemeralStateFixture fixture = new();
+        const string subject = "protected-publisher";
+
+        Assert.True(fixture.Store.RevokePrincipal(subject, TimeSpan.FromMinutes(10)).Succeeded);
+        fixture.Clock.Advance(TimeSpan.FromSeconds(1));
+        Assert.True(fixture.Store.RevokePrincipal(subject, TimeSpan.FromSeconds(1)).Succeeded);
+        fixture.Clock.Advance(TimeSpan.FromSeconds(2));
+        Assert.Equal(
+            StoreResultCode.Revoked,
+            fixture.Store.CreateListing(fixture.ListingCommand(owner: subject)).Code);
+
+        fixture.Clock.Advance(TimeSpan.FromSeconds(597));
+        Assert.True(fixture.Store.CreateListing(fixture.ListingCommand(owner: subject)).Succeeded);
    }

    [Fact]