Rendezvous/tests/FinalFactory.Rendezvous.Tests/TestClient/TestClientProcessIntegrationTests.cs

using System.Collections.Concurrent;
using System.Diagnostics;
using System.Net;
using System.Security.Cryptography;
using System.Text.Json;
using FinalFactory.Rendezvous.Contracts;
using FinalFactory.Rendezvous.Server.Provisioning;
using FinalFactory.Rendezvous.TestClient;

namespace FinalFactory.Rendezvous.Tests.TestClient;

public sealed class TestClientProcessIntegrationTests
{
#if DEBUG
    private const string BuildConfiguration = "Debug";
#else
    private const string BuildConfiguration = "Release";
#endif

    [Fact]
    public async Task ServerHostAndJoinProcessesExchangeAuthenticatedDirectTrafficWithoutLeakingSecrets()
    {
        using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(30));
        string root = RepositoryRoot();
        string serverAssembly = Path.Combine(
            root,
            $"src/FinalFactory.Rendezvous.Server/bin/{BuildConfiguration}/net10.0/FinalFactory.Rendezvous.Server.dll");
        string clientAssembly = Path.Combine(
            root,
            $"src/FinalFactory.Rendezvous.TestClient/bin/{BuildConfiguration}/net8.0/FinalFactory.Rendezvous.TestClient.dll");
        Assert.True(File.Exists(serverAssembly), $"Missing server build output: {serverAssembly}");
        Assert.True(File.Exists(clientAssembly), $"Missing TestClient build output: {clientAssembly}");

        DateTimeOffset now = DateTimeOffset.UtcNow;
        byte[] signingKey = RandomNumberGenerator.GetBytes(32);
        string signingKeyText = Convert.ToBase64String(signingKey);
        string publisherCredential = IssuePublisherCredential(signingKey, now);
        CryptographicOperations.ZeroMemory(signingKey);
        Dictionary<string, string> serverEnvironment = ServerEnvironment(
            signingKeyText,
            now);

        await using ProcessCapture server = Start(
            serverAssembly,
            [],
            serverEnvironment);
        string httpLine = await server.WaitForLineAsync("Now listening on: http://127.0.0.1:", timeout.Token);
        string serviceUrl = ParseServiceUrl(httpLine);
        string udpLine = await server.WaitForLineAsync("UDP mediator listening on 127.0.0.1:", timeout.Token);
        int udpPort = ParseTrailingPort(udpLine);
        await WaitForReadyAsync(serviceUrl, server, timeout.Token);

        string[] emptyBrowseArguments =
        [
            "browse",
            "--service", serviceUrl,
            "--game", "space-game",
            "--environment", "integration",
            "--region", "local",
            "--protocol", "1",
            "--script",
            "--json",
            "--timeout-seconds", "15",
        ];
        await using ProcessCapture emptyBrowse = Start(
            clientAssembly,
            emptyBrowseArguments,
            new Dictionary<string, string>(StringComparer.Ordinal));
        Assert.Equal(
            (int)TestClientExitCode.NoCompatibleSession,
            await emptyBrowse.WaitForExitAsync(timeout.Token));
        Assert.True(emptyBrowse.HasEvent("browse.completed", "complete"), emptyBrowse.DiagnosticText());
        Assert.Contains(
            emptyBrowse.JsonEvents(),
            item => item.GetProperty("event").GetString() == "browse.completed"
                && item.GetProperty("count").GetInt32() == 0);

        string[] missingJoinArguments =
        [
            "join",
            "--service", serviceUrl,
            "--mediator", $"127.0.0.1:{udpPort}",
            "--game", "space-game",
            "--environment", "integration",
            "--region", "local",
            "--protocol", "1",
            "--listing", Guid.NewGuid().ToString("D"),
            "--script",
            "--json",
            "--timeout-seconds", "15",
        ];
        await using ProcessCapture missingJoin = Start(
            clientAssembly,
            missingJoinArguments,
            new Dictionary<string, string>(StringComparer.Ordinal));
        Assert.Equal((int)TestClientExitCode.ServiceFailure, await missingJoin.WaitForExitAsync(timeout.Token));
        Assert.True(missingJoin.HasEvent("join.selection", "failed"), missingJoin.DiagnosticText());

        string[] boundedHostArguments =
        [
            "host",
            "--service", serviceUrl,
            "--mediator", $"127.0.0.1:{udpPort}",
            "--game", "space-game",
            "--environment", "integration",
            "--region", "local",
            "--protocol", "1",
            "--script",
            "--json",
            "--exit-after-echo",
            "--run-seconds", "1",
            "--timeout-seconds", "15",
        ];
        await using ProcessCapture boundedHost = Start(
            clientAssembly,
            boundedHostArguments,
            new Dictionary<string, string>(StringComparer.Ordinal)
            {
                ["RENDEZVOUS_PUBLISHER_CREDENTIAL"] = publisherCredential,
            });
        Assert.Equal((int)TestClientExitCode.DirectTrafficFailed, await boundedHost.WaitForExitAsync(timeout.Token));
        Assert.True(boundedHost.HasEvent("host.direct-traffic", "timed-out"), boundedHost.DiagnosticText());
        Assert.True(boundedHost.HasEvent("host.deregistered", "complete"), boundedHost.DiagnosticText());

        string[] hostArguments =
        [
            "host",
            "--service", serviceUrl,
            "--mediator", $"127.0.0.1:{udpPort}",
            "--game", "space-game",
            "--environment", "integration",
            "--region", "local",
            "--protocol", "1",
            "--script",
            "--json",
            "--exit-after-echo",
            "--timeout-seconds", "15",
        ];
        await using ProcessCapture host = Start(
            clientAssembly,
            hostArguments,
            new Dictionary<string, string>(StringComparer.Ordinal)
            {
                ["RENDEZVOUS_PUBLISHER_CREDENTIAL"] = publisherCredential,
            });
        JsonElement hostRegistered = await host.WaitForEventAsync(
            "host.ready",
            timeout.Token);
        string listingId = Assert.IsType<string>(hostRegistered.GetProperty("listingId").GetString());

        string[] browseArguments =
        [
            "browse",
            "--service", serviceUrl,
            "--game", "space-game",
            "--environment", "integration",
            "--region", "local",
            "--protocol", "1",
            "--script",
            "--json",
            "--timeout-seconds", "15",
        ];
        await using ProcessCapture browse = Start(
            clientAssembly,
            browseArguments,
            new Dictionary<string, string>(StringComparer.Ordinal));
        Assert.Equal(0, await browse.WaitForExitAsync(timeout.Token));
        Assert.True(browse.HasEvent("browse.completed", "complete"), browse.DiagnosticText());
        Assert.True(
            browse.JsonEvents().Any(item =>
                item.GetProperty("event").GetString() == "browse.session"
                && item.GetProperty("status").GetString() == "available"
                && item.GetProperty("listingId").GetString() == listingId),
            browse.DiagnosticText());

        string[] joinArguments =
        [
            "join",
            "--service", serviceUrl,
            "--mediator", $"127.0.0.1:{udpPort}",
            "--game", "space-game",
            "--environment", "integration",
            "--region", "local",
            "--protocol", "1",
            "--listing", listingId,
            "--script",
            "--json",
            "--timeout-seconds", "15",
        ];
        await using ProcessCapture join = Start(
            clientAssembly,
            joinArguments,
            new Dictionary<string, string>(StringComparer.Ordinal));

        Assert.Equal(0, await join.WaitForExitAsync(timeout.Token));
        Assert.Equal(0, await host.WaitForExitAsync(timeout.Token));
        Assert.True(join.HasEvent("join.connected", "connected"), join.DiagnosticText());
        Assert.True(join.HasEvent("join.punch", "started"), join.DiagnosticText());
        Assert.True(join.HasEvent("join.direct-connect", "started"), join.DiagnosticText());
        Assert.True(join.HasEvent("join.direct-traffic", "verified"), join.DiagnosticText());
        Assert.True(join.HasEvent("join.outcome-report", "accepted"), join.DiagnosticText());
        Assert.True(host.HasEvent("host.direct-traffic", "verified"), host.DiagnosticText());
        Assert.True(host.HasEvent("host.punch", "started"), host.DiagnosticText());
        Assert.True(host.HasEvent("host.direct-connect", "connected"), host.DiagnosticText());
        Assert.True(host.HasEvent("host.deregistered", "complete"), host.DiagnosticText());
        Assert.Equal(host.AllLines().Count(), host.JsonEvents().Count);
        Assert.Equal(emptyBrowse.AllLines().Count(), emptyBrowse.JsonEvents().Count);
        Assert.Equal(missingJoin.AllLines().Count(), missingJoin.JsonEvents().Count);
        Assert.Equal(browse.AllLines().Count(), browse.JsonEvents().Count);
        Assert.Equal(join.AllLines().Count(), join.JsonEvents().Count);
        Assert.All(
            host.JsonEvents()
                .Concat(emptyBrowse.JsonEvents())
                .Concat(missingJoin.JsonEvents())
                .Concat(browse.JsonEvents())
                .Concat(join.JsonEvents()),
            AssertAllowlistedEventShape);

        using HttpClient service = new() { BaseAddress = new Uri(serviceUrl) };
        using HttpResponseMessage removed = await service.GetAsync(
            $"v1/sessions/{listingId}?contractVersion=1&gameId=space-game&environmentId=integration&protocolVersion=1",
            timeout.Token);
        Assert.Equal(HttpStatusCode.NotFound, removed.StatusCode);

        string captured = string.Join(
            '\n',
            server.AllLines()
                .Concat(boundedHost.AllLines())
                .Concat(emptyBrowse.AllLines())
                .Concat(missingJoin.AllLines())
                .Concat(host.AllLines())
                .Concat(browse.AllLines())
                .Concat(join.AllLines()));
        Assert.False(
            captured.Contains(publisherCredential, StringComparison.Ordinal),
            "Captured process output contained the publisher credential.");
        Assert.False(
            captured.Contains(signingKeyText, StringComparison.Ordinal),
            "Captured process output contained signing-key material.");
    }

    private static string IssuePublisherCredential(byte[] signingKey, DateTimeOffset now)
    {
        const string secretReference = "env:RENDEZVOUS_INTEGRATION_SIGNING_KEY";
        ProvisioningOptions options = ProvisioningOptions(now, secretReference);
        using DictionarySecretProvider secrets = new(new Dictionary<string, byte[]>(StringComparer.Ordinal)
        {
            [secretReference] = signingKey,
        });
        using ProvisioningRuntime provisioning = ProvisioningRuntime.Create(options, secrets, now);
        return provisioning.Credentials.Issue(
            new DedicatedPublisherPrincipal(
                "test-client-process-host",
                now.AddMinutes(5),
                new GameId("space-game"),
                new EnvironmentId("integration"),
                [new RegionId("local")]),
            now);
    }

    private static ProvisioningOptions ProvisioningOptions(
        DateTimeOffset now,
        string secretReference) => new()
        {
            Issuer = "rendezvous-process-test",
            Audience = "rendezvous-process-test-client",
            ClockSkewSeconds = 5,
            SigningKeys =
            [
                new SigningKeyOptions
                {
                    KeyId = "process-test-key",
                    SecretReference = secretReference,
                    CredentialKinds = [PrincipalCredentialKind.DedicatedPublisher],
                    GameId = "space-game",
                    EnvironmentId = "integration",
                    NotBefore = now.AddMinutes(-1),
                    SignUntil = now.AddMinutes(10),
                    VerifyUntil = now.AddMinutes(20),
                },
            ],
            Games =
            [
                new GamePolicyOptions
                {
                    GameId = "space-game",
                    EnvironmentId = "integration",
                    Enabled = true,
                    ProtocolVersions = [1],
                    Regions = ["local"],
                    VisibilityModes = [ListingVisibility.Public],
                    PublisherTrustModes = [PublisherTrustMode.ManagedDedicated],
                    MetadataMaxBytes = 256,
                    MetadataMaxKeys = 2,
                    MaxListingsPerPrincipal = 4,
                    MaxAnonymousListingsPerAddress = 1,
                    MaxActiveJoinAttempts = 16,
                },
            ],
        };

    private static Dictionary<string, string> ServerEnvironment(
        string signingKey,
        DateTimeOffset now)
    {
        Dictionary<string, string> values = new(StringComparer.Ordinal)
        {
            ["ASPNETCORE_ENVIRONMENT"] = "Production",
            ["ASPNETCORE_URLS"] = "http://127.0.0.1:0",
            ["Rendezvous__Udp__ListenAddress"] = "127.0.0.1",
            ["Rendezvous__Udp__Port"] = "0",
            ["Rendezvous__Udp__PollIntervalMilliseconds"] = "1",
            ["Rendezvous__Provisioning__Issuer"] = "rendezvous-process-test",
            ["Rendezvous__Provisioning__Audience"] = "rendezvous-process-test-client",
            ["Rendezvous__Provisioning__ClockSkewSeconds"] = "5",
            ["Rendezvous__Provisioning__SigningKeys__0__KeyId"] = "process-test-key",
            ["Rendezvous__Provisioning__SigningKeys__0__SecretReference"] =
                "env:RENDEZVOUS_INTEGRATION_SIGNING_KEY",
            ["Rendezvous__Provisioning__SigningKeys__0__CredentialKinds__0"] = "DedicatedPublisher",
            ["Rendezvous__Provisioning__SigningKeys__0__GameId"] = "space-game",
            ["Rendezvous__Provisioning__SigningKeys__0__EnvironmentId"] = "integration",
            ["Rendezvous__Provisioning__SigningKeys__0__NotBefore"] = now.AddMinutes(-1).ToString("O"),
            ["Rendezvous__Provisioning__SigningKeys__0__SignUntil"] = now.AddMinutes(10).ToString("O"),
            ["Rendezvous__Provisioning__SigningKeys__0__VerifyUntil"] = now.AddMinutes(20).ToString("O"),
            ["Rendezvous__Provisioning__Games__0__GameId"] = "space-game",
            ["Rendezvous__Provisioning__Games__0__EnvironmentId"] = "integration",
            ["Rendezvous__Provisioning__Games__0__Enabled"] = "true",
            ["Rendezvous__Provisioning__Games__0__ProtocolVersions__0"] = "1",
            ["Rendezvous__Provisioning__Games__0__Regions__0"] = "local",
            ["Rendezvous__Provisioning__Games__0__VisibilityModes__0"] = "Public",
            ["Rendezvous__Provisioning__Games__0__PublisherTrustModes__0"] = "ManagedDedicated",
            ["Rendezvous__Provisioning__Games__0__MetadataMaxBytes"] = "256",
            ["Rendezvous__Provisioning__Games__0__MetadataMaxKeys"] = "2",
            ["Rendezvous__Provisioning__Games__0__MaxListingsPerPrincipal"] = "4",
            ["Rendezvous__Provisioning__Games__0__MaxAnonymousListingsPerAddress"] = "1",
            ["Rendezvous__Provisioning__Games__0__MaxActiveJoinAttempts"] = "16",
            ["RENDEZVOUS_INTEGRATION_SIGNING_KEY"] = signingKey,
        };
        return values;
    }

    private static ProcessCapture Start(
        string assembly,
        IReadOnlyList<string> arguments,
        IReadOnlyDictionary<string, string> environment)
    {
        ProcessStartInfo start = new()
        {
            FileName = Environment.GetEnvironmentVariable("DOTNET_HOST_PATH") ?? "dotnet",
            UseShellExecute = false,
            RedirectStandardOutput = true,
            RedirectStandardError = true,
            CreateNoWindow = true,
        };
        start.ArgumentList.Add(assembly);
        foreach (string argument in arguments)
        {
            start.ArgumentList.Add(argument);
        }
        foreach (KeyValuePair<string, string> item in environment)
        {
            start.Environment[item.Key] = item.Value;
        }
        return new ProcessCapture(start);
    }

    private static async Task WaitForReadyAsync(
        string serviceUrl,
        ProcessCapture server,
        CancellationToken cancellationToken)
    {
        using HttpClient client = new() { BaseAddress = new Uri(serviceUrl) };
        while (!cancellationToken.IsCancellationRequested)
        {
            if (server.HasExited)
            {
                throw new Xunit.Sdk.XunitException($"Server exited before readiness. {server.DiagnosticText()}");
            }
            try
            {
                using HttpResponseMessage response = await client.GetAsync(
                    "health/ready",
                    cancellationToken);
                if (response.StatusCode == HttpStatusCode.OK)
                {
                    return;
                }
            }
            catch (HttpRequestException)
            {
            }
            await Task.Delay(25, cancellationToken);
        }
        cancellationToken.ThrowIfCancellationRequested();
    }

    private static string ParseServiceUrl(string line)
    {
        const string marker = "Now listening on: ";
        string value = line[(line.IndexOf(marker, StringComparison.Ordinal) + marker.Length)..].Trim();
        return new Uri(value).AbsoluteUri;
    }

    private static int ParseTrailingPort(string line)
    {
        string value = line[(line.LastIndexOf(':') + 1)..].Trim();
        return int.Parse(value, System.Globalization.CultureInfo.InvariantCulture);
    }

    private static string RepositoryRoot()
    {
        DirectoryInfo? directory = new(AppContext.BaseDirectory);
        while (directory is not null)
        {
            if (File.Exists(Path.Combine(directory.FullName, "Rendezvous.slnx")))
            {
                return directory.FullName;
            }
            directory = directory.Parent;
        }
        throw new DirectoryNotFoundException("Could not locate the Rendezvous repository root.");
    }

    private static void AssertAllowlistedEventShape(JsonElement item)
    {
        string[] forbidden = ["credential", "capability", "ticket", "token", "secret", "metadata"];
        foreach (JsonProperty property in item.EnumerateObject())
        {
            Assert.DoesNotContain(forbidden, forbiddenName =>
                property.Name.Contains(forbiddenName, StringComparison.OrdinalIgnoreCase));
        }
    }

    private sealed class ProcessCapture : IAsyncDisposable
    {
        private readonly Process _process;
        private readonly ConcurrentQueue<string> _standardOutput = new();
        private readonly ConcurrentQueue<string> _standardError = new();

        internal ProcessCapture(ProcessStartInfo start)
        {
            _process = new Process { StartInfo = start };
            _process.OutputDataReceived += (_, eventArgs) => Add(eventArgs.Data, _standardOutput);
            _process.ErrorDataReceived += (_, eventArgs) => Add(eventArgs.Data, _standardError);
            Assert.True(_process.Start(), $"Failed to start {start.FileName}.");
            _process.BeginOutputReadLine();
            _process.BeginErrorReadLine();
        }

        internal bool HasExited => _process.HasExited;

        internal async Task<JsonElement> WaitForEventAsync(
            string eventName,
            CancellationToken cancellationToken)
        {
            while (!cancellationToken.IsCancellationRequested)
            {
                foreach (JsonElement item in JsonEvents())
                {
                    if (item.TryGetProperty("event", out JsonElement eventProperty)
                        && eventProperty.GetString() == eventName)
                    {
                        return item;
                    }
                }
                if (_process.HasExited)
                {
                    throw new Xunit.Sdk.XunitException(
                        $"Process exited before event {eventName}. {DiagnosticText()}");
                }
                await Task.Delay(10, cancellationToken);
            }
            cancellationToken.ThrowIfCancellationRequested();
            throw new UnreachableException();
        }

        internal async Task<string> WaitForLineAsync(
            string marker,
            CancellationToken cancellationToken)
        {
            while (!cancellationToken.IsCancellationRequested)
            {
                string? line = AllLines().FirstOrDefault(item =>
                    item.Contains(marker, StringComparison.Ordinal));
                if (line is not null)
                {
                    return line;
                }
                if (_process.HasExited)
                {
                    throw new Xunit.Sdk.XunitException(
                        $"Process exited before output marker {marker}. {DiagnosticText()}");
                }
                await Task.Delay(10, cancellationToken);
            }
            cancellationToken.ThrowIfCancellationRequested();
            throw new UnreachableException();
        }

        internal bool HasEvent(string eventName, string status) => JsonEvents().Any(item =>
            item.TryGetProperty("event", out JsonElement eventProperty)
            && eventProperty.GetString() == eventName
            && item.TryGetProperty("status", out JsonElement statusProperty)
            && statusProperty.GetString() == status);

        internal List<JsonElement> JsonEvents()
        {
            List<JsonElement> items = [];
            foreach (string line in _standardOutput.Concat(_standardError))
            {
                try
                {
                    using JsonDocument document = JsonDocument.Parse(line);
                    items.Add(document.RootElement.Clone());
                }
                catch (JsonException)
                {
                }
            }
            return items;
        }

        internal IEnumerable<string> AllLines() => _standardOutput.Concat(_standardError);

        internal async Task<int> WaitForExitAsync(CancellationToken cancellationToken)
        {
            await _process.WaitForExitAsync(cancellationToken);
            return _process.ExitCode;
        }

        internal string DiagnosticText()
        {
            string events = string.Join(
                ',',
                JsonEvents().Select(static item =>
                    $"{item.GetProperty("event").GetString()}:{item.GetProperty("status").GetString()}"));
            return $"stdoutLines={_standardOutput.Count}; stderrLines={_standardError.Count}; events=[{events}]";
        }

        public async ValueTask DisposeAsync()
        {
            bool cleanupTimedOut = false;
            try
            {
                if (!_process.HasExited)
                {
                    _process.Kill(entireProcessTree: true);
                }
                using CancellationTokenSource cleanup = new(TimeSpan.FromSeconds(5));
                await _process.WaitForExitAsync(cleanup.Token);
            }
            catch (InvalidOperationException)
            {
            }
            catch (OperationCanceledException)
            {
                cleanupTimedOut = !_process.HasExited;
                if (cleanupTimedOut)
                {
                    try
                    {
                        _process.Kill(entireProcessTree: true);
                    }
                    catch (InvalidOperationException)
                    {
                    }
                }
            }
            finally
            {
                _process.Dispose();
            }
            if (cleanupTimedOut)
            {
                throw new Xunit.Sdk.XunitException("Child process did not exit within the cleanup deadline.");
            }
        }

        private static void Add(string? line, ConcurrentQueue<string> destination)
        {
            if (!string.IsNullOrEmpty(line))
            {
                destination.Enqueue(line);
            }
        }
    }
}