Rendezvous/tests/FinalFactory.Rendezvous.Tests/Operations/OperatorEndpointTests.cs

using System.Diagnostics;
using System.Diagnostics.Metrics;
using System.Net;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using System.Text.Json;
using FinalFactory.Rendezvous.Contracts;
using FinalFactory.Rendezvous.Server.Abuse;
using FinalFactory.Rendezvous.Server.Http;
using FinalFactory.Rendezvous.Server.JoinAttempts;
using FinalFactory.Rendezvous.Server.Observability;
using FinalFactory.Rendezvous.Server.Operations;
using FinalFactory.Rendezvous.Server.Provisioning;
using FinalFactory.Rendezvous.Server.Sessions;
using FinalFactory.Rendezvous.Server.State;
using FinalFactory.Rendezvous.Server.Transport;
using FinalFactory.Rendezvous.Tests.Observability;
using FinalFactory.Rendezvous.Tests.Provisioning;
using FinalFactory.Rendezvous.Tests.State;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Hosting.Server;
using Microsoft.AspNetCore.Hosting.Server.Features;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Routing;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;

namespace FinalFactory.Rendezvous.Tests.Operations;

[Collection(RendezvousTelemetryIsolation.Name)]
public sealed class OperatorEndpointTests
{
    [Fact]
    public async Task OperatorSurfaceSeparatesAuthenticationConfirmsActionsAndRedactsInspection()
    {
        await using OperatorTestHost host = await OperatorTestHost.StartAsync();
        List<string> telemetryData = [];
        using MeterListener meterListener = new();
        meterListener.InstrumentPublished = (instrument, listener) =>
        {
            if (instrument.Meter.Name == RendezvousTelemetry.MeterName)
            {
                listener.EnableMeasurementEvents(instrument);
            }
        };
        meterListener.SetMeasurementEventCallback<long>((instrument, value, tags, _) =>
            CaptureMeasurement(telemetryData, instrument, value, tags));
        meterListener.SetMeasurementEventCallback<int>((instrument, value, tags, _) =>
            CaptureMeasurement(telemetryData, instrument, value, tags));
        meterListener.SetMeasurementEventCallback<double>((instrument, value, tags, _) =>
            CaptureMeasurement(telemetryData, instrument, value, tags));
        meterListener.Start();
        using ActivityListener activityListener = new()
        {
            ShouldListenTo = static source => source.Name == RendezvousTelemetry.ActivitySourceName,
            Sample = static (ref ActivityCreationOptions<ActivityContext> _) =>
                ActivitySamplingResult.AllData,
            ActivityStopped = activity =>
            {
                telemetryData.Add(activity.DisplayName);
                telemetryData.AddRange(activity.TagObjects.Select(static tag => $"{tag.Key}={tag.Value}"));
            },
        };
        ActivitySource.AddActivityListener(activityListener);

        using HttpResponseMessage liveBeforeDependencies = await host.Client.GetAsync("/health/live");
        Assert.Equal(HttpStatusCode.OK, liveBeforeDependencies.StatusCode);
        using HttpResponseMessage readyBeforeUdp = await host.Client.GetAsync("/health/ready");
        Assert.Equal(HttpStatusCode.ServiceUnavailable, readyBeforeUdp.StatusCode);

        using HttpResponseMessage unauthenticated = await host.Client.GetAsync("/v1/operator/status");
        Assert.Equal(HttpStatusCode.Unauthorized, unauthenticated.StatusCode);
        AuthenticationHeaderValue challenge = Assert.Single(
            unauthenticated.Headers.WwwAuthenticate);
        Assert.Equal("Bearer", challenge.Scheme);
        Assert.Equal("realm=\"operator\"", challenge.Parameter);

        using HttpResponseMessage publisher = await SendAsync(
            host,
            HttpMethod.Get,
            "/v1/operator/status",
            host.PublisherCredential);
        Assert.Equal(HttpStatusCode.Unauthorized, publisher.StatusCode);

        using HttpResponseMessage status = await SendAsync(
            host,
            HttpMethod.Get,
            "/v1/operator/status",
            host.ReadOnlyOperatorCredential);
        Assert.Equal(HttpStatusCode.OK, status.StatusCode);
        string statusJson = await status.Content.ReadAsStringAsync();
        Assert.Contains("not-ready", statusJson, StringComparison.Ordinal);
        Assert.DoesNotContain(host.OwnerCanary, statusJson, StringComparison.Ordinal);
        Assert.DoesNotContain("203.0.113.25", statusJson, StringComparison.Ordinal);
        Assert.DoesNotContain("metadata", statusJson, StringComparison.OrdinalIgnoreCase);
        OperatorStatusResponse? operatorStatus = JsonSerializer.Deserialize<OperatorStatusResponse>(
            statusJson,
            ContractJson.Options);
        Assert.Contains(operatorStatus!.Tenants, static tenant =>
            tenant.GameId == "space-game"
            && tenant.EnvironmentId == "production"
            && tenant.Status == "enabled");
        Assert.Equal("1.0.0", operatorStatus.Compatibility.ServerVersion);
        Assert.Equal("1.0.0", operatorStatus.Compatibility.MinimumClientVersion);
        Assert.Equal(1, operatorStatus.Compatibility.MaximumClientMajorVersion);
        Assert.Equal([1], operatorStatus.Compatibility.HttpContractVersions);
        Assert.Equal([1], operatorStatus.Compatibility.UdpContractVersions);
        Assert.Equal([1], operatorStatus.Compatibility.ConnectionTicketFormatVersions);
        Assert.Equal(2, operatorStatus.Compatibility.LiteNetLibMajorVersion);
        Assert.Equal(
            "exact-per-tenant",
            operatorStatus.Compatibility.GameplayProtocolCompatibility);
        Assert.Contains(operatorStatus.SigningKeys, static key =>
            key.KeyId == OperatorTestHost.OperatorKeyId
            && key.Status == "signing"
            && key.CredentialKinds.SequenceEqual(["Operator"]));

        await host.StartUdpAsync();
        using HttpResponseMessage readyAfterUdp = await host.Client.GetAsync("/health/ready");
        Assert.Equal(HttpStatusCode.OK, readyAfterUdp.StatusCode);

        using HttpResponseMessage exception = await SendAsync(
            host,
            HttpMethod.Post,
            "/test/exception",
            host.FullOperatorCredential);
        Assert.Equal(HttpStatusCode.InternalServerError, exception.StatusCode);
        using HttpResponseMessage saturatedPublic = await host.Client.GetAsync("/test/public");
        Assert.Equal(HttpStatusCode.TooManyRequests, saturatedPublic.StatusCode);

        using HttpResponseMessage forbidden = await SendAsync(
            host,
            HttpMethod.Post,
            "/v1/operator/drain",
            host.ReadOnlyOperatorCredential,
            new BeginDrainRequest { Confirmation = "DRAIN" });
        Assert.Equal(HttpStatusCode.Forbidden, forbidden.StatusCode);

        SessionListingId listingId = host.CreateListing(host.OwnerCanary);
        using HttpResponseMessage unconfirmedListing = await SendAsync(
            host,
            HttpMethod.Post,
            "/v1/operator/listings/revoke",
            host.FullOperatorCredential,
            new RevokeListingRequest
            {
                ListingId = listingId.ToString(),
                ConfirmListingId = Guid.NewGuid().ToString("D"),
            });
        Assert.Equal(HttpStatusCode.BadRequest, unconfirmedListing.StatusCode);
        Assert.True(host.Store.GetListing(listingId, false).Succeeded);

        using HttpResponseMessage revokedListing = await SendAsync(
            host,
            HttpMethod.Post,
            "/v1/operator/listings/revoke",
            host.FullOperatorCredential,
            new RevokeListingRequest
            {
                ListingId = listingId.ToString(),
                ConfirmListingId = listingId.ToString(),
            });
        Assert.Equal(HttpStatusCode.OK, revokedListing.StatusCode);
        Assert.Equal(StoreResultCode.NotFound, host.Store.GetListing(listingId, false).Code);

        const string principalCanary = "publisher-player-canary";
        SessionListingId principalListing = host.CreateListing(principalCanary);
        using HttpResponseMessage unconfirmedPrincipal = await SendAsync(
            host,
            HttpMethod.Post,
            "/v1/operator/principals/revoke",
            host.FullOperatorCredential,
            new RevokePrincipalRequest
            {
                Subject = principalCanary,
                ConfirmSubject = "different-subject",
                LifetimeSeconds = 60,
            });
        Assert.Equal(HttpStatusCode.BadRequest, unconfirmedPrincipal.StatusCode);
        Assert.True(host.Store.GetListing(principalListing, false).Succeeded);

        using HttpResponseMessage revokedPrincipal = await SendAsync(
            host,
            HttpMethod.Post,
            "/v1/operator/principals/revoke",
            host.FullOperatorCredential,
            new RevokePrincipalRequest
            {
                Subject = principalCanary,
                ConfirmSubject = principalCanary,
                LifetimeSeconds = 60,
            });
        Assert.Equal(HttpStatusCode.OK, revokedPrincipal.StatusCode);
        OperatorActionResponse? principalResult = await revokedPrincipal.Content
            .ReadFromJsonAsync<OperatorActionResponse>(ContractJson.Options);
        Assert.Equal(1, principalResult!.AffectedResources);
        Assert.Equal(StoreResultCode.NotFound, host.Store.GetListing(principalListing, false).Code);
        StoreResult<StoredListing> blockedPublisher = host.CreateListingResult(
            principalCanary,
            out _);
        Assert.Equal(StoreResultCode.Revoked, blockedPublisher.Code);

        using HttpResponseMessage drain = await SendAsync(
            host,
            HttpMethod.Post,
            "/v1/operator/drain",
            host.FullOperatorCredential,
            new BeginDrainRequest { Confirmation = "DRAIN" });
        Assert.Equal(HttpStatusCode.OK, drain.StatusCode);
        Assert.True(host.Store.IsDraining);
        using HttpResponseMessage liveDuringDrain = await host.Client.GetAsync("/health/live");
        Assert.Equal(HttpStatusCode.OK, liveDuringDrain.StatusCode);
        using HttpResponseMessage readyDuringDrain = await host.Client.GetAsync("/health/ready");
        Assert.Equal(HttpStatusCode.ServiceUnavailable, readyDuringDrain.StatusCode);

        using HttpResponseMessage firstPublisherKeyRevocation = await SendAsync(
            host,
            HttpMethod.Post,
            "/v1/operator/keys/revoke",
            host.FullOperatorCredential,
            new RevokeSigningKeyRequest { KeyId = "key-1", ConfirmKeyId = "key-1" });
        Assert.Equal(HttpStatusCode.OK, firstPublisherKeyRevocation.StatusCode);
        using HttpResponseMessage repeatedPublisherKeyRevocation = await SendAsync(
            host,
            HttpMethod.Post,
            "/v1/operator/keys/revoke",
            host.FullOperatorCredential,
            new RevokeSigningKeyRequest { KeyId = "key-1", ConfirmKeyId = "key-1" });
        Assert.Equal(HttpStatusCode.OK, repeatedPublisherKeyRevocation.StatusCode);

        using HttpResponseMessage unconfirmedKey = await SendAsync(
            host,
            HttpMethod.Post,
            "/v1/operator/keys/revoke",
            host.FullOperatorCredential,
            new RevokeSigningKeyRequest
            {
                KeyId = OperatorTestHost.OperatorKeyId,
                ConfirmKeyId = "different-key",
            });
        Assert.Equal(HttpStatusCode.BadRequest, unconfirmedKey.StatusCode);

        using HttpResponseMessage revokedKey = await SendAsync(
            host,
            HttpMethod.Post,
            "/v1/operator/keys/revoke",
            host.FullOperatorCredential,
            new RevokeSigningKeyRequest
            {
                KeyId = OperatorTestHost.OperatorKeyId,
                ConfirmKeyId = OperatorTestHost.OperatorKeyId,
            });
        Assert.Equal(HttpStatusCode.OK, revokedKey.StatusCode);
        using HttpResponseMessage afterKeyRevocation = await SendAsync(
            host,
            HttpMethod.Get,
            "/v1/operator/status",
            host.FullOperatorCredential);
        Assert.Equal(HttpStatusCode.Unauthorized, afterKeyRevocation.StatusCode);

        string auditText = string.Join('|', host.Audit.GetEntriesForTests());
        string logText = string.Join('|', host.AuditLogger.Messages.Concat(host.AllLogs.Messages));
        Assert.DoesNotContain(host.OwnerCanary, auditText, StringComparison.Ordinal);
        Assert.DoesNotContain(principalCanary, auditText, StringComparison.Ordinal);
        Assert.DoesNotContain(listingId.ToString(), auditText, StringComparison.Ordinal);
        Assert.DoesNotContain(host.OwnerCanary, logText, StringComparison.Ordinal);
        Assert.DoesNotContain(principalCanary, logText, StringComparison.Ordinal);
        Assert.DoesNotContain("exception-secret-canary", logText, StringComparison.Ordinal);
        Assert.DoesNotContain(host.FullOperatorCredential, logText, StringComparison.Ordinal);
        meterListener.RecordObservableInstruments();
        string telemetryText = string.Join('|', telemetryData);
        Assert.DoesNotContain(host.OwnerCanary, telemetryText, StringComparison.Ordinal);
        Assert.DoesNotContain(principalCanary, telemetryText, StringComparison.Ordinal);
        Assert.DoesNotContain("exception-secret-canary", telemetryText, StringComparison.Ordinal);
        Assert.DoesNotContain(host.FullOperatorCredential, telemetryText, StringComparison.Ordinal);
        Assert.DoesNotContain(listingId.ToString(), telemetryText, StringComparison.Ordinal);
        Assert.DoesNotContain("203.0.113.25", telemetryText, StringComparison.Ordinal);
        Assert.Contains(host.Audit.GetEntriesForTests(), static entry =>
            entry.Action == "begin-drain" && entry.Result == "succeeded");
        Assert.Contains(host.Audit.GetEntriesForTests(), static entry =>
            entry.Action == "revoke-listing" && entry.Result == "rejected");
        Assert.Contains(host.Audit.GetEntriesForTests(), static entry =>
            entry.Action == "begin-drain" && entry.Result == "forbidden");
    }

    private static async Task<HttpResponseMessage> SendAsync(
        OperatorTestHost host,
        HttpMethod method,
        string path,
        string bearer,
        object? body = null)
    {
        using HttpRequestMessage request = new(method, path);
        request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", bearer);
        if (body is not null)
        {
            request.Content = JsonContent.Create(body, options: ContractJson.Options);
        }

        return await host.Client.SendAsync(request);
    }

    private static void CaptureMeasurement<T>(
        List<string> destination,
        Instrument instrument,
        T value,
        ReadOnlySpan<KeyValuePair<string, object?>> tags)
        where T : struct
    {
        destination.Add($"{instrument.Name}={value}");
        destination.AddRange(tags.ToArray().Select(static tag => $"{tag.Key}={tag.Value}"));
    }

    private sealed class OperatorTestHost : IAsyncDisposable
    {
        private readonly WebApplication _application;
        private int _listingSequence;
        private bool _udpStarted;

        private OperatorTestHost(
            WebApplication application,
            HttpClient client,
            ManualRendezvousClock clock,
            InMemoryEphemeralRendezvousStore store,
            AuditTrail audit,
            CapturingLogger<AuditTrail> auditLogger,
            CapturingLoggerProvider allLogs,
            string publisherCredential,
            string readOnlyOperatorCredential,
            string fullOperatorCredential)
        {
            _application = application;
            Client = client;
            Clock = clock;
            Store = store;
            Audit = audit;
            AuditLogger = auditLogger;
            AllLogs = allLogs;
            PublisherCredential = publisherCredential;
            ReadOnlyOperatorCredential = readOnlyOperatorCredential;
            FullOperatorCredential = fullOperatorCredential;
        }

        internal const string OperatorKeyId = "operator-key";
        internal string OwnerCanary { get; } = "publisher-owner-canary";
        internal HttpClient Client { get; }
        internal ManualRendezvousClock Clock { get; }
        internal InMemoryEphemeralRendezvousStore Store { get; }
        internal AuditTrail Audit { get; }
        internal CapturingLogger<AuditTrail> AuditLogger { get; }
        internal CapturingLoggerProvider AllLogs { get; }
        internal string PublisherCredential { get; }
        internal string ReadOnlyOperatorCredential { get; }
        internal string FullOperatorCredential { get; }

        internal static async Task<OperatorTestHost> StartAsync()
        {
            ManualRendezvousClock clock = new(ProvisioningTestData.Now);
            EphemeralStoreOptions stateOptions = new();
            InMemoryEphemeralRendezvousStore store = new(stateOptions, clock, clock);
            SigningKeyOptions publisherKey = ProvisioningTestData.CreateKey();
            SigningKeyOptions operatorKey = ProvisioningTestData.CreateKey(
                OperatorKeyId,
                "operator-secret",
                credentialKinds: [PrincipalCredentialKind.Operator],
                gameId: null,
                environmentId: null);
            ProvisioningOptions options = ProvisioningTestData.CreateOptions();
            options.SigningKeys = [publisherKey, operatorKey];
            ProvisioningRuntime provisioning = ProvisioningRuntime.Create(
                options,
                ProvisioningTestData.CreateSecrets("secret-1", "operator-secret"),
                clock.UtcNow);
            string publisherCredential = provisioning.Credentials.Issue(
                ProvisioningTestData.CreateDedicatedPublisher(),
                clock.UtcNow);
            string readOnlyCredential = provisioning.Credentials.Issue(
                new OperatorPrincipal(
                    "operator-readonly",
                    clock.UtcNow.AddMinutes(10),
                    [OperatorPermission.ReadPolicy]),
                clock.UtcNow);
            string fullCredential = provisioning.Credentials.Issue(
                new OperatorPrincipal(
                    "operator-full",
                    clock.UtcNow.AddMinutes(10),
                    Enum.GetValues<OperatorPermission>()),
                clock.UtcNow);
            CapturingLogger<AuditTrail> auditLogger = new();
            CapturingLoggerProvider allLogs = new();
            EphemeralCapabilityIssuer capabilities = new();

            WebApplicationBuilder builder = WebApplication.CreateBuilder();
            builder.WebHost.UseUrls("http://127.0.0.1:0");
            builder.Logging.ClearProviders();
            builder.Logging.SetMinimumLevel(LogLevel.Debug);
            builder.Logging.AddProvider(allLogs);
            builder.Logging.AddFilter(
                "Microsoft.AspNetCore.Diagnostics.ExceptionHandlerMiddleware",
                LogLevel.None);
            builder.Services.ConfigureHttpJsonOptions(static json =>
                ContractJson.Configure(json.SerializerOptions));
            builder.Services.Configure<RouteHandlerOptions>(static route =>
                route.ThrowOnBadRequest = true);
            builder.Services.AddProblemDetails();
            builder.Services.AddExceptionHandler<RendezvousExceptionHandler>();
            builder.Services.AddOptions<AbuseProtectionOptions>().Configure(static abuse =>
            {
                abuse.OperatorAllowedAddresses = ["127.0.0.1"];
                abuse.HttpGlobalRequestsPerWindow = 1;
                abuse.HttpOptionalRequestsPerWindow = 1;
                abuse.HttpIpPrefixRequestsPerWindow = 1;
                abuse.HttpOptionalIpPrefixRequestsPerWindow = 1;
            });
            builder.Services.AddOptions<AuditOptions>();
            builder.Services.AddOptions<UdpMediatorOptions>().Configure(static udp =>
            {
                udp.ListenAddress = "127.0.0.1";
                udp.Port = 0;
            });
            builder.Services.AddSingleton(provisioning);
            builder.Services.AddSingleton(provisioning.Policies);
            builder.Services.AddSingleton(provisioning.Credentials);
            builder.Services.AddSingleton(provisioning.PublisherAuthorization);
            builder.Services.AddSingleton(store);
            builder.Services.AddSingleton<IEphemeralRendezvousStore>(store);
            builder.Services.AddSingleton<IWallClock>(clock);
            builder.Services.AddSingleton<IMonotonicClock>(clock);
            builder.Services.AddSingleton(capabilities);
            builder.Services.AddSingleton<ISessionCapabilityService>(capabilities);
            builder.Services.AddSingleton<JoinAttemptCursorCodec>();
            builder.Services.AddSingleton<JoinAttemptService>();
            builder.Services.AddSingleton<RendezvousTelemetry>();
            builder.Services.AddSingleton<AbuseProtectionService>();
            builder.Services.AddSingleton<NatMediationProcessor>();
            builder.Services.AddSingleton<UdpMediatorService>();
            builder.Services.AddSingleton(new ProvisioningReadiness(true));
            builder.Services.AddSingleton<RendezvousReadiness>();
            builder.Services.AddSingleton<ILogger<AuditTrail>>(auditLogger);
            builder.Services.AddSingleton<AuditTrail>();
            builder.Services.AddSingleton<OperatorService>();

            WebApplication app = builder.Build();
            app.UseMiddleware<TelemetryMiddleware>();
            app.UseExceptionHandler();
            app.UseMiddleware<HttpAbuseProtectionMiddleware>();
            app.MapOperatorEndpoints();
            app.MapRendezvousHealthEndpoints();
            app.MapGet("/test/public", static () => Results.Ok()).WithName("TestPublic");
            app.MapPost(
                    "/test/exception",
                    static IResult () => throw new InvalidOperationException("exception-secret-canary"))
                .WithName("TestSecretException");
            await app.StartAsync();
            IServer server = app.Services.GetRequiredService<IServer>();
            string address = Assert.Single(server.Features.Get<IServerAddressesFeature>()!.Addresses);
            return new OperatorTestHost(
                app,
                new HttpClient { BaseAddress = new Uri(address) },
                clock,
                store,
                app.Services.GetRequiredService<AuditTrail>(),
                auditLogger,
                allLogs,
                publisherCredential,
                readOnlyCredential,
                fullCredential);
        }

        internal SessionListingId CreateListing(string owner)
        {
            StoreResult<StoredListing> result = CreateListingResult(owner, out SessionListingId listingId);
            Assert.True(result.Succeeded);
            return listingId;
        }

        internal StoreResult<StoredListing> CreateListingResult(
            string owner,
            out SessionListingId listingId)
        {
            int sequence = Interlocked.Increment(ref _listingSequence);
            listingId = new(Guid.NewGuid());
            StoreResult<StoredListing> result = Store.CreateListing(new(
                $"operator-listing-{sequence}",
                $"operator-request-{sequence}",
                new ListingDefinition
                {
                    ListingId = listingId,
                    LeaseId = new(Guid.NewGuid()),
                    Scope = new(new GameId("space-game"), new EnvironmentId("production")),
                    OwnerSubject = owner,
                    RegionId = new("eu-central"),
                    ProtocolVersion = 7,
                    BuildVersion = "1.0.0",
                    DisplayName = "Operator test listing",
                    Visibility = ListingVisibility.Public,
                    TrustMode = PublisherTrustMode.ManagedDedicated,
                    CurrentPlayers = 1,
                    MaximumPlayers = 4,
                    Metadata = new Dictionary<string, string> { ["mode"] = "online-coop" },
                    LeaseFingerprint = new("lease-fingerprint"),
                    HostPresenceHandle = new(Guid.NewGuid()),
                    HostPresenceFingerprint = new("presence-fingerprint"),
                    CapabilityDerivationSalt = new string('A', 43),
                }));
            return result;
        }

        internal async Task StartUdpAsync()
        {
            await _application.Services.GetRequiredService<UdpMediatorService>()
                .StartAsync(CancellationToken.None);
            _udpStarted = true;
        }

        public async ValueTask DisposeAsync()
        {
            Client.Dispose();
            if (_udpStarted)
            {
                await _application.Services.GetRequiredService<UdpMediatorService>()
                    .StopAsync(CancellationToken.None);
            }
            await _application.StopAsync();
            await _application.DisposeAsync();
        }
    }
}